mirror of
https://github.com/zyycn/codex-proxy-rs.git
synced 2026-11-07 00:05:49 +00:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
b206ee8271 | ||
|
|
70ff923033 | ||
|
|
a2fbc073a4 | ||
|
|
9944041dc8 | ||
|
|
18a04f1213 | ||
|
|
35988d0eb7 | ||
|
|
6673b6bd1e | ||
|
|
2a7fd08e9e | ||
|
|
7f69e81176 | ||
|
|
93dc417b79 | ||
|
|
f905e4b3fb | ||
|
|
ed3e23887b | ||
|
|
91f35d88cf | ||
|
|
f465a337e8 | ||
|
|
37465e2872 | ||
|
|
70c18dadd6 | ||
|
|
881a0eaef6 | ||
|
|
95f7a5a636 | ||
|
|
871406ab24 | ||
|
|
38a1cc6933 | ||
|
|
d38e0d4296 | ||
|
|
d12d75a037 |
@@ -0,0 +1,242 @@
|
||||
name: Verify container
|
||||
description: Scan and validate a built image with the deployment Compose configuration
|
||||
inputs:
|
||||
image:
|
||||
description: Local image tag or immutable registry digest to validate
|
||||
required: true
|
||||
platform:
|
||||
description: Native runner image platform
|
||||
required: true
|
||||
artifact-name:
|
||||
description: Unique diagnostics artifact name within the workflow run
|
||||
required: true
|
||||
runs:
|
||||
using: composite
|
||||
steps:
|
||||
- name: Define isolated runtime values
|
||||
shell: bash
|
||||
env:
|
||||
IMAGE: ${{ inputs.image }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
admin_password="$(openssl rand -hex 24)"
|
||||
postgres_password="$(openssl rand -hex 24)"
|
||||
redis_password="$(openssl rand -hex 24)"
|
||||
echo "::add-mask::$admin_password"
|
||||
echo "::add-mask::$postgres_password"
|
||||
echo "::add-mask::$redis_password"
|
||||
{
|
||||
echo "CPR_IMAGE=$IMAGE"
|
||||
echo "COMPOSE_PROJECT_NAME=cpr-check-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${RUNNER_ARCH,,}"
|
||||
echo "DIAGNOSTICS_DIR=$RUNNER_TEMP/codex-proxy-rs-diagnostics"
|
||||
echo "CI_ADMIN_PASSWORD=$admin_password"
|
||||
echo "CI_POSTGRES_PASSWORD=$postgres_password"
|
||||
echo "CI_REDIS_PASSWORD=$redis_password"
|
||||
} >> "$GITHUB_ENV"
|
||||
|
||||
- name: Scan container image
|
||||
uses: aquasecurity/trivy-action@ed142fd0673e97e23eac54620cfb913e5ce36c25 # v0.36.0
|
||||
env:
|
||||
TRIVY_PLATFORM: ${{ inputs.platform }}
|
||||
TRIVY_QUIET: 'true'
|
||||
TRIVY_SKIP_VERSION_CHECK: 'true'
|
||||
with:
|
||||
image-ref: ${{ inputs.image }}
|
||||
version: v0.72.0
|
||||
format: json
|
||||
output: trivy-results.json
|
||||
severity: HIGH,CRITICAL
|
||||
vuln-type: os,library
|
||||
scanners: vuln
|
||||
ignore-unfixed: true
|
||||
exit-code: '0'
|
||||
|
||||
- name: Generate Trivy SARIF
|
||||
shell: bash
|
||||
run: trivy convert --format sarif --output trivy-results.sarif trivy-results.json
|
||||
|
||||
- name: Upload Trivy SARIF
|
||||
if: >-
|
||||
${{ always() && hashFiles('trivy-results.sarif') != '' &&
|
||||
(github.event_name != 'pull_request' ||
|
||||
github.event.pull_request.head.repo.full_name == github.repository) }}
|
||||
uses: github/codeql-action/upload-sarif@cdf488f595d80d6e07e03d4674febd5ab45fa938 # v4.37.9
|
||||
with:
|
||||
sarif_file: trivy-results.sarif
|
||||
category: runtime-container-${{ inputs.platform }}
|
||||
|
||||
- name: Enforce Trivy vulnerability gate
|
||||
shell: bash
|
||||
run: trivy convert --format table --severity HIGH,CRITICAL --exit-code 1 trivy-results.json
|
||||
|
||||
- name: Prepare isolated runtime
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
umask 0077
|
||||
# Bind roots hand off ownership to the image-specific database users.
|
||||
# Sticky directories keep that initialization compatible with user-namespaced runners.
|
||||
install -d -m 1777 .runtime/postgres .runtime/redis
|
||||
mkdir -p "$DIAGNOSTICS_DIR"
|
||||
sudo install -d -m 0770 -o "$(id -u)" -g 10001 \
|
||||
.runtime/data .runtime/logs
|
||||
ruby <<'RUBY'
|
||||
path = "deploy/config.example.yaml"
|
||||
content = File.read(path)
|
||||
replacements = {
|
||||
"password: &postgres_password ''" => "password: &postgres_password '#{ENV.fetch("CI_POSTGRES_PASSWORD")}'",
|
||||
"password: &redis_password ''" => "password: &redis_password '#{ENV.fetch("CI_REDIS_PASSWORD")}'",
|
||||
"default_password: ''" => "default_password: '#{ENV.fetch("CI_ADMIN_PASSWORD")}'",
|
||||
}
|
||||
replacements.each do |placeholder, configured|
|
||||
raise "missing config placeholder: #{placeholder}" unless content.sub!(placeholder, configured)
|
||||
end
|
||||
File.write("deploy/config.yaml", content)
|
||||
RUBY
|
||||
sudo chown "$(id -u):10001" deploy/config.yaml
|
||||
chmod 0640 deploy/config.yaml
|
||||
|
||||
- name: Validate Docker Compose
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
docker compose --project-name "$COMPOSE_PROJECT_NAME" \
|
||||
-f deploy/compose.yaml config --quiet
|
||||
|
||||
- name: Configure Redis host memory overcommit
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
sudo sysctl -w vm.overcommit_memory=1
|
||||
test "$(sysctl -n vm.overcommit_memory)" = 1
|
||||
|
||||
- name: Verify PostgreSQL and Redis volume persistence
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
compose=(docker compose --project-name "$COMPOSE_PROJECT_NAME" \
|
||||
-f deploy/compose.yaml)
|
||||
"${compose[@]}" up -d --no-build --wait postgres redis
|
||||
test "$("${compose[@]}" exec -T -e PGPASSWORD="$CI_POSTGRES_PASSWORD" postgres \
|
||||
psql -At -U codex_proxy -d codex_proxy -c "SHOW TimeZone;" | tr -d '\r')" = Asia/Shanghai
|
||||
"${compose[@]}" exec -T -e PGPASSWORD="$CI_POSTGRES_PASSWORD" postgres \
|
||||
psql -v ON_ERROR_STOP=1 -U codex_proxy -d codex_proxy -c \
|
||||
"CREATE TABLE cpr_ci_persistence_probe (value text PRIMARY KEY); INSERT INTO cpr_ci_persistence_probe VALUES ('persisted');"
|
||||
"${compose[@]}" exec -T -e REDISCLI_AUTH="$CI_REDIS_PASSWORD" redis \
|
||||
redis-cli SET cpr:ci:persistence-probe persisted
|
||||
"${compose[@]}" up -d --no-build --wait --force-recreate postgres redis
|
||||
test "$("${compose[@]}" exec -T -e PGPASSWORD="$CI_POSTGRES_PASSWORD" postgres \
|
||||
psql -At -U codex_proxy -d codex_proxy -c \
|
||||
"SELECT value FROM cpr_ci_persistence_probe;")" = persisted
|
||||
test "$("${compose[@]}" exec -T -e REDISCLI_AUTH="$CI_REDIS_PASSWORD" redis \
|
||||
redis-cli --raw GET cpr:ci:persistence-probe | tr -d '\r')" = persisted
|
||||
"${compose[@]}" exec -T -e PGPASSWORD="$CI_POSTGRES_PASSWORD" postgres \
|
||||
psql -v ON_ERROR_STOP=1 -U codex_proxy -d codex_proxy -c \
|
||||
"DROP TABLE cpr_ci_persistence_probe;"
|
||||
"${compose[@]}" exec -T -e REDISCLI_AUTH="$CI_REDIS_PASSWORD" redis \
|
||||
redis-cli DEL cpr:ci:persistence-probe
|
||||
|
||||
- name: Start and verify application
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
compose=(docker compose --project-name "$COMPOSE_PROJECT_NAME" \
|
||||
-f deploy/compose.yaml)
|
||||
"${compose[@]}" up -d --no-build --wait codex-proxy-rs
|
||||
status="$(curl --silent --show-error --output /dev/null --write-out '%{http_code}' \
|
||||
"http://127.0.0.1:8080/healthz")"
|
||||
test "$status" = 204
|
||||
test "$("${compose[@]}" exec -T codex-proxy-rs id -u | tr -d '\r')" = 10001
|
||||
test "$("${compose[@]}" exec -T codex-proxy-rs id -g | tr -d '\r')" = 10001
|
||||
test "$("${compose[@]}" exec -T codex-proxy-rs date +%z | tr -d '\r')" = +0800
|
||||
test "$("${compose[@]}" exec -T codex-proxy-rs cat /etc/timezone | tr -d '\r')" = Asia/Shanghai
|
||||
"${compose[@]}" exec -T codex-proxy-rs sh -c \
|
||||
'touch /app/.runtime/data/.ci-write /app/.runtime/logs/.ci-write && rm /app/.runtime/data/.ci-write /app/.runtime/logs/.ci-write'
|
||||
|
||||
- name: Verify clean logs and graceful shutdown
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
compose=(docker compose --project-name "$COMPOSE_PROJECT_NAME" \
|
||||
-f deploy/compose.yaml)
|
||||
"${compose[@]}" stop --timeout 30 codex-proxy-rs
|
||||
container_id="$("${compose[@]}" ps -a -q codex-proxy-rs)"
|
||||
test -n "$container_id"
|
||||
test "$(docker inspect --format '{{.State.ExitCode}}' "$container_id")" = 0
|
||||
"${compose[@]}" logs --no-color --timestamps codex-proxy-rs \
|
||||
> "$DIAGNOSTICS_DIR/application.log"
|
||||
"${compose[@]}" logs --no-color --timestamps postgres \
|
||||
> "$DIAGNOSTICS_DIR/postgres.log"
|
||||
"${compose[@]}" logs --no-color --timestamps redis \
|
||||
> "$DIAGNOSTICS_DIR/redis.log"
|
||||
if grep --extended-regexp --ignore-case \
|
||||
'"level"[[:space:]]*:[[:space:]]*"(warn|error|fatal|panic)"|(^|[^[:alnum:]_])panic([^[:alnum:]_]|$)' \
|
||||
"$DIAGNOSTICS_DIR/application.log" \
|
||||
|| grep --extended-regexp --ignore-case \
|
||||
'(^|[[:space:]])(warning|error|fatal|panic):' \
|
||||
"$DIAGNOSTICS_DIR/postgres.log" \
|
||||
|| grep --extended-regexp --ignore-case \
|
||||
'[0-9]+:[A-Z][[:space:]].*[[:space:]]#[[:space:]]|(^|[|[:space:]])(warning|error|fatal|panic)(:|[[:space:]])' \
|
||||
"$DIAGNOSTICS_DIR/redis.log"; then
|
||||
echo "container logs contain a warning or error severity" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
- name: Collect container diagnostics
|
||||
if: always()
|
||||
shell: bash
|
||||
run: |
|
||||
set +e
|
||||
mkdir -p "$DIAGNOSTICS_DIR"
|
||||
raw_dir="$(mktemp -d)"
|
||||
compose=(docker compose --project-name "$COMPOSE_PROJECT_NAME" \
|
||||
-f deploy/compose.yaml)
|
||||
"${compose[@]}" config > "$raw_dir/compose-config.yaml" 2>&1
|
||||
"${compose[@]}" ps -a > "$raw_dir/compose-ps.txt" 2>&1
|
||||
"${compose[@]}" logs --no-color --timestamps > "$raw_dir/compose.log" 2>&1
|
||||
mapfile -t container_ids < <("${compose[@]}" ps -a -q)
|
||||
if [ "${#container_ids[@]}" -gt 0 ]; then
|
||||
docker inspect "${container_ids[@]}" > "$raw_dir/docker-inspect.json" 2>&1
|
||||
fi
|
||||
RAW_DIR="$raw_dir" OUTPUT_DIR="$DIAGNOSTICS_DIR" ruby <<'RUBY'
|
||||
raw_dir = ENV.fetch("RAW_DIR")
|
||||
output_dir = ENV.fetch("OUTPUT_DIR")
|
||||
secrets = [
|
||||
ENV.fetch("CI_ADMIN_PASSWORD"),
|
||||
ENV.fetch("CI_POSTGRES_PASSWORD"),
|
||||
ENV.fetch("CI_REDIS_PASSWORD"),
|
||||
]
|
||||
encoded_secrets = secrets.map do |secret|
|
||||
secret.encode("UTF-8").bytes.map do |byte|
|
||||
character = byte.chr
|
||||
character.match?(/[A-Za-z0-9]/) ? character : format("%%%02X", byte)
|
||||
end.join
|
||||
end
|
||||
|
||||
Dir.children(raw_dir).each do |name|
|
||||
content = File.binread(File.join(raw_dir, name))
|
||||
(secrets + encoded_secrets).each do |secret|
|
||||
content = content.gsub(secret, "<redacted>")
|
||||
end
|
||||
File.binwrite(File.join(output_dir, name), content)
|
||||
end
|
||||
RUBY
|
||||
rm -rf "$raw_dir"
|
||||
cp trivy-results.json trivy-results.sarif "$DIAGNOSTICS_DIR/" 2>/dev/null || true
|
||||
|
||||
- name: Upload container diagnostics
|
||||
if: always()
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: ${{ inputs.artifact-name }}
|
||||
path: ${{ runner.temp }}/codex-proxy-rs-diagnostics
|
||||
if-no-files-found: warn
|
||||
retention-days: 7
|
||||
|
||||
- name: Remove isolated runtime
|
||||
if: always()
|
||||
shell: bash
|
||||
run: |
|
||||
docker compose --project-name "$COMPOSE_PROJECT_NAME" \
|
||||
-f deploy/compose.yaml down --remove-orphans || true
|
||||
sudo rm -rf "$GITHUB_WORKSPACE/.runtime" "$GITHUB_WORKSPACE/deploy/config.yaml"
|
||||
@@ -17,36 +17,44 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 45
|
||||
env:
|
||||
COMPOSE_PROJECT_NAME: cpr-ci-${{ github.run_id }}-${{ github.run_attempt }}
|
||||
CPR_IMAGE: codex-proxy-rs:ci-${{ inputs.ref }}
|
||||
TRIVY_QUIET: 'true'
|
||||
TRIVY_SKIP_VERSION_CHECK: 'true'
|
||||
steps:
|
||||
- name: Define isolated runtime values
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
admin_password="$(openssl rand -hex 24)"
|
||||
postgres_password="$(openssl rand -hex 24)"
|
||||
redis_password="$(openssl rand -hex 24)"
|
||||
echo "::add-mask::$admin_password"
|
||||
echo "::add-mask::$postgres_password"
|
||||
echo "::add-mask::$redis_password"
|
||||
{
|
||||
echo "DIAGNOSTICS_DIR=$RUNNER_TEMP/codex-proxy-rs-diagnostics"
|
||||
echo "CI_ADMIN_PASSWORD=$admin_password"
|
||||
echo "CI_POSTGRES_PASSWORD=$postgres_password"
|
||||
echo "CI_REDIS_PASSWORD=$redis_password"
|
||||
} >> "$GITHUB_ENV"
|
||||
|
||||
- name: Checkout
|
||||
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
with:
|
||||
ref: ${{ inputs.ref }}
|
||||
|
||||
- name: Setup Docker Buildx
|
||||
id: buildx
|
||||
uses: docker/setup-buildx-action@37fe631027851001ddb9b187196cc803df7f5f0e # v4.3.0
|
||||
|
||||
- name: Cache Cargo mounts
|
||||
id: cargo-cache
|
||||
uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
|
||||
with:
|
||||
path: ${{ runner.temp }}/container-cargo
|
||||
key: container-cargo-v1-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('backend/Cargo.lock', 'backend/**/Cargo.toml', 'deploy/Dockerfile') }}-${{ inputs.ref }}
|
||||
restore-keys: |
|
||||
container-cargo-v1-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('backend/Cargo.lock', 'backend/**/Cargo.toml', 'deploy/Dockerfile') }}-
|
||||
container-cargo-v1-${{ runner.os }}-${{ runner.arch }}-
|
||||
|
||||
# The GHA layer exporter does not persist RUN --mount=type=cache contents.
|
||||
# Extraction runs before actions/cache saves its directories in post steps.
|
||||
- name: Restore Cargo mounts to BuildKit
|
||||
uses: reproducible-containers/buildkit-cache-dance@5422eac04292c961a382e0f584ea0f03ad9da723 # v3.4.0
|
||||
with:
|
||||
builder: ${{ steps.buildx.outputs.name }}
|
||||
scratch-dir: ${{ runner.temp }}/container-cache-dance
|
||||
cache-map: |
|
||||
{
|
||||
"${{ runner.temp }}/container-cargo/registry": "/usr/local/cargo/registry",
|
||||
"${{ runner.temp }}/container-cargo/git": "/usr/local/cargo/git",
|
||||
"${{ runner.temp }}/container-cargo/target": "/app/backend/target"
|
||||
}
|
||||
skip-extraction: ${{ steps.cargo-cache.outputs.cache-hit }}
|
||||
|
||||
- name: Build runtime image
|
||||
uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0
|
||||
with:
|
||||
@@ -61,209 +69,9 @@ jobs:
|
||||
cache-from: type=gha,scope=runtime-ci
|
||||
cache-to: type=gha,mode=max,scope=runtime-ci
|
||||
|
||||
- name: Generate Trivy SARIF
|
||||
uses: aquasecurity/trivy-action@ed142fd0673e97e23eac54620cfb913e5ce36c25 # v0.36.0
|
||||
- name: Verify runtime image
|
||||
uses: ./.github/actions/verify-container
|
||||
with:
|
||||
image-ref: ${{ env.CPR_IMAGE }}
|
||||
version: v0.72.0
|
||||
format: sarif
|
||||
output: trivy-results.sarif
|
||||
severity: HIGH,CRITICAL
|
||||
vuln-type: os,library
|
||||
scanners: vuln
|
||||
ignore-unfixed: true
|
||||
exit-code: '0'
|
||||
|
||||
- name: Upload Trivy SARIF
|
||||
if: >-
|
||||
${{ hashFiles('trivy-results.sarif') != '' &&
|
||||
(github.event_name != 'pull_request' ||
|
||||
github.event.pull_request.head.repo.full_name == github.repository) }}
|
||||
uses: github/codeql-action/upload-sarif@cdf488f595d80d6e07e03d4674febd5ab45fa938 # v4.37.9
|
||||
with:
|
||||
sarif_file: trivy-results.sarif
|
||||
category: runtime-container
|
||||
|
||||
- name: Enforce Trivy vulnerability gate
|
||||
uses: aquasecurity/trivy-action@ed142fd0673e97e23eac54620cfb913e5ce36c25 # v0.36.0
|
||||
with:
|
||||
image-ref: ${{ env.CPR_IMAGE }}
|
||||
version: v0.72.0
|
||||
format: table
|
||||
severity: HIGH,CRITICAL
|
||||
vuln-type: os,library
|
||||
scanners: vuln
|
||||
ignore-unfixed: true
|
||||
exit-code: '1'
|
||||
|
||||
- name: Prepare isolated runtime
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
umask 0077
|
||||
# Bind roots hand off ownership to the image-specific database users.
|
||||
# Sticky directories keep that initialization compatible with user-namespaced runners.
|
||||
install -d -m 1777 .runtime/postgres .runtime/redis
|
||||
mkdir -p "$DIAGNOSTICS_DIR"
|
||||
sudo install -d -m 0770 -o "$(id -u)" -g 10001 \
|
||||
.runtime/data .runtime/logs
|
||||
ruby <<'RUBY'
|
||||
path = "deploy/config.example.yaml"
|
||||
content = File.read(path)
|
||||
replacements = {
|
||||
"password: &postgres_password ''" => "password: &postgres_password '#{ENV.fetch("CI_POSTGRES_PASSWORD")}'",
|
||||
"password: &redis_password ''" => "password: &redis_password '#{ENV.fetch("CI_REDIS_PASSWORD")}'",
|
||||
"default_password: ''" => "default_password: '#{ENV.fetch("CI_ADMIN_PASSWORD")}'",
|
||||
}
|
||||
replacements.each do |placeholder, configured|
|
||||
raise "missing config placeholder: #{placeholder}" unless content.sub!(placeholder, configured)
|
||||
end
|
||||
File.write("deploy/config.yaml", content)
|
||||
RUBY
|
||||
sudo chown "$(id -u):10001" deploy/config.yaml
|
||||
chmod 0640 deploy/config.yaml
|
||||
|
||||
- name: Validate Docker Compose
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
docker compose --project-name "$COMPOSE_PROJECT_NAME" \
|
||||
-f deploy/compose.yaml config --quiet
|
||||
|
||||
- name: Configure Redis host memory overcommit
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
sudo sysctl -w vm.overcommit_memory=1
|
||||
test "$(sysctl -n vm.overcommit_memory)" = 1
|
||||
|
||||
- name: Verify PostgreSQL and Redis volume persistence
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
compose=(docker compose --project-name "$COMPOSE_PROJECT_NAME" \
|
||||
-f deploy/compose.yaml)
|
||||
"${compose[@]}" up -d --no-build --wait postgres redis
|
||||
test "$("${compose[@]}" exec -T -e PGPASSWORD="$CI_POSTGRES_PASSWORD" postgres \
|
||||
psql -At -U codex_proxy -d codex_proxy -c "SHOW TimeZone;" | tr -d '\r')" = Asia/Shanghai
|
||||
"${compose[@]}" exec -T -e PGPASSWORD="$CI_POSTGRES_PASSWORD" postgres \
|
||||
psql -v ON_ERROR_STOP=1 -U codex_proxy -d codex_proxy -c \
|
||||
"CREATE TABLE cpr_ci_persistence_probe (value text PRIMARY KEY); INSERT INTO cpr_ci_persistence_probe VALUES ('persisted');"
|
||||
"${compose[@]}" exec -T -e REDISCLI_AUTH="$CI_REDIS_PASSWORD" redis \
|
||||
redis-cli SET cpr:ci:persistence-probe persisted
|
||||
"${compose[@]}" up -d --no-build --wait --force-recreate postgres redis
|
||||
test "$("${compose[@]}" exec -T -e PGPASSWORD="$CI_POSTGRES_PASSWORD" postgres \
|
||||
psql -At -U codex_proxy -d codex_proxy -c \
|
||||
"SELECT value FROM cpr_ci_persistence_probe;")" = persisted
|
||||
test "$("${compose[@]}" exec -T -e REDISCLI_AUTH="$CI_REDIS_PASSWORD" redis \
|
||||
redis-cli --raw GET cpr:ci:persistence-probe | tr -d '\r')" = persisted
|
||||
"${compose[@]}" exec -T -e PGPASSWORD="$CI_POSTGRES_PASSWORD" postgres \
|
||||
psql -v ON_ERROR_STOP=1 -U codex_proxy -d codex_proxy -c \
|
||||
"DROP TABLE cpr_ci_persistence_probe;"
|
||||
"${compose[@]}" exec -T -e REDISCLI_AUTH="$CI_REDIS_PASSWORD" redis \
|
||||
redis-cli DEL cpr:ci:persistence-probe
|
||||
|
||||
- name: Start and verify application
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
compose=(docker compose --project-name "$COMPOSE_PROJECT_NAME" \
|
||||
-f deploy/compose.yaml)
|
||||
"${compose[@]}" up -d --no-build --wait codex-proxy-rs
|
||||
status="$(curl --silent --show-error --output /dev/null --write-out '%{http_code}' \
|
||||
"http://127.0.0.1:8080/healthz")"
|
||||
test "$status" = 204
|
||||
test "$("${compose[@]}" exec -T codex-proxy-rs id -u | tr -d '\r')" = 10001
|
||||
test "$("${compose[@]}" exec -T codex-proxy-rs id -g | tr -d '\r')" = 10001
|
||||
test "$("${compose[@]}" exec -T codex-proxy-rs date +%z | tr -d '\r')" = +0800
|
||||
test "$("${compose[@]}" exec -T codex-proxy-rs cat /etc/timezone | tr -d '\r')" = Asia/Shanghai
|
||||
"${compose[@]}" exec -T codex-proxy-rs sh -c \
|
||||
'touch /app/.runtime/data/.ci-write /app/.runtime/logs/.ci-write && rm /app/.runtime/data/.ci-write /app/.runtime/logs/.ci-write'
|
||||
|
||||
- name: Verify clean logs and graceful shutdown
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
compose=(docker compose --project-name "$COMPOSE_PROJECT_NAME" \
|
||||
-f deploy/compose.yaml)
|
||||
"${compose[@]}" stop --timeout 30 codex-proxy-rs
|
||||
container_id="$("${compose[@]}" ps -a -q codex-proxy-rs)"
|
||||
test -n "$container_id"
|
||||
test "$(docker inspect --format '{{.State.ExitCode}}' "$container_id")" = 0
|
||||
"${compose[@]}" logs --no-color --timestamps codex-proxy-rs \
|
||||
> "$DIAGNOSTICS_DIR/application.log"
|
||||
"${compose[@]}" logs --no-color --timestamps postgres \
|
||||
> "$DIAGNOSTICS_DIR/postgres.log"
|
||||
"${compose[@]}" logs --no-color --timestamps redis \
|
||||
> "$DIAGNOSTICS_DIR/redis.log"
|
||||
if grep --extended-regexp --ignore-case \
|
||||
'"level"[[:space:]]*:[[:space:]]*"(warn|error|fatal|panic)"|(^|[^[:alnum:]_])panic([^[:alnum:]_]|$)' \
|
||||
"$DIAGNOSTICS_DIR/application.log" \
|
||||
|| grep --extended-regexp --ignore-case \
|
||||
'(^|[[:space:]])(warning|error|fatal|panic):' \
|
||||
"$DIAGNOSTICS_DIR/postgres.log" \
|
||||
|| grep --extended-regexp --ignore-case \
|
||||
'[0-9]+:[A-Z][[:space:]].*[[:space:]]#[[:space:]]|(^|[|[:space:]])(warning|error|fatal|panic)(:|[[:space:]])' \
|
||||
"$DIAGNOSTICS_DIR/redis.log"; then
|
||||
echo "container logs contain a warning or error severity" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
- name: Collect container diagnostics
|
||||
if: always()
|
||||
shell: bash
|
||||
run: |
|
||||
set +e
|
||||
mkdir -p "$DIAGNOSTICS_DIR"
|
||||
raw_dir="$(mktemp -d)"
|
||||
compose=(docker compose --project-name "$COMPOSE_PROJECT_NAME" \
|
||||
-f deploy/compose.yaml)
|
||||
"${compose[@]}" config > "$raw_dir/compose-config.yaml" 2>&1
|
||||
"${compose[@]}" ps -a > "$raw_dir/compose-ps.txt" 2>&1
|
||||
"${compose[@]}" logs --no-color --timestamps > "$raw_dir/compose.log" 2>&1
|
||||
mapfile -t container_ids < <("${compose[@]}" ps -a -q)
|
||||
if [ "${#container_ids[@]}" -gt 0 ]; then
|
||||
docker inspect "${container_ids[@]}" > "$raw_dir/docker-inspect.json" 2>&1
|
||||
fi
|
||||
RAW_DIR="$raw_dir" OUTPUT_DIR="$DIAGNOSTICS_DIR" ruby <<'RUBY'
|
||||
raw_dir = ENV.fetch("RAW_DIR")
|
||||
output_dir = ENV.fetch("OUTPUT_DIR")
|
||||
secrets = [
|
||||
ENV.fetch("CI_ADMIN_PASSWORD"),
|
||||
ENV.fetch("CI_POSTGRES_PASSWORD"),
|
||||
ENV.fetch("CI_REDIS_PASSWORD"),
|
||||
]
|
||||
encoded_secrets = secrets.map do |secret|
|
||||
secret.encode("UTF-8").bytes.map do |byte|
|
||||
character = byte.chr
|
||||
character.match?(/[A-Za-z0-9]/) ? character : format("%%%02X", byte)
|
||||
end.join
|
||||
end
|
||||
|
||||
Dir.children(raw_dir).each do |name|
|
||||
content = File.binread(File.join(raw_dir, name))
|
||||
(secrets + encoded_secrets).each do |secret|
|
||||
content = content.gsub(secret, "<redacted>")
|
||||
end
|
||||
File.binwrite(File.join(output_dir, name), content)
|
||||
end
|
||||
RUBY
|
||||
rm -rf "$raw_dir"
|
||||
cp trivy-results.sarif "$DIAGNOSTICS_DIR/" 2>/dev/null || true
|
||||
|
||||
- name: Upload container diagnostics
|
||||
if: always()
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: container-diagnostics-${{ github.run_id }}-${{ github.run_attempt }}
|
||||
path: ${{ runner.temp }}/codex-proxy-rs-diagnostics
|
||||
if-no-files-found: warn
|
||||
retention-days: 7
|
||||
|
||||
- name: Remove isolated runtime
|
||||
if: always()
|
||||
shell: bash
|
||||
run: |
|
||||
docker compose --project-name "$COMPOSE_PROJECT_NAME" \
|
||||
-f deploy/compose.yaml down --remove-orphans || true
|
||||
sudo rm -rf "$GITHUB_WORKSPACE/.runtime" "$GITHUB_WORKSPACE/deploy/config.yaml"
|
||||
image: ${{ env.CPR_IMAGE }}
|
||||
platform: linux/amd64
|
||||
artifact-name: container-diagnostics-${{ github.run_id }}-${{ github.run_attempt }}-amd64
|
||||
|
||||
@@ -50,6 +50,7 @@ jobs:
|
||||
- 'deploy/**'
|
||||
- '.dockerignore'
|
||||
- 'release/version.yaml'
|
||||
- '.github/actions/**'
|
||||
- '.github/workflows/_container.yml'
|
||||
- '.github/workflows/ci.yml'
|
||||
|
||||
|
||||
@@ -163,15 +163,6 @@ jobs:
|
||||
ref: ${{ needs.release-info.outputs.git_sha }}
|
||||
upload_frontend: true
|
||||
|
||||
container:
|
||||
needs: [release-info]
|
||||
permissions:
|
||||
contents: read
|
||||
security-events: write
|
||||
uses: ./.github/workflows/_container.yml
|
||||
with:
|
||||
ref: ${{ needs.release-info.outputs.git_sha }}
|
||||
|
||||
build-docker-binary:
|
||||
name: build-docker-binary (${{ matrix.os }}-${{ matrix.arch }})
|
||||
needs: release-info
|
||||
@@ -194,8 +185,9 @@ jobs:
|
||||
~/.cargo/registry
|
||||
~/.cargo/git
|
||||
backend/target
|
||||
key: cargo-v2-${{ runner.os }}-${{ matrix.target }}-${{ hashFiles('backend/Cargo.lock', 'backend/rust-toolchain.toml', 'backend/**/Cargo.toml', 'backend/.cargo/config.toml') }}
|
||||
key: cargo-v2-${{ runner.os }}-${{ matrix.target }}-${{ hashFiles('backend/Cargo.lock', 'backend/rust-toolchain.toml', 'backend/**/Cargo.toml', 'backend/.cargo/config.toml') }}-${{ needs.release-info.outputs.git_sha }}
|
||||
restore-keys: |
|
||||
cargo-v2-${{ runner.os }}-${{ matrix.target }}-${{ hashFiles('backend/Cargo.lock', 'backend/rust-toolchain.toml', 'backend/**/Cargo.toml', 'backend/.cargo/config.toml') }}
|
||||
cargo-v2-${{ runner.os }}-${{ matrix.target }}-
|
||||
|
||||
- name: Build binary
|
||||
@@ -259,8 +251,9 @@ jobs:
|
||||
~/.cargo/registry
|
||||
~/.cargo/git
|
||||
backend/target
|
||||
key: cargo-v2-${{ runner.os }}-${{ matrix.target }}-${{ hashFiles('backend/Cargo.lock', 'backend/rust-toolchain.toml', 'backend/**/Cargo.toml', 'backend/.cargo/config.toml') }}
|
||||
key: cargo-v2-${{ runner.os }}-${{ matrix.target }}-${{ hashFiles('backend/Cargo.lock', 'backend/rust-toolchain.toml', 'backend/**/Cargo.toml', 'backend/.cargo/config.toml') }}-${{ needs.release-info.outputs.git_sha }}
|
||||
restore-keys: |
|
||||
cargo-v2-${{ runner.os }}-${{ matrix.target }}-${{ hashFiles('backend/Cargo.lock', 'backend/rust-toolchain.toml', 'backend/**/Cargo.toml', 'backend/.cargo/config.toml') }}
|
||||
cargo-v2-${{ runner.os }}-${{ matrix.target }}-
|
||||
|
||||
- name: Build binary
|
||||
@@ -408,6 +401,7 @@ jobs:
|
||||
contents: read
|
||||
packages: write
|
||||
id-token: write
|
||||
security-events: write
|
||||
env:
|
||||
TRIVY_QUIET: 'true'
|
||||
TRIVY_SKIP_VERSION_CHECK: 'true'
|
||||
@@ -491,22 +485,34 @@ jobs:
|
||||
org.opencontainers.image.source=https://github.com/zyycn/codex-proxy-rs
|
||||
tags: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:sha-${{ needs.release-info.outputs.git_sha }}-${{ matrix.arch }}
|
||||
|
||||
- name: Scan exact app image digest
|
||||
uses: aquasecurity/trivy-action@ed142fd0673e97e23eac54620cfb913e5ce36c25 # v0.36.0
|
||||
env:
|
||||
TRIVY_PLATFORM: ${{ matrix.dockerPlatform }}
|
||||
- name: Verify exact app image digest
|
||||
uses: ./.github/actions/verify-container
|
||||
with:
|
||||
image-ref: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}@${{ steps.build-and-push.outputs.digest }}
|
||||
version: v0.72.0
|
||||
format: table
|
||||
severity: HIGH,CRITICAL
|
||||
vuln-type: os,library
|
||||
scanners: vuln
|
||||
ignore-unfixed: true
|
||||
exit-code: '1'
|
||||
image: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}@${{ steps.build-and-push.outputs.digest }}
|
||||
platform: ${{ matrix.dockerPlatform }}
|
||||
artifact-name: container-diagnostics-${{ github.run_id }}-${{ github.run_attempt }}-${{ matrix.arch }}
|
||||
|
||||
- name: Record verified image digest
|
||||
shell: bash
|
||||
env:
|
||||
IMAGE_DIGEST: ${{ steps.build-and-push.outputs.digest }}
|
||||
IMAGE_ARCH: ${{ matrix.arch }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
[[ "$IMAGE_DIGEST" =~ ^sha256:[a-f0-9]{64}$ ]]
|
||||
mkdir -p dist/digests
|
||||
printf '%s\n' "$IMAGE_DIGEST" > "dist/digests/linux-${IMAGE_ARCH}.txt"
|
||||
|
||||
- name: Upload verified image digest
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: image-digest-${{ matrix.os }}-${{ matrix.arch }}
|
||||
path: dist/digests/*.txt
|
||||
if-no-files-found: error
|
||||
retention-days: 1
|
||||
|
||||
publish:
|
||||
needs: [release-info, package-assets, build-docker-image, container]
|
||||
needs: [release-info, package-assets, build-docker-image]
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 20
|
||||
permissions:
|
||||
@@ -527,6 +533,13 @@ jobs:
|
||||
name: release-assets
|
||||
path: dist/release
|
||||
|
||||
- name: Download verified image digests
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
with:
|
||||
pattern: image-digest-*
|
||||
path: dist/digests
|
||||
merge-multiple: true
|
||||
|
||||
- name: Setup Docker Buildx
|
||||
uses: docker/setup-buildx-action@37fe631027851001ddb9b187196cc803df7f5f0e # v4.3.0
|
||||
|
||||
@@ -558,7 +571,12 @@ jobs:
|
||||
platform="${platform//[[:space:]]/}"
|
||||
[ -n "${platform}" ] || continue
|
||||
arch="${platform##*/}"
|
||||
sources+=("${image}:sha-${GIT_SHA}-${arch}")
|
||||
digest="$(cat "dist/digests/linux-${arch}.txt")"
|
||||
if [[ ! "$digest" =~ ^sha256:[a-f0-9]{64}$ ]]; then
|
||||
echo "invalid verified image digest for ${platform}" >&2
|
||||
exit 1
|
||||
fi
|
||||
sources+=("${image}@${digest}")
|
||||
done
|
||||
|
||||
if [ "${#sources[@]}" -eq 0 ]; then
|
||||
@@ -626,7 +644,7 @@ jobs:
|
||||
cosign sign --yes "${images[@]}"
|
||||
|
||||
- name: Publish GitHub Release
|
||||
uses: softprops/action-gh-release@3d0d9888cb7fd7b750713d6e236d1fcb99157228 # v3.0.2
|
||||
uses: softprops/action-gh-release@efb35369e0ad2afab669f228072c1b0d510eae64 # v3.0.3
|
||||
with:
|
||||
tag_name: ${{ needs.release-info.outputs.tag }}
|
||||
generate_release_notes: true
|
||||
|
||||
@@ -5,27 +5,31 @@
|
||||
|
||||
# Codex Proxy RS
|
||||
|
||||
集中管理 OpenAI、xAI 账号,为 Codex 和兼容客户端提供统一的访问地址与密钥。
|
||||
面向 Codex 的自托管多账号 AI 网关。
|
||||
|
||||
[](https://github.com/zyycn/codex-proxy-rs/actions/workflows/ci.yml)
|
||||
[](https://github.com/zyycn/codex-proxy-rs/releases)
|
||||
[](https://github.com/zyycn/codex-proxy-rs/pkgs/container/codex-proxy-rs)
|
||||
[](https://opensource.org/license/mit)
|
||||
|
||||
[快速开始](#快速开始) · [客户端接入](#客户端接入) · [常见问题](#常见问题) · [部署说明](deploy/README.md)
|
||||
[功能](#功能) · [快速开始](#快速开始) · [客户端接入](#客户端接入) · [文档](#文档)
|
||||
|
||||
</div>
|
||||
|
||||
将 OpenAI、xAI 账号接入同一个服务,通过网页管理账号、分配客户端密钥和查看使用情况。
|
||||
客户端使用统一的访问地址与代理密钥,上游账号凭据由服务端管理。
|
||||
|
||||
## 功能
|
||||
|
||||
- 在网页中添加账号,查看额度、请求记录和费用,刷新或重新授权过期账号。
|
||||
- 用账号分组控制不同密钥可使用的账号,设置并发和请求频率。
|
||||
- 接入 Codex CLI、桌面端和支持 Responses API 的客户端。
|
||||
- 支持流式对话、WebSocket、图片生成与编辑、Codex 独立搜索。
|
||||
- 支持 S3/R2 数据库备份、同大版本在线更新和回滚。
|
||||
- **账号管理**:OAuth 授权、账号导入、凭据刷新与额度查看。
|
||||
- **访问控制**:通过账号分组分配密钥权限,设置并发和请求频率。
|
||||
- **客户端接入**:支持 Codex CLI、桌面端与 Responses API 客户端,提供 HTTP 流式响应和 WebSocket。
|
||||
- **请求诊断**:查看请求记录、用量、费用、延迟和错误详情。
|
||||
- **部署运维**:Docker Compose 部署、S3/R2 数据库备份、同大版本在线更新与回滚。
|
||||
|
||||
支持 OpenAI、xAI 账号,但图片与独立搜索目前仅通过 OpenAI 提供。模型和功能是否可用取决于上游账号权限。
|
||||
**不支持 `/v1/chat/completions`**;仅支持该接口的客户端不能直接接入。
|
||||
图片生成、编辑和 Codex 独立搜索由 OpenAI 账号提供;模型与功能的可用性取决于上游账号权限。
|
||||
|
||||
> [!NOTE]
|
||||
> 本项目提供 Responses API,不支持 `/v1/chat/completions`。接入前请确认客户端支持 Responses 协议。
|
||||
|
||||
## 快速开始
|
||||
|
||||
@@ -85,76 +89,31 @@ curl -i http://127.0.0.1:8080/healthz
|
||||
默认地址只能在服务器本机访问。从其他设备使用时,需要配置
|
||||
[HTTPS 反向代理](deploy/README.md#公网访问)。
|
||||
|
||||
### 3. 添加账号和密钥
|
||||
### 3. 添加账号与客户端密钥
|
||||
|
||||
1. 在「账号」中添加 OpenAI 或 xAI 账号。OpenAI 支持 OAuth 授权、AT/RT 和账号 JSON;xAI 支持 OAuth 授权和账号 JSON。
|
||||
2. 按需创建账号分组,并把账号加入分组。
|
||||
3. 创建客户端密钥,选择可用分组。**不选分组表示可使用全部账号**。
|
||||
4. 点击该密钥的「使用密钥」,复制客户端配置。
|
||||
|
||||
只导入 AT 的账号不能自动续期。xAI 暂不支持用 API Key 导入上游账号。
|
||||
1. 在「账号」中添加 OpenAI 或 xAI 账号,完成授权或导入。
|
||||
2. 按需建立账号分组,再创建客户端密钥并选择可用分组。**不选分组表示可使用全部账号**。
|
||||
3. 打开密钥的「使用密钥」,复制客户端配置。
|
||||
|
||||
## 客户端接入
|
||||
|
||||
### Codex CLI 和桌面端
|
||||
**Codex CLI / 桌面端**:在「使用密钥」中按操作系统复制配置,或通过 CCSwitch 导入。
|
||||
合并到客户端配置后重启 Codex。完整步骤、生图配置与排障见[客户端配置](deploy/README.md#客户端配置)。
|
||||
|
||||
在「使用密钥」中按系统复制 `config.toml` 和 `auth.json`,也可以通过 CCSwitch 导入。
|
||||
已有文件先备份,再合并配置并重启 Codex。客户端使用代理密钥,OpenAI 账号登录状态由服务端管理。
|
||||
|
||||
新配置已启用原生生图。可以直接提出需求,例如:
|
||||
|
||||
> 帮我做一个咖啡店首页,生成一张咖啡主题配图并用到页面中。
|
||||
|
||||
是否调用生图由 Codex 根据任务决定,也可以明确要求生成图片。需要支持生图的客户端、模型和 OpenAI 账号;
|
||||
普通对话可用不代表账号一定有生图额度。
|
||||
|
||||
完整配置和旧版配置迁移见 [客户端配置](deploy/README.md#客户端配置)。
|
||||
|
||||
### 其他客户端
|
||||
|
||||
选择支持 **Responses API** 的客户端,填写:
|
||||
**其他 Responses API 客户端**:填写以下信息。
|
||||
|
||||
| 配置 | 值 |
|
||||
| --- | --- |
|
||||
| Base URL | `http://127.0.0.1:8080/v1`,远程使用时换成服务器的 HTTPS 地址 |
|
||||
| API Key | 管理端创建的 `sk_...` 密钥 |
|
||||
| Base URL | `http://127.0.0.1:8080/v1`;远程接入使用服务器的 HTTPS 地址 |
|
||||
| API Key | 管理端创建的 `sk_...` 客户端密钥 |
|
||||
|
||||
可先检查密钥是否能读取模型列表:
|
||||
可用模型以该密钥查询到的模型列表为准:
|
||||
|
||||
```bash
|
||||
curl http://127.0.0.1:8080/v1/models \
|
||||
-H 'Authorization: Bearer <client-api-key>'
|
||||
```
|
||||
|
||||
接口列表与请求示例见 [接口文档](docs/api.md)。
|
||||
|
||||
## 常见问题
|
||||
|
||||
### 仍然要求登录,或无法生成图片
|
||||
|
||||
重新从管理端复制配置,确认修改的是客户端实际读取的用户配置目录,然后完全退出并重启客户端。
|
||||
保留仅含代理密钥的 `auth.json` 不会使新配置失效。详细检查步骤见
|
||||
[登录与生图排查](deploy/README.md#登录与生图排查)。
|
||||
|
||||
### 没有可用账号或模型
|
||||
|
||||
检查账号是否启用、凭据是否过期、额度是否耗尽,以及密钥绑定的分组是否包含可用账号。
|
||||
「恢复账号」只清除本地错误状态,不能恢复上游权限或增加额度。
|
||||
|
||||
### 如何更新或查看日志
|
||||
|
||||
同一大版本内拉取发布镜像并重建应用容器:
|
||||
|
||||
```bash
|
||||
docker compose -f deploy/compose.yaml pull codex-proxy-rs
|
||||
docker compose -f deploy/compose.yaml up -d --no-build codex-proxy-rs
|
||||
docker compose -f deploy/compose.yaml logs --tail=100 -f codex-proxy-rs
|
||||
```
|
||||
|
||||
> [!WARNING]
|
||||
> 升级前先备份。跨大版本需使用新的数据目录重新部署,不支持直接沿用旧库在线升级。
|
||||
> `.runtime/` 包含数据库和运行数据,请勿删除;账号文件、密钥和日志也不要公开分享。
|
||||
|
||||
## 文档
|
||||
|
||||
- [客户端接入与生图](deploy/README.md#客户端配置)
|
||||
|
||||
Generated
+25
-25
@@ -154,9 +154,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "aws-lc-rs"
|
||||
version = "1.16.2"
|
||||
version = "1.18.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a054912289d18629dc78375ba2c3726a3afe3ff71b4edba9dedfca0e3446d1fc"
|
||||
checksum = "b281d307588d634de920874890732659e2e7672f72b5e10e81badc1a8a83621e"
|
||||
dependencies = [
|
||||
"aws-lc-sys",
|
||||
"untrusted 0.7.1",
|
||||
@@ -165,14 +165,15 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "aws-lc-sys"
|
||||
version = "0.39.0"
|
||||
version = "0.45.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1fa7e52a4c5c547c741610a2c6f123f3881e409b714cd27e6798ef020c514f0a"
|
||||
checksum = "9bff6c3b54fad79a2e60b8102caf565819711497c1f5f092f49508e2f5c31b27"
|
||||
dependencies = [
|
||||
"cc",
|
||||
"cmake",
|
||||
"dunce",
|
||||
"fs_extra",
|
||||
"pkg-config",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -400,9 +401,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "aws-smithy-runtime-api"
|
||||
version = "1.15.0"
|
||||
version = "1.16.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "954c563ce84507722d2679f07a35d21b9c6466b3872d513020d0281fc8112ac9"
|
||||
checksum = "9c054752dd9e4dc73d0b75748c99ac2d0feafbf2f25c7b0516f03a3534161223"
|
||||
dependencies = [
|
||||
"aws-smithy-async",
|
||||
"aws-smithy-runtime-api-macros",
|
||||
@@ -440,9 +441,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "aws-smithy-types"
|
||||
version = "1.6.2"
|
||||
version = "1.6.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "fce83ce9abbb198d25bc7131e468d0f9fe1257125e58c39f3f9fc9f5098c9647"
|
||||
checksum = "8f94d16e797ec62cd999fc9d5942b48fa7050c3093ddadff48e4d7528d16fcb9"
|
||||
dependencies = [
|
||||
"base64-simd",
|
||||
"bytes",
|
||||
@@ -1257,9 +1258,9 @@ checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582"
|
||||
|
||||
[[package]]
|
||||
name = "flate2"
|
||||
version = "1.1.9"
|
||||
version = "1.1.10"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c"
|
||||
checksum = "6e634e2e0ebac1ee034020da1ca582e17ffe4e0f5e985823721e168928136dcb"
|
||||
dependencies = [
|
||||
"crc32fast",
|
||||
"miniz_oxide",
|
||||
@@ -1466,7 +1467,7 @@ dependencies = [
|
||||
"tokio",
|
||||
"tokio-tungstenite 0.29.0",
|
||||
"tower",
|
||||
"tower-http 0.7.0",
|
||||
"tower-http 0.7.1",
|
||||
"tracing",
|
||||
"url",
|
||||
"uuid",
|
||||
@@ -1835,9 +1836,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "hyper"
|
||||
version = "1.8.1"
|
||||
version = "1.11.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2ab2d4f250c3d7b1c9fcdff1cece94ea4e2dfbec68614f7b87cb205f24ca9d11"
|
||||
checksum = "27b501faa50e7a26c3d3560ca625132f4078a17771f4810baf70475ae48cbe43"
|
||||
dependencies = [
|
||||
"atomic-waker",
|
||||
"bytes",
|
||||
@@ -1850,7 +1851,6 @@ dependencies = [
|
||||
"httpdate",
|
||||
"itoa",
|
||||
"pin-project-lite",
|
||||
"pin-utils",
|
||||
"smallvec",
|
||||
"tokio",
|
||||
"want",
|
||||
@@ -2213,9 +2213,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "miniz_oxide"
|
||||
version = "0.8.9"
|
||||
version = "0.9.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316"
|
||||
checksum = "b63fbc4a50860e98e7b2aa7804ded1db5cbc3aff9193adaff57a6931bf7c4b4c"
|
||||
dependencies = [
|
||||
"adler2",
|
||||
"simd-adler32",
|
||||
@@ -3026,9 +3026,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustls-native-certs"
|
||||
version = "0.8.3"
|
||||
version = "0.8.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "612460d5f7bea540c490b2b6395d8e34a953e52b491accd6c86c8164c5932a63"
|
||||
checksum = "dab5152771c58876a2146916e53e35057e1a4dfa2b9df0f0305b07f611fdea4d"
|
||||
dependencies = [
|
||||
"openssl-probe",
|
||||
"rustls-pki-types",
|
||||
@@ -3038,9 +3038,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustls-pki-types"
|
||||
version = "1.14.0"
|
||||
version = "1.15.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "be040f8b0a225e40375822a563fa9524378b9d63112f53e19ffff34df5d33fdd"
|
||||
checksum = "2f4925028c7eb5d1fcdaf196971378ed9d2c1c4efc7dc5d011256f76c99c0a96"
|
||||
dependencies = [
|
||||
"web-time",
|
||||
"zeroize",
|
||||
@@ -3704,7 +3704,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd"
|
||||
dependencies = [
|
||||
"fastrand",
|
||||
"getrandom 0.3.4",
|
||||
"getrandom 0.4.3",
|
||||
"once_cell",
|
||||
"rustix",
|
||||
"windows-sys 0.61.2",
|
||||
@@ -3805,9 +3805,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20"
|
||||
|
||||
[[package]]
|
||||
name = "tokio"
|
||||
version = "1.52.3"
|
||||
version = "1.53.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8fc7f01b389ac15039e4dc9531aa973a135d7a4135281b12d7c1bc79fd57fffe"
|
||||
checksum = "202caea871b69668250d242070849eb495be178ed697a3e98aebce5bc81a0bed"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"libc",
|
||||
@@ -3928,9 +3928,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "tower-http"
|
||||
version = "0.7.0"
|
||||
version = "0.7.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b11f75e912b0c2be01b63d8cf8057b8c3f97cf34abb3d431a3a4c8675498e233"
|
||||
checksum = "08a05a66a4fdd61cbbe0a1d755ffe0ca6aba159dd4820936a0ff8a8278245b9c"
|
||||
dependencies = [
|
||||
"bitflags",
|
||||
"bytes",
|
||||
|
||||
@@ -87,6 +87,7 @@ pub async fn run() -> Result<(), BootstrapError> {
|
||||
|
||||
let mut probes = store.health_probes();
|
||||
probes.extend(core.health_probes());
|
||||
probes.push(host.logging_health_probe());
|
||||
let api = gateway_api::initialize(
|
||||
api,
|
||||
core.execution_service(),
|
||||
|
||||
@@ -213,13 +213,17 @@ fn capitalize_first(value: &str) -> String {
|
||||
|
||||
pub(crate) fn billing_view(billing: Option<&domain::UsageBilling>) -> Option<BillingView> {
|
||||
match billing? {
|
||||
domain::UsageBilling::Total { total, .. } => Some(BillingView {
|
||||
domain::UsageBilling::Total { source, total } => Some(BillingView {
|
||||
input_amount_display: "—".to_owned(),
|
||||
output_amount_display: "—".to_owned(),
|
||||
cache_read_amount_display: "—".to_owned(),
|
||||
cache_write_amount_display: "—".to_owned(),
|
||||
standard_amount_display: "—".to_owned(),
|
||||
total_amount_display: format_money(total),
|
||||
total_amount_display: if source == "calculated" {
|
||||
format!("≈ {}", format_money(total))
|
||||
} else {
|
||||
format_money(total)
|
||||
},
|
||||
input_price_display: "—".to_owned(),
|
||||
output_price_display: "—".to_owned(),
|
||||
cache_read_price_display: "—".to_owned(),
|
||||
|
||||
@@ -15,8 +15,8 @@ use serde_json::Value;
|
||||
use crate::ApiState;
|
||||
use crate::openai::{
|
||||
auth::{authenticate_client, client_access_error_response},
|
||||
endpoint::collect_raw_json_response,
|
||||
error::gateway_error_response,
|
||||
provider_endpoint::collect_raw_json_response,
|
||||
responses::{OpenAiRequestHeaders, request_client_context},
|
||||
};
|
||||
|
||||
|
||||
@@ -1,10 +1,10 @@
|
||||
//! OpenAI 客户端协议 adapter。
|
||||
|
||||
pub mod auth;
|
||||
mod endpoint;
|
||||
pub mod error;
|
||||
pub mod images;
|
||||
pub mod models;
|
||||
mod provider_endpoint;
|
||||
pub mod responses;
|
||||
pub mod router;
|
||||
pub mod search;
|
||||
|
||||
@@ -23,6 +23,7 @@ const OUTBOUND_COMMAND_BUFFER: usize = 32;
|
||||
const INBOUND_EVENT_BUFFER: usize = 32;
|
||||
const DOWNSTREAM_WRITE_TIMEOUT: Duration = Duration::from_secs(5 * 60);
|
||||
const CONNECTION_MAX_AGE: Duration = Duration::from_secs(60 * 60);
|
||||
const DOWNSTREAM_PING_INTERVAL: Duration = Duration::from_secs(25);
|
||||
|
||||
/// WebSocket pump 的写入与生命周期预算。
|
||||
#[derive(Clone, Copy)]
|
||||
@@ -171,9 +172,27 @@ struct ConnectionStats {
|
||||
command_backpressure_count: AtomicU64,
|
||||
ping_received_count: AtomicU64,
|
||||
pong_written_count: AtomicU64,
|
||||
ping_written_count: AtomicU64,
|
||||
pong_received_count: AtomicU64,
|
||||
last_read_ms: AtomicU64,
|
||||
last_write_ms: AtomicU64,
|
||||
}
|
||||
|
||||
impl ConnectionStats {
|
||||
fn record_read(&self, opened_at: Instant) {
|
||||
self.last_read_ms.store(
|
||||
u64::try_from(opened_at.elapsed().as_millis()).unwrap_or(u64::MAX),
|
||||
Ordering::Relaxed,
|
||||
);
|
||||
}
|
||||
|
||||
fn record_write(&self, opened_at: Instant) {
|
||||
self.last_write_ms.store(
|
||||
u64::try_from(opened_at.elapsed().as_millis()).unwrap_or(u64::MAX),
|
||||
Ordering::Relaxed,
|
||||
);
|
||||
}
|
||||
|
||||
fn observe_command_queue(&self, sender: &mpsc::Sender<ConnectionCommand>) {
|
||||
let queued = OUTBOUND_COMMAND_BUFFER
|
||||
.saturating_sub(sender.capacity())
|
||||
@@ -400,10 +419,15 @@ impl ResponsesWebSocketConnection {
|
||||
}
|
||||
|
||||
pub(super) fn log_summary(&self, request_count: u64) {
|
||||
let connection_age_ms = self.age().as_millis();
|
||||
let read_idle_ms = connection_age_ms
|
||||
.saturating_sub(u128::from(self.stats.last_read_ms.load(Ordering::Relaxed)));
|
||||
let write_idle_ms = connection_age_ms
|
||||
.saturating_sub(u128::from(self.stats.last_write_ms.load(Ordering::Relaxed)));
|
||||
tracing::info!(
|
||||
websocket_connection_id = %self.connection_id,
|
||||
request_count,
|
||||
connection_age_ms = self.age().as_millis(),
|
||||
connection_age_ms,
|
||||
pump_exit_reason = self
|
||||
.exit_reason
|
||||
.map_or("unknown", PumpExitReason::as_str),
|
||||
@@ -417,6 +441,10 @@ impl ResponsesWebSocketConnection {
|
||||
.load(Ordering::Relaxed),
|
||||
ping_received_count = self.stats.ping_received_count.load(Ordering::Relaxed),
|
||||
pong_written_count = self.stats.pong_written_count.load(Ordering::Relaxed),
|
||||
ping_written_count = self.stats.ping_written_count.load(Ordering::Relaxed),
|
||||
pong_received_count = self.stats.pong_received_count.load(Ordering::Relaxed),
|
||||
read_idle_ms,
|
||||
write_idle_ms,
|
||||
"Responses WebSocket disconnected"
|
||||
);
|
||||
}
|
||||
@@ -493,6 +521,12 @@ async fn run_pump<S, E>(
|
||||
let deadline = tokio::time::sleep_until(opened_at + config.max_age);
|
||||
tokio::pin!(deadline);
|
||||
let mut deadline_elapsed = false;
|
||||
let mut heartbeat = tokio::time::interval_at(
|
||||
opened_at + DOWNSTREAM_PING_INTERVAL,
|
||||
DOWNSTREAM_PING_INTERVAL,
|
||||
);
|
||||
heartbeat.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
|
||||
let mut ping_sequence = 0_u64;
|
||||
let reason = loop {
|
||||
tokio::select! {
|
||||
biased;
|
||||
@@ -531,6 +565,9 @@ async fn run_pump<S, E>(
|
||||
ConnectionWriteError::Transport { message }
|
||||
});
|
||||
let failed = result.is_err();
|
||||
if !failed {
|
||||
stats.record_write(opened_at);
|
||||
}
|
||||
let _ = acknowledged.send(result);
|
||||
if failed {
|
||||
break PumpExitReason::WriteError;
|
||||
@@ -539,45 +576,79 @@ async fn run_pump<S, E>(
|
||||
break PumpExitReason::ServerClose;
|
||||
}
|
||||
}
|
||||
message = socket.next() => match message {
|
||||
Some(Ok(Message::Text(payload))) => {
|
||||
if let Err(reason) = emit_incoming(&incoming, ConnectionEvent::Text(payload.to_string())) {
|
||||
break reason;
|
||||
// 官方 Codex WsStream 在独立 pump 中回复 Pong,且不将控制帧交给业务流。
|
||||
// 这里只为下游链路保活,不设置 Pong deadline,也不延长业务请求预算。
|
||||
_ = heartbeat.tick() => {
|
||||
ping_sequence = ping_sequence.wrapping_add(1);
|
||||
let ping = Message::Ping(ping_sequence.to_be_bytes().to_vec().into());
|
||||
let result = tokio::select! {
|
||||
biased;
|
||||
() = cancellation.cancelled() => break PumpExitReason::LifecycleShutdown,
|
||||
result = timeout(config.write_timeout, socket.send(ping)) => result,
|
||||
};
|
||||
match result {
|
||||
Ok(Ok(())) => {
|
||||
stats.ping_written_count.fetch_add(1, Ordering::Relaxed);
|
||||
stats.record_write(opened_at);
|
||||
}
|
||||
}
|
||||
Some(Ok(Message::Binary(_))) => {
|
||||
if let Err(reason) = emit_incoming(&incoming, ConnectionEvent::Binary) {
|
||||
break reason;
|
||||
Ok(Err(error)) => {
|
||||
tracing::info!(
|
||||
websocket_connection_id = %connection_id,
|
||||
error = %error,
|
||||
"Responses WebSocket Ping write failed"
|
||||
);
|
||||
break PumpExitReason::WriteError;
|
||||
}
|
||||
Err(_) => break PumpExitReason::WriteTimeout,
|
||||
}
|
||||
Some(Ok(Message::Ping(payload))) => {
|
||||
stats.ping_received_count.fetch_add(1, Ordering::Relaxed);
|
||||
match timeout(config.write_timeout, socket.send(Message::Pong(payload))).await {
|
||||
Ok(Ok(())) => {
|
||||
stats.pong_written_count.fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
message = socket.next() => {
|
||||
if matches!(&message, Some(Ok(_))) {
|
||||
stats.record_read(opened_at);
|
||||
}
|
||||
match message {
|
||||
Some(Ok(Message::Text(payload))) => {
|
||||
if let Err(reason) = emit_incoming(&incoming, ConnectionEvent::Text(payload.to_string())) {
|
||||
break reason;
|
||||
}
|
||||
Ok(Err(error)) => {
|
||||
tracing::info!(
|
||||
websocket_connection_id = %connection_id,
|
||||
error = %error,
|
||||
"Responses WebSocket Pong write failed"
|
||||
);
|
||||
break PumpExitReason::WriteError;
|
||||
}
|
||||
Err(_) => break PumpExitReason::WriteTimeout,
|
||||
}
|
||||
Some(Ok(Message::Binary(_))) => {
|
||||
if let Err(reason) = emit_incoming(&incoming, ConnectionEvent::Binary) {
|
||||
break reason;
|
||||
}
|
||||
}
|
||||
Some(Ok(Message::Ping(payload))) => {
|
||||
stats.ping_received_count.fetch_add(1, Ordering::Relaxed);
|
||||
match timeout(config.write_timeout, socket.send(Message::Pong(payload))).await {
|
||||
Ok(Ok(())) => {
|
||||
stats.pong_written_count.fetch_add(1, Ordering::Relaxed);
|
||||
stats.record_write(opened_at);
|
||||
}
|
||||
Ok(Err(error)) => {
|
||||
tracing::info!(
|
||||
websocket_connection_id = %connection_id,
|
||||
error = %error,
|
||||
"Responses WebSocket Pong write failed"
|
||||
);
|
||||
break PumpExitReason::WriteError;
|
||||
}
|
||||
Err(_) => break PumpExitReason::WriteTimeout,
|
||||
}
|
||||
}
|
||||
Some(Ok(Message::Pong(_))) => {
|
||||
stats.pong_received_count.fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
Some(Ok(Message::Close(_))) => break PumpExitReason::ClientClose,
|
||||
Some(Err(error)) => {
|
||||
tracing::info!(
|
||||
websocket_connection_id = %connection_id,
|
||||
error = %error,
|
||||
"Responses WebSocket receive failed"
|
||||
);
|
||||
break PumpExitReason::ReadError;
|
||||
}
|
||||
None => break PumpExitReason::PeerEof,
|
||||
}
|
||||
Some(Ok(Message::Pong(_))) => {}
|
||||
Some(Ok(Message::Close(_))) => break PumpExitReason::ClientClose,
|
||||
Some(Err(error)) => {
|
||||
tracing::info!(
|
||||
websocket_connection_id = %connection_id,
|
||||
error = %error,
|
||||
"Responses WebSocket receive failed"
|
||||
);
|
||||
break PumpExitReason::ReadError;
|
||||
}
|
||||
None => break PumpExitReason::PeerEof,
|
||||
},
|
||||
}
|
||||
};
|
||||
|
||||
@@ -247,7 +247,7 @@ async fn serve_responses_websocket(socket: WebSocket, session: ResponsesWebSocke
|
||||
}
|
||||
};
|
||||
started.session.trace().record("client.connection", serde_json::json!({
|
||||
"transport": "websocket", "connectionId": connection.id(), "correlationId": correlation_id,
|
||||
"transport": "websocket", "connectionId": connection.id(), "correlationId": correlation_id.as_ref(),
|
||||
"requestIndex": request_count,
|
||||
}));
|
||||
started
|
||||
|
||||
@@ -14,8 +14,8 @@ use gateway_core::operation::{Operation, RawJsonPayload, StandaloneSearchRequest
|
||||
use crate::ApiState;
|
||||
use crate::openai::{
|
||||
auth::{authenticate_client, client_access_error_response},
|
||||
endpoint::collect_raw_json_response,
|
||||
error::gateway_error_response,
|
||||
provider_endpoint::collect_raw_json_response,
|
||||
responses::{OpenAiRequestHeaders, request_client_context},
|
||||
};
|
||||
|
||||
|
||||
@@ -919,6 +919,22 @@ async fn usage_route_should_expose_table_facts_without_detail_payload() {
|
||||
compact: true,
|
||||
started_at: Utc::now(),
|
||||
});
|
||||
{
|
||||
let mut records = fixture.usage_records.lock().expect("usage records");
|
||||
for source in ["calculated", "provider_reported"] {
|
||||
let mut image = records[0].clone();
|
||||
image.id = format!("image_{source}");
|
||||
image.endpoint = "/v1/images/generations".to_owned();
|
||||
image.provider_kind = Some("openai".to_owned());
|
||||
image.requested_model_id = Some("gpt-image-2".to_owned());
|
||||
image.upstream_model_id = None;
|
||||
image.billing = Some(UsageBilling::Total {
|
||||
source: source.to_owned(),
|
||||
total: usd("0.00696"),
|
||||
});
|
||||
records.push(image);
|
||||
}
|
||||
}
|
||||
let response = observability::router::<AdminTestState>()
|
||||
.with_state(fixture.state())
|
||||
.oneshot(
|
||||
@@ -1016,4 +1032,12 @@ async fn usage_route_should_expose_table_facts_without_detail_payload() {
|
||||
})
|
||||
);
|
||||
assert!(value["data"]["items"][0].get("metadata").is_none());
|
||||
assert_eq!(
|
||||
value["data"]["items"][1]["billing"]["totalAmountDisplay"],
|
||||
"≈ $0.007"
|
||||
);
|
||||
assert_eq!(
|
||||
value["data"]["items"][2]["billing"]["totalAmountDisplay"],
|
||||
"$0.007"
|
||||
);
|
||||
}
|
||||
|
||||
@@ -61,7 +61,7 @@ fn source_tree_should_match_frozen_machine_manifest() {
|
||||
"src/openai/images/mod.rs",
|
||||
"src/openai/mod.rs",
|
||||
"src/openai/models.rs",
|
||||
"src/openai/provider_endpoint.rs",
|
||||
"src/openai/endpoint.rs",
|
||||
"src/openai/responses/error.rs",
|
||||
"src/openai/responses/http.rs",
|
||||
"src/openai/responses/mod.rs",
|
||||
@@ -107,7 +107,7 @@ fn test_tree_should_match_frozen_rust_mirror() {
|
||||
"tests/openai/images/mod.rs",
|
||||
"tests/openai/mod.rs",
|
||||
"tests/openai/models.rs",
|
||||
"tests/openai/provider_endpoint.rs",
|
||||
"tests/openai/endpoint.rs",
|
||||
"tests/openai/responses/http.rs",
|
||||
"tests/openai/responses/mod.rs",
|
||||
"tests/openai/responses/websocket/connection.rs",
|
||||
|
||||
@@ -1,8 +1,8 @@
|
||||
mod auth;
|
||||
mod endpoint;
|
||||
mod error;
|
||||
mod images;
|
||||
mod models;
|
||||
mod provider_endpoint;
|
||||
mod responses;
|
||||
mod router;
|
||||
mod search;
|
||||
|
||||
@@ -444,21 +444,6 @@ async fn inbound_overload_should_close_without_executing_queued_frames() {
|
||||
));
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn pump_does_not_emit_an_active_ping() {
|
||||
let PumpHarness {
|
||||
connection,
|
||||
incoming: _incoming,
|
||||
mut written,
|
||||
..
|
||||
} = test_connection(false);
|
||||
tokio::time::advance(Duration::from_secs(5 * 60)).await;
|
||||
tokio::task::yield_now().await;
|
||||
|
||||
assert!(matches!(written.try_recv(), Err(TryRecvError::Empty)));
|
||||
drop(connection);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn outbound_commands_are_written_in_acknowledged_order() {
|
||||
let PumpHarness {
|
||||
@@ -582,13 +567,17 @@ async fn idle_connection_reaches_the_official_limit_without_starting_an_executio
|
||||
|
||||
tokio::time::advance(Duration::from_secs(60 * 60)).await;
|
||||
tokio::task::yield_now().await;
|
||||
let message = tokio::time::timeout(Duration::from_secs(1), socket.next())
|
||||
.await
|
||||
.expect("connection limit response timeout")
|
||||
.expect("connection remains available for the limit error")
|
||||
.expect("valid WebSocket frame");
|
||||
let ClientMessage::Text(text) = message else {
|
||||
panic!("connection limit must be a text error event");
|
||||
let text = loop {
|
||||
let message = tokio::time::timeout(Duration::from_secs(1), socket.next())
|
||||
.await
|
||||
.expect("connection limit response timeout")
|
||||
.expect("connection remains available for the limit error")
|
||||
.expect("valid WebSocket frame");
|
||||
match message {
|
||||
ClientMessage::Text(text) => break text,
|
||||
ClientMessage::Ping(_) => {}
|
||||
other => panic!("unexpected frame before connection limit: {other:?}"),
|
||||
}
|
||||
};
|
||||
let value = serde_json::from_str::<Value>(&text).expect("connection limit JSON");
|
||||
|
||||
@@ -606,7 +595,8 @@ async fn idle_connection_reaches_the_official_limit_without_starting_an_executio
|
||||
);
|
||||
assert_eq!(trace.starts.load(Ordering::Acquire), 0);
|
||||
|
||||
socket.close(None).await.expect("close WebSocket");
|
||||
// 服务端已因生命周期到期关闭;不再向关闭中的连接写入。
|
||||
drop(socket);
|
||||
server.abort();
|
||||
}
|
||||
|
||||
@@ -949,3 +939,134 @@ async fn client_close_should_cancel_active_execution_without_starting_queued_req
|
||||
assert!(trace.cancelled.load(Ordering::Acquire));
|
||||
server.abort();
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn idle_pump_sends_heartbeat_without_bursting_after_a_delay() {
|
||||
let PumpHarness {
|
||||
connection,
|
||||
incoming: _incoming,
|
||||
mut written,
|
||||
..
|
||||
} = test_connection(false);
|
||||
tokio::task::yield_now().await;
|
||||
assert!(matches!(written.try_recv(), Err(TryRecvError::Empty)));
|
||||
|
||||
tokio::time::advance(Duration::from_secs(25)).await;
|
||||
let first = written.recv().await.expect("first heartbeat");
|
||||
assert!(matches!(first, Message::Ping(_)));
|
||||
|
||||
tokio::time::advance(Duration::from_secs(120)).await;
|
||||
let second = written.recv().await.expect("heartbeat after delay");
|
||||
assert!(matches!(&second, Message::Ping(_)));
|
||||
assert_ne!(first, second);
|
||||
tokio::task::yield_now().await;
|
||||
assert!(matches!(written.try_recv(), Err(TryRecvError::Empty)));
|
||||
drop(connection);
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn missing_pongs_do_not_cancel_the_connection_or_block_business_frames() {
|
||||
let PumpHarness {
|
||||
mut connection,
|
||||
incoming,
|
||||
mut written,
|
||||
..
|
||||
} = test_connection(false);
|
||||
for _ in 0..13 {
|
||||
tokio::time::advance(Duration::from_secs(25)).await;
|
||||
assert!(matches!(written.recv().await, Some(Message::Ping(_))));
|
||||
}
|
||||
|
||||
incoming
|
||||
.send(Ok(Message::Text("still connected".into())))
|
||||
.expect("send business frame after 325 seconds without Pong");
|
||||
assert!(matches!(
|
||||
connection.next_event().await,
|
||||
Some(ConnectionEvent::Text(text)) if text == "still connected"
|
||||
));
|
||||
incoming.send(Ok(Message::Close(None))).expect("close peer");
|
||||
assert_eq!(
|
||||
connection.wait_for_exit().await,
|
||||
PumpExitReason::ClientClose
|
||||
);
|
||||
tokio::time::advance(Duration::from_secs(25)).await;
|
||||
assert!(written.recv().await.is_none());
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn stalled_heartbeat_can_be_cancelled_without_waiting_for_write_timeout() {
|
||||
let PumpHarness {
|
||||
mut connection,
|
||||
incoming: _incoming,
|
||||
written: _written,
|
||||
dropped,
|
||||
cancellation,
|
||||
} = test_connection(true);
|
||||
tokio::time::advance(Duration::from_secs(25)).await;
|
||||
tokio::task::yield_now().await;
|
||||
cancellation.cancel();
|
||||
let reason = tokio::time::timeout(Duration::from_secs(1), connection.wait_for_exit())
|
||||
.await
|
||||
.expect("cancel a blocked heartbeat promptly");
|
||||
assert_eq!(reason, PumpExitReason::LifecycleShutdown);
|
||||
assert!(dropped.load(Ordering::Acquire));
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn stalled_heartbeat_obeys_the_existing_write_timeout() {
|
||||
let PumpHarness {
|
||||
mut connection,
|
||||
incoming: _incoming,
|
||||
written: _written,
|
||||
dropped,
|
||||
..
|
||||
} = test_connection(true);
|
||||
let started = tokio::time::Instant::now();
|
||||
assert_eq!(
|
||||
connection.wait_for_exit().await,
|
||||
PumpExitReason::WriteTimeout
|
||||
);
|
||||
assert_eq!(started.elapsed(), Duration::from_secs(25 + 300));
|
||||
assert!(dropped.load(Ordering::Acquire));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn active_response_receives_heartbeats_during_two_hundred_seconds_of_silence() {
|
||||
let (trace, mut socket, server) = start_active_response().await;
|
||||
for _ in 0..8 {
|
||||
tokio::time::pause();
|
||||
tokio::time::advance(Duration::from_secs(25)).await;
|
||||
tokio::task::yield_now().await;
|
||||
tokio::time::resume();
|
||||
let message = tokio::time::timeout(Duration::from_secs(2), socket.next())
|
||||
.await
|
||||
.expect("heartbeat while the upstream response is pending")
|
||||
.expect("connection remains open")
|
||||
.expect("valid heartbeat frame");
|
||||
let ClientMessage::Ping(payload) = message else {
|
||||
panic!("expected heartbeat while business stream is silent: {message:?}");
|
||||
};
|
||||
// 与官方 WsStream 一样,在业务响应未完成时回复同 payload 的 Pong。
|
||||
socket
|
||||
.send(ClientMessage::Pong(payload))
|
||||
.await
|
||||
.expect("reply to heartbeat");
|
||||
}
|
||||
assert_eq!(trace.starts.load(Ordering::Acquire), 1);
|
||||
assert!(!trace.cancelled.load(Ordering::Acquire));
|
||||
|
||||
trace.release_terminal.notify_one();
|
||||
let message = tokio::time::timeout(Duration::from_secs(2), socket.next())
|
||||
.await
|
||||
.expect("terminal response after heartbeats")
|
||||
.expect("connection remains open")
|
||||
.expect("valid terminal frame");
|
||||
let ClientMessage::Text(text) = message else {
|
||||
panic!("expected completed response: {message:?}");
|
||||
};
|
||||
let value: Value = serde_json::from_str(&text).expect("terminal JSON");
|
||||
assert_eq!(value["type"], "response.completed");
|
||||
assert_eq!(trace.starts.load(Ordering::Acquire), 1);
|
||||
socket.close(None).await.expect("close WebSocket");
|
||||
server.abort();
|
||||
}
|
||||
|
||||
@@ -22,7 +22,7 @@ use gateway_core::upstream::UpstreamSendState;
|
||||
use serde_json::{Value, json};
|
||||
use tower::ServiceExt;
|
||||
|
||||
use super::provider_endpoint::BufferedJsonSession;
|
||||
use super::endpoint::BufferedJsonSession;
|
||||
use super::{api_router, authenticated_client};
|
||||
|
||||
const SEARCH_RESPONSE: &[u8] = br#"{ "encrypted_output":"ciphertext", "output":"search result", "results":[{"type":"text_result","ref_id":"turn0search0","future":9007199254740993}] }"#;
|
||||
|
||||
@@ -1854,6 +1854,9 @@ fn duration_ms(duration: Duration) -> u64 {
|
||||
fn record_trace_error(trace: &TraceContext, error: &ProviderError) {
|
||||
trace.record("attempt.failed", json!({
|
||||
"kind": error.kind().as_str(), "sendState": format!("{:?}", error.send_state()),
|
||||
"diagnostic": error.diagnostic().map(|diagnostic| json!({
|
||||
"stage": diagnostic.stage(), "code": diagnostic.code(), "message": diagnostic.as_str(),
|
||||
})),
|
||||
"upstreamStatus": error.upstream_status(),
|
||||
"upstreamCode": error.upstream_code().map(|code| code.as_str()),
|
||||
"rawError": error.raw_upstream_error().map(|raw| {
|
||||
|
||||
@@ -524,7 +524,7 @@ impl DefaultExecutionService {
|
||||
operation: operation.kind(),
|
||||
endpoint: metadata.endpoint,
|
||||
client_transport: metadata.transport.as_str().to_owned(),
|
||||
requested_model: target.into_public_model(),
|
||||
requested_model: target.into_public_model().or(observation.requested_model),
|
||||
client_ip: metadata.client_ip,
|
||||
user_agent: metadata.user_agent,
|
||||
reasoning_effort: observation.reasoning_effort,
|
||||
|
||||
@@ -20,7 +20,9 @@ use crate::event::{EventSequenceValidator, ProviderEvent};
|
||||
use crate::identity::ProviderKind;
|
||||
use crate::operation::Operation;
|
||||
use crate::policy::ClientApiKeyId;
|
||||
use crate::routing::{ModelCapabilities, ModelPresentation, ProviderCandidate, UpstreamModelId};
|
||||
use crate::routing::{
|
||||
ModelCapabilities, ModelPresentation, ProviderCandidate, PublicModelId, UpstreamModelId,
|
||||
};
|
||||
use crate::upstream::OpaqueUpstreamValue;
|
||||
use crate::upstream::{UpstreamSendState, UpstreamTransport};
|
||||
|
||||
@@ -416,6 +418,8 @@ pub struct ProviderRequest {
|
||||
/// Provider 对公共观测表可解释的请求语义;未知字段保持空值。
|
||||
#[derive(Debug, Clone, Default, PartialEq, Eq)]
|
||||
pub struct ProviderRequestObservation {
|
||||
/// 独立 Provider 端点声明的请求模型,仅用于观测,不参与文本模型目录路由。
|
||||
pub requested_model: Option<PublicModelId>,
|
||||
/// 客户端原始请求中的推理强度。
|
||||
pub reasoning_effort: Option<String>,
|
||||
pub reasoning_preset: Option<String>,
|
||||
|
||||
@@ -172,17 +172,72 @@ impl fmt::Debug for ClientVisibleUpstreamError {
|
||||
/// Adapter 只能放入不含凭据、cookie、原始请求正文和 opaque response ID 的有界摘要;
|
||||
/// 原始上游错误返回由 [`RawUpstreamError`] 单独承载。
|
||||
#[derive(Clone, PartialEq, Eq)]
|
||||
pub struct ProviderDiagnostic(String);
|
||||
pub struct ProviderDiagnostic {
|
||||
message: String,
|
||||
stage: Option<&'static str>,
|
||||
code: Option<&'static str>,
|
||||
}
|
||||
|
||||
impl ProviderDiagnostic {
|
||||
#[must_use]
|
||||
pub fn new(message: impl Into<String>) -> Self {
|
||||
Self(message.into())
|
||||
Self {
|
||||
message: message.into(),
|
||||
stage: None,
|
||||
code: None,
|
||||
}
|
||||
}
|
||||
|
||||
/// 附着本地定义的阶段与原因码;独立于发送状态和重试策略。
|
||||
#[must_use]
|
||||
pub const fn with_classification(mut self, stage: &'static str, code: &'static str) -> Self {
|
||||
self.stage = Some(stage);
|
||||
self.code = Some(code);
|
||||
self
|
||||
}
|
||||
|
||||
/// 从错误链补充可证明的 I/O 原因;只保留本地原因码,不格式化原始错误。
|
||||
#[must_use]
|
||||
pub fn with_io_cause(mut self, error: &(dyn std::error::Error + 'static)) -> Self {
|
||||
let mut source = Some(error);
|
||||
while let Some(error) = source {
|
||||
if let Some(error) = error.downcast_ref::<std::io::Error>() {
|
||||
let code = match error.kind() {
|
||||
std::io::ErrorKind::ConnectionRefused => Some("connection_refused"),
|
||||
std::io::ErrorKind::ConnectionReset => Some("tcp_reset"),
|
||||
std::io::ErrorKind::ConnectionAborted => Some("connection_aborted"),
|
||||
std::io::ErrorKind::BrokenPipe => Some("broken_pipe"),
|
||||
std::io::ErrorKind::UnexpectedEof => Some("unexpected_eof"),
|
||||
std::io::ErrorKind::TimedOut => Some("io_timeout"),
|
||||
std::io::ErrorKind::NetworkUnreachable => Some("network_unreachable"),
|
||||
std::io::ErrorKind::HostUnreachable => Some("host_unreachable"),
|
||||
_ => None,
|
||||
};
|
||||
if let Some(code) = code {
|
||||
self.message.push_str("; I/O cause: ");
|
||||
self.message.push_str(code);
|
||||
self.code = Some(code);
|
||||
break;
|
||||
}
|
||||
}
|
||||
source = error.source();
|
||||
}
|
||||
self
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn stage(&self) -> Option<&'static str> {
|
||||
self.stage
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub const fn code(&self) -> Option<&'static str> {
|
||||
self.code
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn as_str(&self) -> &str {
|
||||
&self.0
|
||||
&self.message
|
||||
}
|
||||
}
|
||||
|
||||
@@ -817,7 +872,10 @@ impl fmt::Debug for ProviderError {
|
||||
"sensitive_context",
|
||||
&self.sensitive_context_redacted.then_some("<redacted>"),
|
||||
)
|
||||
.field("diagnostic", &self.diagnostic.as_ref().map(|_| "<present>"))
|
||||
.field(
|
||||
"diagnostic",
|
||||
&self.diagnostic.as_deref().map(|_| "<present>"),
|
||||
)
|
||||
.field(
|
||||
"raw_upstream_error",
|
||||
&self.raw_upstream_error.as_ref().map(|_| "<present>"),
|
||||
@@ -915,7 +973,7 @@ impl GatewayErrorKind {
|
||||
pub struct GatewayError {
|
||||
kind: GatewayErrorKind,
|
||||
message: &'static str,
|
||||
diagnostic: Option<ProviderDiagnostic>,
|
||||
diagnostic: Option<Box<ProviderDiagnostic>>,
|
||||
client_visible_upstream_error: Option<ClientVisibleUpstreamError>,
|
||||
}
|
||||
|
||||
@@ -993,7 +1051,7 @@ impl GatewayError {
|
||||
/// 附加只供运维观测使用的脱敏 Provider 摘要。
|
||||
#[must_use]
|
||||
pub fn with_diagnostic(mut self, diagnostic: ProviderDiagnostic) -> Self {
|
||||
self.diagnostic = Some(diagnostic);
|
||||
self.diagnostic = Some(Box::new(diagnostic));
|
||||
self
|
||||
}
|
||||
|
||||
@@ -1019,7 +1077,7 @@ impl GatewayError {
|
||||
/// 返回可安全持久化和展示的 Provider 诊断摘要。
|
||||
#[must_use]
|
||||
pub fn diagnostic(&self) -> Option<&ProviderDiagnostic> {
|
||||
self.diagnostic.as_ref()
|
||||
self.diagnostic.as_deref()
|
||||
}
|
||||
|
||||
/// 返回优先用于原客户端协议响应的 message;持久化和日志必须继续使用
|
||||
@@ -1054,7 +1112,10 @@ impl fmt::Debug for GatewayError {
|
||||
.debug_struct("GatewayError")
|
||||
.field("kind", &self.kind)
|
||||
.field("message", &self.message)
|
||||
.field("diagnostic", &self.diagnostic.as_ref().map(|_| "<present>"))
|
||||
.field(
|
||||
"diagnostic",
|
||||
&self.diagnostic.as_deref().map(|_| "<present>"),
|
||||
)
|
||||
.field(
|
||||
"client_visible_upstream_error",
|
||||
&self
|
||||
|
||||
@@ -1292,7 +1292,13 @@ fn discarded_attempt_observation_does_not_leak_into_retry_result() {
|
||||
Ok(ProviderEvent::observation(first_observation)),
|
||||
Err(
|
||||
ProviderError::new(ProviderErrorKind::Unavailable, UpstreamSendState::Sent)
|
||||
.with_replay_safe(),
|
||||
.with_replay_safe()
|
||||
.with_diagnostic(
|
||||
gateway_core::error::ProviderDiagnostic::new(
|
||||
"response event idle timeout after 300s",
|
||||
)
|
||||
.with_classification("receive", "receive_idle_timeout"),
|
||||
),
|
||||
),
|
||||
],
|
||||
},
|
||||
@@ -1329,6 +1335,16 @@ fn discarded_attempt_observation_does_not_leak_into_retry_result() {
|
||||
.iter()
|
||||
.any(|event| event["stage"] == "attempt.started" && event["attemptIndex"] == 2)
|
||||
);
|
||||
let failed = events
|
||||
.iter()
|
||||
.find(|event| event["stage"] == "attempt.failed")
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
failed["data"]["diagnostic"],
|
||||
serde_json::json!({
|
||||
"stage": "receive", "code": "receive_idle_timeout", "message": "response event idle timeout after 300s"
|
||||
})
|
||||
);
|
||||
assert_eq!(events.last().unwrap()["stage"], "request.finished");
|
||||
|
||||
assert_eq!(finalization.upstream_transport.as_deref(), Some("http_sse"));
|
||||
|
||||
@@ -26,7 +26,7 @@ use gateway_core::engine::execution::{
|
||||
use gateway_core::engine::probe::{AccountProbe, AccountProbeErrorSource, AccountProbeRequest};
|
||||
use gateway_core::engine::provider::{
|
||||
Provider, ProviderCallMetadata, ProviderCatalogGeneration, ProviderModelCapabilities,
|
||||
ProviderRegistry, ProviderRequest, ProviderStream,
|
||||
ProviderRegistry, ProviderRequest, ProviderRequestObservation, ProviderStream,
|
||||
};
|
||||
use gateway_core::engine::{
|
||||
AttemptContext, AttemptRecord, ExecutionStore, IntermediateFailure, ModelRequestFinalization,
|
||||
@@ -260,7 +260,9 @@ impl Provider for LocalFailingProvider {
|
||||
}
|
||||
}
|
||||
|
||||
struct ColdFailingProvider;
|
||||
struct ColdFailingProvider {
|
||||
requested_model: Option<PublicModelId>,
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl Provider for ColdFailingProvider {
|
||||
@@ -272,6 +274,13 @@ impl Provider for ColdFailingProvider {
|
||||
ProviderCatalogGeneration::default()
|
||||
}
|
||||
|
||||
fn request_observation(&self, _: &Operation, _: &ClientApiKeyId) -> ProviderRequestObservation {
|
||||
ProviderRequestObservation {
|
||||
requested_model: self.requested_model.clone(),
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
async fn query_model_capabilities(
|
||||
&self,
|
||||
) -> Result<Vec<ProviderModelCapabilities>, ProviderError> {
|
||||
@@ -323,12 +332,19 @@ fn successful_authentication_should_record_client_key_usage() {
|
||||
|
||||
#[test]
|
||||
fn provider_endpoint_should_persist_its_real_v1_endpoint() {
|
||||
assert_provider_endpoint_observation(None);
|
||||
assert_provider_endpoint_observation(Some("gpt-image-2"));
|
||||
}
|
||||
|
||||
fn assert_provider_endpoint_observation(model: Option<&str>) {
|
||||
let store = Arc::new(TrackingExecutionStore::default());
|
||||
let service = DefaultExecutionService::new(
|
||||
RuntimeSnapshotHandle::new(client_snapshot()),
|
||||
store.clone(),
|
||||
ProviderRegistry::new([Arc::new(ColdFailingProvider) as Arc<dyn Provider>])
|
||||
.expect("provider registry"),
|
||||
ProviderRegistry::new([Arc::new(ColdFailingProvider {
|
||||
requested_model: model.map(|model| PublicModelId::new(model).expect("model")),
|
||||
}) as Arc<dyn Provider>])
|
||||
.expect("provider registry"),
|
||||
Arc::new(UnusedAdmissions),
|
||||
Arc::new(UnusedCircuits),
|
||||
Arc::new(UnusedContinuation),
|
||||
@@ -369,7 +385,7 @@ fn provider_endpoint_should_persist_its_real_v1_endpoint() {
|
||||
vec!["/v1/images/generations".to_owned()],
|
||||
"execution error: {error:?}"
|
||||
);
|
||||
assert_eq!(store.requested_models(), vec![None]);
|
||||
assert_eq!(store.requested_models(), vec![model.map(str::to_owned)]);
|
||||
let upstream_models = store.upstream_models();
|
||||
assert!(!upstream_models.is_empty());
|
||||
assert!(upstream_models.iter().all(Option::is_none));
|
||||
|
||||
@@ -30,7 +30,10 @@ fn provider_error_debug_should_not_print_classified_upstream_values() {
|
||||
fn classified_provider_diagnostic_survives_clone_without_entering_debug() {
|
||||
let message = "OpenAI WebSocket closed before terminal response (close code 1000)";
|
||||
let error = ProviderError::new(ProviderErrorKind::Transport, UpstreamSendState::Ambiguous)
|
||||
.with_diagnostic(ProviderDiagnostic::new(message));
|
||||
.with_diagnostic(
|
||||
ProviderDiagnostic::new(message)
|
||||
.with_classification("receive", "closed_before_terminal"),
|
||||
);
|
||||
let cloned = error.clone();
|
||||
let gateway = GatewayError::from_provider(&cloned);
|
||||
|
||||
@@ -38,6 +41,14 @@ fn classified_provider_diagnostic_survives_clone_without_entering_debug() {
|
||||
gateway.diagnostic().map(ProviderDiagnostic::as_str),
|
||||
Some(message)
|
||||
);
|
||||
assert_eq!(
|
||||
gateway.diagnostic().and_then(ProviderDiagnostic::stage),
|
||||
Some("receive")
|
||||
);
|
||||
assert_eq!(
|
||||
gateway.diagnostic().and_then(ProviderDiagnostic::code),
|
||||
Some("closed_before_terminal")
|
||||
);
|
||||
assert!(!format!("{error:?}").contains(message));
|
||||
assert!(!format!("{gateway:?}").contains(message));
|
||||
}
|
||||
|
||||
@@ -118,16 +118,31 @@ pub struct LoggingConfig {
|
||||
pub level: String,
|
||||
pub stdout: bool,
|
||||
pub file: FileLoggingConfig,
|
||||
/// 将 OAuth 原始 AT/RT 写入独立恢复日志;默认关闭,沿用 file 的目录与保留期。
|
||||
#[serde(default)]
|
||||
pub oauth_recovery: bool,
|
||||
/// 将完整请求原文写入独立诊断日志;默认关闭,因为内容包含凭据。
|
||||
/// 请求报文的留存窗口由 request_dump_retention_days 独立控制。
|
||||
#[serde(default)]
|
||||
pub request_dump: bool,
|
||||
/// 完整报文的最少保留天数;按 UTC 日期整组清理,默认 1 天。
|
||||
#[serde(default = "default_request_dump_retention_days")]
|
||||
pub request_dump_retention_days: usize,
|
||||
}
|
||||
|
||||
fn default_file_retention_days() -> usize {
|
||||
7
|
||||
}
|
||||
|
||||
fn default_request_dump_retention_days() -> usize {
|
||||
1
|
||||
}
|
||||
|
||||
impl LoggingConfig {
|
||||
fn resolve_and_validate(&mut self, source_dir: &Path) -> Result<(), ConfigError> {
|
||||
EnvFilter::try_new(&self.level)
|
||||
.map_err(|_| ConfigError::InvalidField("host.logging.level"))?;
|
||||
if !self.stdout && !self.file.enabled {
|
||||
if !self.stdout && !self.file.enabled && !self.oauth_recovery && !self.request_dump {
|
||||
return Err(ConfigError::InvalidField("host.logging"));
|
||||
}
|
||||
if self.file.directory.as_os_str().is_empty() {
|
||||
@@ -143,8 +158,10 @@ impl LoggingConfig {
|
||||
"host.logging.file.max_file_size_mb",
|
||||
));
|
||||
}
|
||||
if self.file.max_files == 0 {
|
||||
return Err(ConfigError::InvalidField("host.logging.file.max_files"));
|
||||
if self.request_dump_retention_days == 0 {
|
||||
return Err(ConfigError::InvalidField(
|
||||
"host.logging.request_dump_retention_days",
|
||||
));
|
||||
}
|
||||
resolve_relative_path(source_dir, &mut self.file.directory);
|
||||
Ok(())
|
||||
@@ -157,9 +174,10 @@ impl LoggingConfig {
|
||||
pub struct FileLoggingConfig {
|
||||
pub enabled: bool,
|
||||
pub directory: PathBuf,
|
||||
/// 至少保留的完整天数,默认 7 天;文件数量不参与清理。
|
||||
#[serde(default = "default_file_retention_days")]
|
||||
pub retention_days: usize,
|
||||
pub max_file_size_mb: u64,
|
||||
pub max_files: usize,
|
||||
}
|
||||
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
|
||||
@@ -13,7 +13,7 @@ use axum::Router;
|
||||
use gateway_admin::ports::{
|
||||
client_distribution::ClientDistributionResolver, system::SystemOperations,
|
||||
};
|
||||
use gateway_core::health::WorkerHealthSource;
|
||||
use gateway_core::health::{HealthProbe, WorkerHealthSource};
|
||||
use gateway_core::lifecycle::CancellationToken;
|
||||
use gateway_core::lifecycle::ConnectionLifecycle;
|
||||
use gateway_core::task::{WorkerContribution, WorkerLeaderLeasePort};
|
||||
@@ -29,7 +29,7 @@ use self::workers::WorkerSupervisor;
|
||||
/// Host 初始化的能力集;字段全部私有,不暴露内部监督器或进程状态。
|
||||
pub struct HostBundle {
|
||||
config: HostConfig,
|
||||
_log_guard: LogGuard,
|
||||
log_guard: LogGuard,
|
||||
cancellation: CancellationToken,
|
||||
connections: Arc<ConnectionTracker>,
|
||||
workers: WorkerSupervisor,
|
||||
@@ -50,7 +50,7 @@ pub async fn initialize(config: HostConfig) -> Result<HostBundle, HostError> {
|
||||
let client_distribution = Arc::new(RgAdguardClientDistribution::new());
|
||||
Ok(HostBundle {
|
||||
config,
|
||||
_log_guard: log_guard,
|
||||
log_guard,
|
||||
cancellation,
|
||||
connections,
|
||||
workers,
|
||||
@@ -86,6 +86,12 @@ impl HostBundle {
|
||||
self.connections.clone()
|
||||
}
|
||||
|
||||
/// 报告文件日志写入或归档失败,避免将缺失日志误报为完整。
|
||||
#[must_use]
|
||||
pub fn logging_health_probe(&self) -> Arc<dyn HealthProbe> {
|
||||
self.log_guard.health_probe()
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn worker_health(&self) -> Arc<dyn WorkerHealthSource> {
|
||||
self.workers.health_source()
|
||||
|
||||
@@ -1,385 +0,0 @@
|
||||
//! 进程结构化日志与按日期/大小轮转的文件 writer。
|
||||
|
||||
use std::env;
|
||||
use std::fs::{self, File, OpenOptions};
|
||||
use std::io::{self, Write};
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
use chrono::Utc;
|
||||
use tracing_appender::non_blocking::{NonBlocking, NonBlockingBuilder, WorkerGuard};
|
||||
use tracing_subscriber::EnvFilter;
|
||||
use tracing_subscriber::{Layer as _, layer::SubscriberExt as _, util::SubscriberInitExt as _};
|
||||
|
||||
use crate::config::{FileLoggingConfig, LoggingConfig};
|
||||
|
||||
const LOG_FILE_PREFIX: &str = "codex-proxy-rs";
|
||||
const OAUTH_RECOVERY_LOG_FILE_PREFIX: &str = "codex-proxy-rs-oath";
|
||||
/// OAuth 恢复记录含原始 AT/RT,文件日志开启时必须落到独立文件。
|
||||
const OAUTH_RECOVERY_LOG_TARGET: &str = "oauth_recovery";
|
||||
const REQUEST_DUMP_LOG_FILE_PREFIX: &str = "codex-proxy-rs-request-dump";
|
||||
/// 请求转储含完整凭据与正文,只允许在显式开启时写入独立文件。
|
||||
const REQUEST_DUMP_LOG_TARGET: &str = "request_dump";
|
||||
|
||||
/// non-blocking 日志 writer 的进程级守卫。
|
||||
pub struct LogGuard {
|
||||
_writers: Vec<WorkerGuard>,
|
||||
}
|
||||
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub enum LogError {
|
||||
#[error("log IO failed")]
|
||||
Io(#[from] io::Error),
|
||||
#[error("logging filter is invalid")]
|
||||
InvalidFilter,
|
||||
#[error("global tracing subscriber is already initialized")]
|
||||
AlreadyInitialized,
|
||||
#[error("logging size limit is too large")]
|
||||
SizeOverflow,
|
||||
}
|
||||
|
||||
/// 按自然日、单文件大小、保留天数和文件总数初始化结构化日志。
|
||||
pub fn initialize_logging(config: &LoggingConfig) -> Result<LogGuard, LogError> {
|
||||
let directive = env::var("RUST_LOG").unwrap_or_else(|_| config.level.clone());
|
||||
let file_filter = application_file_filter(&directive)?;
|
||||
let recovery_file_filter = oauth_recovery_file_filter();
|
||||
let request_dump_file_filter = request_dump_file_filter();
|
||||
let stdout_filter =
|
||||
EnvFilter::try_new(stdout_filter_directive(&directive, config.file.enabled))
|
||||
.map_err(|_| LogError::InvalidFilter)?;
|
||||
let mut guards = Vec::new();
|
||||
|
||||
let stdout_writer = config.stdout.then(|| {
|
||||
let (writer, guard) = NonBlockingBuilder::default()
|
||||
.thread_name("gateway-log-stdout")
|
||||
.finish(io::stdout());
|
||||
guards.push(guard);
|
||||
writer
|
||||
});
|
||||
let (file_writer, recovery_file_writer) = if config.file.enabled {
|
||||
let (file_writer, file_guard) =
|
||||
rotating_file_writer(&config.file, LOG_FILE_PREFIX, "gateway-log-file")?;
|
||||
let (recovery_file_writer, recovery_file_guard) = rotating_file_writer(
|
||||
&config.file,
|
||||
OAUTH_RECOVERY_LOG_FILE_PREFIX,
|
||||
"gateway-log-oauth-recovery-file",
|
||||
)?;
|
||||
guards.push(file_guard);
|
||||
guards.push(recovery_file_guard);
|
||||
(Some(file_writer), Some(recovery_file_writer))
|
||||
} else {
|
||||
(None, None)
|
||||
};
|
||||
let request_dump_file_writer = if config.request_dump {
|
||||
let (writer, guard) = rotating_file_writer(
|
||||
&config.file,
|
||||
REQUEST_DUMP_LOG_FILE_PREFIX,
|
||||
"gateway-log-request-dump-file",
|
||||
)?;
|
||||
guards.push(guard);
|
||||
Some(writer)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
let stdout_layer = stdout_writer.map(|writer| {
|
||||
tracing_subscriber::fmt::layer()
|
||||
.compact()
|
||||
.with_writer(writer)
|
||||
.with_target(false)
|
||||
.with_ansi(false)
|
||||
.with_filter(stdout_filter)
|
||||
});
|
||||
let file_layer = file_writer.map(|writer| {
|
||||
tracing_subscriber::fmt::layer()
|
||||
.json()
|
||||
.with_writer(writer)
|
||||
.with_target(true)
|
||||
.with_file(true)
|
||||
.with_line_number(true)
|
||||
.with_thread_ids(true)
|
||||
.with_thread_names(true)
|
||||
.with_current_span(true)
|
||||
.with_span_list(true)
|
||||
.with_filter(file_filter)
|
||||
});
|
||||
let recovery_file_layer = recovery_file_writer.map(|writer| {
|
||||
tracing_subscriber::fmt::layer()
|
||||
.json()
|
||||
.with_writer(writer)
|
||||
.with_target(true)
|
||||
.with_file(true)
|
||||
.with_line_number(true)
|
||||
.with_thread_ids(true)
|
||||
.with_thread_names(true)
|
||||
.with_current_span(true)
|
||||
.with_span_list(true)
|
||||
.with_filter(recovery_file_filter)
|
||||
});
|
||||
let request_dump_file_layer = request_dump_file_writer.map(|writer| {
|
||||
tracing_subscriber::fmt::layer()
|
||||
.json()
|
||||
.with_writer(writer)
|
||||
.with_target(true)
|
||||
.with_file(true)
|
||||
.with_line_number(true)
|
||||
.with_thread_ids(true)
|
||||
.with_thread_names(true)
|
||||
.with_current_span(true)
|
||||
.with_span_list(true)
|
||||
.with_filter(request_dump_file_filter)
|
||||
});
|
||||
tracing_subscriber::registry()
|
||||
.with(stdout_layer)
|
||||
.with(file_layer)
|
||||
.with(recovery_file_layer)
|
||||
.with(request_dump_file_layer)
|
||||
.try_init()
|
||||
.map_err(|_| LogError::AlreadyInitialized)?;
|
||||
Ok(LogGuard { _writers: guards })
|
||||
}
|
||||
|
||||
fn application_file_filter(directive: &str) -> Result<EnvFilter, LogError> {
|
||||
let recovery_directive = format!("{OAUTH_RECOVERY_LOG_TARGET}=off")
|
||||
.parse()
|
||||
.expect("static OAuth recovery log directive is valid");
|
||||
let request_dump_directive = format!("{REQUEST_DUMP_LOG_TARGET}=off")
|
||||
.parse()
|
||||
.expect("static request dump log directive is valid");
|
||||
EnvFilter::try_new(directive)
|
||||
.map(|filter| filter.add_directive(recovery_directive))
|
||||
.map(|filter| filter.add_directive(request_dump_directive))
|
||||
.map_err(|_| LogError::InvalidFilter)
|
||||
}
|
||||
|
||||
fn oauth_recovery_file_filter() -> EnvFilter {
|
||||
format!("off,{OAUTH_RECOVERY_LOG_TARGET}=info")
|
||||
.parse()
|
||||
.expect("static OAuth recovery log filter is valid")
|
||||
}
|
||||
|
||||
fn request_dump_file_filter() -> EnvFilter {
|
||||
format!("off,{REQUEST_DUMP_LOG_TARGET}=info")
|
||||
.parse()
|
||||
.expect("static request dump log filter is valid")
|
||||
}
|
||||
|
||||
fn stdout_filter_directive(directive: &str, persistent_log_enabled: bool) -> String {
|
||||
if !persistent_log_enabled {
|
||||
format!("{directive},{REQUEST_DUMP_LOG_TARGET}=off")
|
||||
} else {
|
||||
"off,gateway_startup=info".to_owned()
|
||||
}
|
||||
}
|
||||
|
||||
fn rotating_file_writer(
|
||||
config: &FileLoggingConfig,
|
||||
file_prefix: &'static str,
|
||||
thread_name: &'static str,
|
||||
) -> Result<(NonBlocking, WorkerGuard), LogError> {
|
||||
let maximum_bytes = config
|
||||
.max_file_size_mb
|
||||
.checked_mul(1024 * 1024)
|
||||
.ok_or(LogError::SizeOverflow)?;
|
||||
let writer = RotatingLogWriter::open(
|
||||
config.directory.clone(),
|
||||
file_prefix,
|
||||
maximum_bytes,
|
||||
config.retention_days,
|
||||
config.max_files,
|
||||
)?;
|
||||
Ok(NonBlockingBuilder::default()
|
||||
.thread_name(thread_name)
|
||||
.finish(writer))
|
||||
}
|
||||
|
||||
struct RotatingLogWriter {
|
||||
directory: PathBuf,
|
||||
file_prefix: &'static str,
|
||||
maximum_bytes: u64,
|
||||
retention_days: usize,
|
||||
maximum_files: usize,
|
||||
date: chrono::NaiveDate,
|
||||
segment: usize,
|
||||
bytes_written: u64,
|
||||
file: File,
|
||||
}
|
||||
|
||||
impl RotatingLogWriter {
|
||||
fn open(
|
||||
directory: PathBuf,
|
||||
file_prefix: &'static str,
|
||||
maximum_bytes: u64,
|
||||
retention_days: usize,
|
||||
maximum_files: usize,
|
||||
) -> io::Result<Self> {
|
||||
fs::create_dir_all(&directory)?;
|
||||
let date = Utc::now().date_naive();
|
||||
cleanup_log_files(&directory, file_prefix, date, retention_days, maximum_files)?;
|
||||
let (segment, bytes_written) =
|
||||
writable_log_segment(&directory, file_prefix, date, maximum_bytes)?;
|
||||
let file = open_log_segment(&directory, file_prefix, date, segment)?;
|
||||
Ok(Self {
|
||||
directory,
|
||||
file_prefix,
|
||||
maximum_bytes,
|
||||
retention_days,
|
||||
maximum_files,
|
||||
date,
|
||||
segment,
|
||||
bytes_written,
|
||||
file,
|
||||
})
|
||||
}
|
||||
|
||||
fn rotate_if_required(&mut self, incoming_bytes: usize) -> io::Result<()> {
|
||||
let date = Utc::now().date_naive();
|
||||
let incoming_bytes = u64::try_from(incoming_bytes).unwrap_or(u64::MAX);
|
||||
let day_changed = date != self.date;
|
||||
let size_exceeded = !day_changed
|
||||
&& self.bytes_written > 0
|
||||
&& self.bytes_written.saturating_add(incoming_bytes) > self.maximum_bytes;
|
||||
if !day_changed && !size_exceeded {
|
||||
return Ok(());
|
||||
}
|
||||
self.file.flush()?;
|
||||
if day_changed {
|
||||
self.date = date;
|
||||
self.segment = 0;
|
||||
} else {
|
||||
self.segment = self.segment.saturating_add(1);
|
||||
}
|
||||
self.file = open_log_segment(&self.directory, self.file_prefix, self.date, self.segment)?;
|
||||
self.bytes_written = self.file.metadata()?.len();
|
||||
cleanup_log_files(
|
||||
&self.directory,
|
||||
self.file_prefix,
|
||||
self.date,
|
||||
self.retention_days,
|
||||
self.maximum_files,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
impl Write for RotatingLogWriter {
|
||||
fn write(&mut self, buffer: &[u8]) -> io::Result<usize> {
|
||||
self.rotate_if_required(buffer.len())?;
|
||||
let written = self.file.write(buffer)?;
|
||||
self.bytes_written = self
|
||||
.bytes_written
|
||||
.saturating_add(u64::try_from(written).unwrap_or(u64::MAX));
|
||||
Ok(written)
|
||||
}
|
||||
|
||||
fn flush(&mut self) -> io::Result<()> {
|
||||
self.file.flush()
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
struct ManagedLogFile {
|
||||
date: chrono::NaiveDate,
|
||||
segment: usize,
|
||||
path: PathBuf,
|
||||
}
|
||||
|
||||
fn writable_log_segment(
|
||||
directory: &Path,
|
||||
file_prefix: &str,
|
||||
date: chrono::NaiveDate,
|
||||
maximum_bytes: u64,
|
||||
) -> io::Result<(usize, u64)> {
|
||||
let latest = managed_log_files(directory, file_prefix)?
|
||||
.into_iter()
|
||||
.filter(|entry| entry.date == date)
|
||||
.max_by_key(|entry| entry.segment);
|
||||
let Some(latest) = latest else {
|
||||
return Ok((0, 0));
|
||||
};
|
||||
let length = latest.path.metadata()?.len();
|
||||
if length >= maximum_bytes {
|
||||
Ok((latest.segment.saturating_add(1), 0))
|
||||
} else {
|
||||
Ok((latest.segment, length))
|
||||
}
|
||||
}
|
||||
|
||||
fn open_log_segment(
|
||||
directory: &Path,
|
||||
file_prefix: &str,
|
||||
date: chrono::NaiveDate,
|
||||
segment: usize,
|
||||
) -> io::Result<File> {
|
||||
OpenOptions::new()
|
||||
.create(true)
|
||||
.append(true)
|
||||
.open(directory.join(log_file_name(file_prefix, date, segment)))
|
||||
}
|
||||
|
||||
fn log_file_name(file_prefix: &str, date: chrono::NaiveDate, segment: usize) -> String {
|
||||
if segment == 0 {
|
||||
format!("{file_prefix}.{date}.log")
|
||||
} else {
|
||||
format!("{file_prefix}.{date}.{segment}.log")
|
||||
}
|
||||
}
|
||||
|
||||
fn managed_log_files(directory: &Path, file_prefix: &str) -> io::Result<Vec<ManagedLogFile>> {
|
||||
Ok(fs::read_dir(directory)?
|
||||
.filter_map(Result::ok)
|
||||
.filter_map(|entry| {
|
||||
parse_log_file_name(entry.file_name().to_string_lossy().as_ref(), file_prefix).map(
|
||||
|(date, segment)| ManagedLogFile {
|
||||
date,
|
||||
segment,
|
||||
path: entry.path(),
|
||||
},
|
||||
)
|
||||
})
|
||||
.collect())
|
||||
}
|
||||
|
||||
fn parse_log_file_name(name: &str, file_prefix: &str) -> Option<(chrono::NaiveDate, usize)> {
|
||||
let body = name
|
||||
.strip_prefix(&format!("{file_prefix}."))?
|
||||
.strip_suffix(".log")?;
|
||||
let (date, segment) = body.split_once('.').map_or((body, 0), |(date, segment)| {
|
||||
(date, segment.parse().ok().unwrap_or(usize::MAX))
|
||||
});
|
||||
if segment == usize::MAX {
|
||||
return None;
|
||||
}
|
||||
chrono::NaiveDate::parse_from_str(date, "%Y-%m-%d")
|
||||
.ok()
|
||||
.map(|date| (date, segment))
|
||||
}
|
||||
|
||||
fn cleanup_log_files(
|
||||
directory: &Path,
|
||||
file_prefix: &str,
|
||||
today: chrono::NaiveDate,
|
||||
retention_days: usize,
|
||||
maximum_files: usize,
|
||||
) -> io::Result<()> {
|
||||
let retention_offset = i64::try_from(retention_days.saturating_sub(1)).unwrap_or(i64::MAX);
|
||||
let cutoff = today
|
||||
.checked_sub_signed(chrono::Duration::days(retention_offset))
|
||||
.unwrap_or(chrono::NaiveDate::MIN);
|
||||
let mut retained = Vec::new();
|
||||
for entry in managed_log_files(directory, file_prefix)? {
|
||||
if entry.date < cutoff {
|
||||
fs::remove_file(entry.path)?;
|
||||
} else {
|
||||
retained.push(entry);
|
||||
}
|
||||
}
|
||||
retained.sort_by(|left, right| {
|
||||
right
|
||||
.date
|
||||
.cmp(&left.date)
|
||||
.then_with(|| right.segment.cmp(&left.segment))
|
||||
});
|
||||
for entry in retained.into_iter().skip(maximum_files) {
|
||||
fs::remove_file(entry.path)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
@@ -0,0 +1,230 @@
|
||||
//! 进程结构化日志与按日期/大小轮转的文件 writer。
|
||||
|
||||
use std::env;
|
||||
use std::io;
|
||||
use std::sync::Arc;
|
||||
|
||||
use gateway_core::health::HealthProbe;
|
||||
use tracing_appender::non_blocking::{NonBlockingBuilder, WorkerGuard};
|
||||
use tracing_subscriber::EnvFilter;
|
||||
use tracing_subscriber::{Layer as _, layer::SubscriberExt as _, util::SubscriberInitExt as _};
|
||||
|
||||
use crate::config::LoggingConfig;
|
||||
|
||||
mod sink;
|
||||
mod writer;
|
||||
|
||||
use sink::{FileLogGuard, FileLogSink, LogHealth};
|
||||
use writer::RotatingLogWriter;
|
||||
|
||||
// 文件前缀统一为 codex-proxy-rs-<kebab-case 类别>;专用 tracing target 使用 snake_case。
|
||||
const APPLICATION_LOG_FILE_PREFIX: &str = "codex-proxy-rs-application";
|
||||
|
||||
/// 内部分类标识;由 host.logging.oauth_recovery 控制是否写入独立文件。
|
||||
const OAUTH_RECOVERY_LOG_TARGET: &str = "oauth_recovery";
|
||||
const OAUTH_RECOVERY_LOG_FILE_PREFIX: &str = "codex-proxy-rs-oauth-recovery";
|
||||
|
||||
/// 请求转储含完整凭据与正文,只允许在显式开启时写入独立文件。
|
||||
const REQUEST_DUMP_LOG_TARGET: &str = "request_dump";
|
||||
const REQUEST_DUMP_LOG_FILE_PREFIX: &str = "codex-proxy-rs-request-dump";
|
||||
|
||||
/// 文件日志在正常退出时排空队列并等待落盘;控制台日志单独持有守卫。
|
||||
pub struct LogGuard {
|
||||
_stdout: Vec<WorkerGuard>,
|
||||
_files: Vec<FileLogGuard>,
|
||||
health: Arc<LogHealth>,
|
||||
}
|
||||
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub enum LogError {
|
||||
#[error("log IO failed")]
|
||||
Io(#[from] io::Error),
|
||||
#[error("logging filter is invalid")]
|
||||
InvalidFilter,
|
||||
#[error("global tracing subscriber is already initialized")]
|
||||
AlreadyInitialized,
|
||||
#[error("logging size limit is too large")]
|
||||
SizeOverflow,
|
||||
}
|
||||
|
||||
/// 初始化日志;文件按完整日期留存,不以分片数量淘汰窗口内记录。
|
||||
pub fn initialize_logging(config: &LoggingConfig) -> Result<LogGuard, LogError> {
|
||||
let directive = env::var("RUST_LOG").unwrap_or_else(|_| config.level.clone());
|
||||
let file_filter = application_file_filter(&directive)?;
|
||||
let recovery_file_filter = oauth_recovery_file_filter();
|
||||
let request_dump_file_filter = request_dump_file_filter();
|
||||
let stdout_filter =
|
||||
EnvFilter::try_new(stdout_filter_directive(&directive, config.file.enabled))
|
||||
.map_err(|_| LogError::InvalidFilter)?;
|
||||
let mut guards = Vec::new();
|
||||
let mut file_guards = Vec::new();
|
||||
let health = Arc::new(LogHealth::default());
|
||||
|
||||
let stdout_writer = config.stdout.then(|| {
|
||||
let (writer, guard) = NonBlockingBuilder::default()
|
||||
.thread_name("gateway-log-stdout")
|
||||
.finish(io::stdout());
|
||||
guards.push(guard);
|
||||
writer
|
||||
});
|
||||
let file_writer = if config.file.enabled {
|
||||
let (writer, guard) = create_file_writer(
|
||||
config,
|
||||
config.file.retention_days,
|
||||
APPLICATION_LOG_FILE_PREFIX,
|
||||
"gateway-log-application-file",
|
||||
Arc::clone(&health),
|
||||
)?;
|
||||
file_guards.push(guard);
|
||||
Some(writer)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let recovery_file_writer = if config.oauth_recovery {
|
||||
let (writer, guard) = create_file_writer(
|
||||
config,
|
||||
config.file.retention_days,
|
||||
OAUTH_RECOVERY_LOG_FILE_PREFIX,
|
||||
"gateway-log-oauth-recovery-file",
|
||||
Arc::clone(&health),
|
||||
)?;
|
||||
file_guards.push(guard);
|
||||
Some(writer)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let request_dump_file_writer = if config.request_dump {
|
||||
let (writer, guard) = create_file_writer(
|
||||
config,
|
||||
config.request_dump_retention_days,
|
||||
REQUEST_DUMP_LOG_FILE_PREFIX,
|
||||
"gateway-log-request-dump-file",
|
||||
Arc::clone(&health),
|
||||
)?;
|
||||
file_guards.push(guard);
|
||||
Some(writer)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
let stdout_layer = stdout_writer.map(|writer| {
|
||||
tracing_subscriber::fmt::layer()
|
||||
.compact()
|
||||
.with_writer(writer)
|
||||
.with_target(false)
|
||||
.with_ansi(false)
|
||||
.with_filter(stdout_filter)
|
||||
});
|
||||
let file_layer = file_writer.map(|writer| {
|
||||
tracing_subscriber::fmt::layer()
|
||||
.json()
|
||||
.with_writer(writer)
|
||||
.with_target(true)
|
||||
.with_file(true)
|
||||
.with_line_number(true)
|
||||
.with_thread_ids(true)
|
||||
.with_thread_names(true)
|
||||
.with_current_span(true)
|
||||
.with_span_list(true)
|
||||
.with_filter(file_filter)
|
||||
});
|
||||
let recovery_file_layer = recovery_file_writer.map(|writer| {
|
||||
tracing_subscriber::fmt::layer()
|
||||
.json()
|
||||
.with_writer(writer)
|
||||
.with_target(true)
|
||||
.with_file(true)
|
||||
.with_line_number(true)
|
||||
.with_thread_ids(true)
|
||||
.with_thread_names(true)
|
||||
.with_current_span(true)
|
||||
.with_span_list(true)
|
||||
.with_filter(recovery_file_filter)
|
||||
});
|
||||
let request_dump_file_layer = request_dump_file_writer.map(|writer| {
|
||||
tracing_subscriber::fmt::layer()
|
||||
.json()
|
||||
.with_writer(writer)
|
||||
.with_target(true)
|
||||
.with_file(true)
|
||||
.with_line_number(true)
|
||||
.with_thread_ids(true)
|
||||
.with_thread_names(true)
|
||||
.with_current_span(true)
|
||||
.with_span_list(true)
|
||||
.with_filter(request_dump_file_filter)
|
||||
});
|
||||
tracing_subscriber::registry()
|
||||
.with(stdout_layer)
|
||||
.with(file_layer)
|
||||
.with(recovery_file_layer)
|
||||
.with(request_dump_file_layer)
|
||||
.try_init()
|
||||
.map_err(|_| LogError::AlreadyInitialized)?;
|
||||
Ok(LogGuard {
|
||||
_stdout: guards,
|
||||
_files: file_guards,
|
||||
health,
|
||||
})
|
||||
}
|
||||
|
||||
fn application_file_filter(directive: &str) -> Result<EnvFilter, LogError> {
|
||||
let recovery_directive = format!("{OAUTH_RECOVERY_LOG_TARGET}=off")
|
||||
.parse()
|
||||
.expect("static OAuth recovery log directive is valid");
|
||||
let request_dump_directive = format!("{REQUEST_DUMP_LOG_TARGET}=off")
|
||||
.parse()
|
||||
.expect("static request dump log directive is valid");
|
||||
EnvFilter::try_new(directive)
|
||||
.map(|filter| filter.add_directive(recovery_directive))
|
||||
.map(|filter| filter.add_directive(request_dump_directive))
|
||||
.map_err(|_| LogError::InvalidFilter)
|
||||
}
|
||||
|
||||
fn oauth_recovery_file_filter() -> EnvFilter {
|
||||
format!("off,{OAUTH_RECOVERY_LOG_TARGET}=info")
|
||||
.parse()
|
||||
.expect("static OAuth recovery log filter is valid")
|
||||
}
|
||||
|
||||
fn request_dump_file_filter() -> EnvFilter {
|
||||
format!("off,{REQUEST_DUMP_LOG_TARGET}=info")
|
||||
.parse()
|
||||
.expect("static request dump log filter is valid")
|
||||
}
|
||||
|
||||
fn stdout_filter_directive(directive: &str, persistent_log_enabled: bool) -> String {
|
||||
if !persistent_log_enabled {
|
||||
format!("{directive},{OAUTH_RECOVERY_LOG_TARGET}=off,{REQUEST_DUMP_LOG_TARGET}=off")
|
||||
} else {
|
||||
"off,gateway_startup=info".to_owned()
|
||||
}
|
||||
}
|
||||
|
||||
impl LogGuard {
|
||||
pub(crate) fn health_probe(&self) -> Arc<dyn HealthProbe> {
|
||||
self.health.clone()
|
||||
}
|
||||
}
|
||||
|
||||
fn create_file_writer(
|
||||
config: &LoggingConfig,
|
||||
retention_days: usize,
|
||||
prefix: &'static str,
|
||||
thread_name: &'static str,
|
||||
health: Arc<LogHealth>,
|
||||
) -> Result<(FileLogSink, FileLogGuard), LogError> {
|
||||
let maximum_bytes = config
|
||||
.file
|
||||
.max_file_size_mb
|
||||
.checked_mul(1024 * 1024)
|
||||
.ok_or(LogError::SizeOverflow)?;
|
||||
let writer = RotatingLogWriter::open(
|
||||
config.file.directory.clone(),
|
||||
prefix,
|
||||
maximum_bytes,
|
||||
retention_days,
|
||||
Arc::clone(&health),
|
||||
)?;
|
||||
Ok(FileLogSink::spawn(writer, thread_name, health)?)
|
||||
}
|
||||
@@ -0,0 +1,154 @@
|
||||
//! 有界文件日志队列:拥堵时背压,正常退出时排空;写入失败显式影响健康状态。
|
||||
|
||||
use std::io::{self, Write};
|
||||
use std::sync::atomic::{AtomicU64, Ordering};
|
||||
use std::sync::{Arc, mpsc};
|
||||
use std::thread::{self, JoinHandle};
|
||||
|
||||
use futures::future::BoxFuture;
|
||||
use gateway_core::health::{HealthProbe, HealthState};
|
||||
use tracing_subscriber::fmt::MakeWriter;
|
||||
|
||||
use super::writer::RotatingLogWriter;
|
||||
|
||||
const QUEUE_CAPACITY: usize = 64;
|
||||
|
||||
enum Command {
|
||||
Write(Vec<u8>),
|
||||
Shutdown,
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
pub(super) struct LogHealth {
|
||||
write_failures: AtomicU64,
|
||||
maintenance_failures: AtomicU64,
|
||||
}
|
||||
|
||||
impl LogHealth {
|
||||
pub(super) fn write_failed(&self, kind: io::ErrorKind) {
|
||||
if self.write_failures.fetch_add(1, Ordering::Relaxed) == 0 {
|
||||
eprintln!("file_logging: log completeness compromised; write failure ({kind:?})");
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn maintenance_failed(&self, kind: io::ErrorKind) {
|
||||
if self.maintenance_failures.fetch_add(1, Ordering::Relaxed) == 0 {
|
||||
eprintln!("file_logging: archive/retention maintenance failed ({kind:?})");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl HealthProbe for LogHealth {
|
||||
fn name(&self) -> &'static str {
|
||||
"file_logging"
|
||||
}
|
||||
|
||||
fn check(&self) -> BoxFuture<'_, HealthState> {
|
||||
Box::pin(async move {
|
||||
let failures = self.write_failures.load(Ordering::Relaxed);
|
||||
if failures > 0 {
|
||||
return HealthState::Unhealthy(format!(
|
||||
"log completeness compromised: {failures} write/flush failures since startup"
|
||||
));
|
||||
}
|
||||
let failures = self.maintenance_failures.load(Ordering::Relaxed);
|
||||
if failures > 0 {
|
||||
return HealthState::Degraded(format!(
|
||||
"log archive/retention maintenance failed {failures} times since startup"
|
||||
));
|
||||
}
|
||||
HealthState::Healthy
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
pub(super) struct FileLogSink {
|
||||
sender: mpsc::SyncSender<Command>,
|
||||
health: Arc<LogHealth>,
|
||||
}
|
||||
|
||||
pub(super) struct FileLogGuard {
|
||||
sender: mpsc::SyncSender<Command>,
|
||||
worker: Option<JoinHandle<()>>,
|
||||
health: Arc<LogHealth>,
|
||||
}
|
||||
|
||||
impl FileLogSink {
|
||||
pub(super) fn spawn(
|
||||
mut writer: RotatingLogWriter,
|
||||
name: &'static str,
|
||||
health: Arc<LogHealth>,
|
||||
) -> io::Result<(Self, FileLogGuard)> {
|
||||
let (sender, receiver) = mpsc::sync_channel(QUEUE_CAPACITY);
|
||||
let worker_health = Arc::clone(&health);
|
||||
let worker = thread::Builder::new()
|
||||
.name(name.to_owned())
|
||||
.spawn(move || {
|
||||
while let Ok(command) = receiver.recv() {
|
||||
match command {
|
||||
Command::Write(bytes) => {
|
||||
if let Err(error) = writer.write_all(&bytes) {
|
||||
worker_health.write_failed(error.kind());
|
||||
}
|
||||
}
|
||||
Command::Shutdown => break,
|
||||
}
|
||||
}
|
||||
if let Err(error) = writer.sync() {
|
||||
worker_health.write_failed(error.kind());
|
||||
}
|
||||
})?;
|
||||
Ok((
|
||||
Self {
|
||||
sender: sender.clone(),
|
||||
health: Arc::clone(&health),
|
||||
},
|
||||
FileLogGuard {
|
||||
sender,
|
||||
worker: Some(worker),
|
||||
health,
|
||||
},
|
||||
))
|
||||
}
|
||||
}
|
||||
|
||||
impl<'a> MakeWriter<'a> for FileLogSink {
|
||||
type Writer = Self;
|
||||
|
||||
fn make_writer(&'a self) -> Self::Writer {
|
||||
self.clone()
|
||||
}
|
||||
}
|
||||
|
||||
impl Write for FileLogSink {
|
||||
fn write(&mut self, buffer: &[u8]) -> io::Result<usize> {
|
||||
self.sender
|
||||
.send(Command::Write(buffer.to_vec()))
|
||||
.map_err(|_| {
|
||||
self.health.write_failed(io::ErrorKind::BrokenPipe);
|
||||
io::Error::new(io::ErrorKind::BrokenPipe, "file log worker unavailable")
|
||||
})?;
|
||||
Ok(buffer.len())
|
||||
}
|
||||
|
||||
fn flush(&mut self) -> io::Result<()> {
|
||||
// Files are unbuffered. The guard is the lifecycle owner of queue drain and sync.
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for FileLogGuard {
|
||||
fn drop(&mut self) {
|
||||
// Shutdown follows every accepted record. Unlike a timed guard, joining cannot
|
||||
// silently abandon queued log records during a normal process shutdown.
|
||||
let _ = self.sender.send(Command::Shutdown);
|
||||
if self
|
||||
.worker
|
||||
.take()
|
||||
.is_some_and(|worker| worker.join().is_err())
|
||||
{
|
||||
self.health.write_failed(io::ErrorKind::BrokenPipe);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,246 @@
|
||||
//! 按大小轮转、压缩已关闭分片、按完整 UTC 日期组清理。
|
||||
|
||||
use std::collections::BTreeMap;
|
||||
use std::fs::{self, File, OpenOptions};
|
||||
use std::io::{self, Write};
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::Arc;
|
||||
|
||||
use chrono::{NaiveDate, Utc};
|
||||
use flate2::{Compression, write::GzEncoder};
|
||||
|
||||
use super::sink::LogHealth;
|
||||
|
||||
pub(super) struct RotatingLogWriter {
|
||||
directory: PathBuf,
|
||||
prefix: &'static str,
|
||||
maximum_bytes: u64,
|
||||
retention_days: usize,
|
||||
date: NaiveDate,
|
||||
segment: usize,
|
||||
bytes_written: u64,
|
||||
file: File,
|
||||
health: Arc<LogHealth>,
|
||||
}
|
||||
|
||||
impl RotatingLogWriter {
|
||||
pub(super) fn open(
|
||||
directory: PathBuf,
|
||||
prefix: &'static str,
|
||||
maximum_bytes: u64,
|
||||
retention_days: usize,
|
||||
health: Arc<LogHealth>,
|
||||
) -> io::Result<Self> {
|
||||
fs::create_dir_all(&directory)?;
|
||||
let date = Utc::now().date_naive();
|
||||
cleanup_log_files(&directory, prefix, date, retention_days)?;
|
||||
let latest = managed_log_files(&directory, prefix)?
|
||||
.into_iter()
|
||||
.filter(|entry| entry.date == date)
|
||||
.max_by_key(|entry| (entry.segment, entry.compressed));
|
||||
let (segment, bytes_written) = match latest {
|
||||
Some(entry) if !entry.compressed && entry.path.metadata()?.len() < maximum_bytes => {
|
||||
(entry.segment, entry.path.metadata()?.len())
|
||||
}
|
||||
Some(entry) => (next_segment(entry.segment)?, 0),
|
||||
None => (0, 0),
|
||||
};
|
||||
let path = directory.join(log_file_name(prefix, date, segment));
|
||||
let file = open_log_segment(&path)?;
|
||||
let writer = Self {
|
||||
directory,
|
||||
prefix,
|
||||
maximum_bytes,
|
||||
retention_days,
|
||||
date,
|
||||
segment,
|
||||
bytes_written,
|
||||
file,
|
||||
health,
|
||||
};
|
||||
// Recover unfinished archive work after restart; the active file is never compressed.
|
||||
for entry in managed_log_files(&writer.directory, prefix)? {
|
||||
if !entry.compressed
|
||||
&& entry.path != path
|
||||
&& let Err(error) = compress_log_file(&entry.path)
|
||||
{
|
||||
writer.health.maintenance_failed(error.kind());
|
||||
}
|
||||
}
|
||||
Ok(writer)
|
||||
}
|
||||
|
||||
fn rotate_if_required(&mut self, incoming_bytes: usize) -> io::Result<()> {
|
||||
let date = Utc::now().date_naive();
|
||||
let day_changed = date != self.date;
|
||||
let size_exceeded = self.bytes_written > 0
|
||||
&& self.bytes_written.saturating_add(incoming_bytes as u64) > self.maximum_bytes;
|
||||
if !day_changed && !size_exceeded {
|
||||
return Ok(());
|
||||
}
|
||||
let previous = self
|
||||
.directory
|
||||
.join(log_file_name(self.prefix, self.date, self.segment));
|
||||
self.file.sync_all()?;
|
||||
let segment = if day_changed {
|
||||
managed_log_files(&self.directory, self.prefix)?
|
||||
.into_iter()
|
||||
.filter(|entry| entry.date == date)
|
||||
.map(|entry| entry.segment)
|
||||
.max()
|
||||
.map_or(Ok(0), next_segment)?
|
||||
} else {
|
||||
next_segment(self.segment)?
|
||||
};
|
||||
let path = self
|
||||
.directory
|
||||
.join(log_file_name(self.prefix, date, segment));
|
||||
let file = open_log_segment(&path)?;
|
||||
// Commit rotation state only after opening the new file succeeds.
|
||||
self.file = file;
|
||||
self.date = date;
|
||||
self.segment = segment;
|
||||
self.bytes_written = self.file.metadata()?.len();
|
||||
if let Err(error) = compress_log_file(&previous) {
|
||||
self.health.maintenance_failed(error.kind());
|
||||
}
|
||||
// Maintenance errors must not discard the record that triggered rotation.
|
||||
if let Err(error) =
|
||||
cleanup_log_files(&self.directory, self.prefix, date, self.retention_days)
|
||||
{
|
||||
self.health.maintenance_failed(error.kind());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(super) fn sync(&mut self) -> io::Result<()> {
|
||||
self.file.sync_all()
|
||||
}
|
||||
}
|
||||
|
||||
impl Write for RotatingLogWriter {
|
||||
fn write(&mut self, buffer: &[u8]) -> io::Result<usize> {
|
||||
self.rotate_if_required(buffer.len())?;
|
||||
if let Err(error) = self.file.write_all(buffer) {
|
||||
self.bytes_written = self.file.metadata()?.len();
|
||||
return Err(error);
|
||||
}
|
||||
self.bytes_written = self.bytes_written.saturating_add(buffer.len() as u64);
|
||||
Ok(buffer.len())
|
||||
}
|
||||
|
||||
fn flush(&mut self) -> io::Result<()> {
|
||||
self.file.flush()
|
||||
}
|
||||
}
|
||||
|
||||
fn next_segment(segment: usize) -> io::Result<usize> {
|
||||
segment
|
||||
.checked_add(1)
|
||||
.ok_or_else(|| io::Error::other("log segment sequence exhausted"))
|
||||
}
|
||||
|
||||
fn open_log_segment(path: &Path) -> io::Result<File> {
|
||||
OpenOptions::new().create(true).append(true).open(path)
|
||||
}
|
||||
|
||||
fn log_file_name(prefix: &str, date: NaiveDate, segment: usize) -> String {
|
||||
if segment == 0 {
|
||||
format!("{prefix}.{date}.log")
|
||||
} else {
|
||||
format!("{prefix}.{date}.{segment}.log")
|
||||
}
|
||||
}
|
||||
|
||||
struct ManagedLogFile {
|
||||
date: NaiveDate,
|
||||
segment: usize,
|
||||
compressed: bool,
|
||||
path: PathBuf,
|
||||
}
|
||||
|
||||
fn managed_log_files(directory: &Path, prefix: &str) -> io::Result<Vec<ManagedLogFile>> {
|
||||
let mut files = Vec::new();
|
||||
for entry in fs::read_dir(directory)? {
|
||||
let entry = entry?;
|
||||
let name = entry.file_name();
|
||||
let name = name.to_string_lossy();
|
||||
let Some(body) = name.strip_prefix(&format!("{prefix}.")) else {
|
||||
continue;
|
||||
};
|
||||
let (body, compressed) = match body.strip_suffix(".log.gz") {
|
||||
Some(body) => (body, true),
|
||||
None => match body.strip_suffix(".log") {
|
||||
Some(body) => (body, false),
|
||||
None => continue,
|
||||
},
|
||||
};
|
||||
let (date, segment) = body
|
||||
.split_once('.')
|
||||
.map_or((body, Some(0)), |(date, segment)| {
|
||||
(date, segment.parse::<usize>().ok())
|
||||
});
|
||||
if let (Ok(date), Some(segment)) = (NaiveDate::parse_from_str(date, "%Y-%m-%d"), segment) {
|
||||
files.push(ManagedLogFile {
|
||||
date,
|
||||
segment,
|
||||
compressed,
|
||||
path: entry.path(),
|
||||
});
|
||||
}
|
||||
}
|
||||
Ok(files)
|
||||
}
|
||||
|
||||
fn cleanup_log_files(
|
||||
directory: &Path,
|
||||
prefix: &str,
|
||||
today: NaiveDate,
|
||||
retention_days: usize,
|
||||
) -> io::Result<()> {
|
||||
let days =
|
||||
u64::try_from(retention_days).map_err(|_| io::Error::other("log retention overflow"))?;
|
||||
let cutoff = today
|
||||
.checked_sub_days(chrono::Days::new(days))
|
||||
.unwrap_or(NaiveDate::MIN);
|
||||
let mut dates = BTreeMap::<NaiveDate, Vec<PathBuf>>::new();
|
||||
for entry in managed_log_files(directory, prefix)? {
|
||||
dates.entry(entry.date).or_default().push(entry.path);
|
||||
}
|
||||
for (date, paths) in dates {
|
||||
if date >= cutoff {
|
||||
continue;
|
||||
}
|
||||
// Keep a whole date if any segment was written more recently than its filename.
|
||||
// This also protects archives recovered after clock corrections or manual restoration.
|
||||
let mut latest = date;
|
||||
for path in &paths {
|
||||
let modified: chrono::DateTime<Utc> = path.metadata()?.modified()?.into();
|
||||
latest = latest.max(modified.date_naive());
|
||||
}
|
||||
if latest >= cutoff {
|
||||
continue;
|
||||
}
|
||||
for path in paths {
|
||||
fs::remove_file(path)?;
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn compress_log_file(path: &Path) -> io::Result<()> {
|
||||
let mut source = File::open(path)?;
|
||||
let modified = source.metadata()?.modified()?;
|
||||
let archive = path.with_extension("log.gz");
|
||||
let temporary = path.with_extension("log.gz.tmp");
|
||||
// Only a closed, owner-managed segment can have this temporary archive.
|
||||
let output = File::create(&temporary)?;
|
||||
let mut encoder = GzEncoder::new(output, Compression::fast());
|
||||
io::copy(&mut source, &mut encoder)?;
|
||||
let output = encoder.finish()?;
|
||||
output.set_modified(modified)?;
|
||||
output.sync_all()?;
|
||||
fs::rename(&temporary, &archive)?;
|
||||
// The original survives until a complete, synced archive has been published.
|
||||
fs::remove_file(path)
|
||||
}
|
||||
@@ -80,6 +80,51 @@ fn host_config_rejects_zero_drain_window() {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn logging_rejects_zero_retention_windows() {
|
||||
for request_dump in [false, true] {
|
||||
let mut config = valid_config();
|
||||
if request_dump {
|
||||
config.logging.request_dump_retention_days = 0;
|
||||
} else {
|
||||
config.logging.file.retention_days = 0;
|
||||
}
|
||||
assert!(
|
||||
config
|
||||
.resolve_and_validate(std::path::Path::new("/srv/gateway"))
|
||||
.is_err()
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn logging_defaults_to_seven_days_and_one_day_for_payloads_and_rejects_count_retention() {
|
||||
let mut value = serde_json::json!({
|
||||
"level": "info", "stdout": true,
|
||||
"file": { "enabled": true, "directory": "logs", "max_file_size_mb": 20 }
|
||||
});
|
||||
let config: LoggingConfig = serde_json::from_value(value.clone()).unwrap();
|
||||
assert_eq!(config.request_dump_retention_days, 1);
|
||||
assert_eq!(config.file.retention_days, 7);
|
||||
assert!(!config.oauth_recovery);
|
||||
value["file"]["max_files"] = serde_json::json!(20);
|
||||
assert!(serde_json::from_value::<LoggingConfig>(value).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn dedicated_file_channels_can_be_enabled_without_application_sinks() {
|
||||
for oauth_recovery in [false, true] {
|
||||
let mut config = valid_config();
|
||||
config.logging.stdout = false;
|
||||
config.logging.file.enabled = false;
|
||||
config.logging.oauth_recovery = oauth_recovery;
|
||||
config.logging.request_dump = !oauth_recovery;
|
||||
config
|
||||
.resolve_and_validate(std::path::Path::new("/srv/gateway"))
|
||||
.unwrap();
|
||||
}
|
||||
}
|
||||
|
||||
fn valid_config() -> HostConfig {
|
||||
let system_update = SystemUpdateConfig {
|
||||
update_state_file: PathBuf::from("update-state.json"),
|
||||
@@ -101,9 +146,10 @@ fn valid_config() -> HostConfig {
|
||||
directory: PathBuf::from(".runtime/logs"),
|
||||
retention_days: 7,
|
||||
max_file_size_mb: 100,
|
||||
max_files: 30,
|
||||
},
|
||||
oauth_recovery: false,
|
||||
request_dump: false,
|
||||
request_dump_retention_days: 1,
|
||||
},
|
||||
system_update,
|
||||
drain_timeout_seconds: 30,
|
||||
|
||||
+139
-39
@@ -1,16 +1,20 @@
|
||||
use std::env;
|
||||
use std::fs;
|
||||
use std::io::Read;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::process::Command;
|
||||
|
||||
use gateway_host::config::{FileLoggingConfig, HostConfig, ListenConfig, LoggingConfig};
|
||||
use gateway_host::system_update::SystemUpdateConfig;
|
||||
|
||||
mod sink;
|
||||
mod writer;
|
||||
|
||||
const LOG_DIRECTORY_ENV: &str = "CPR_LOGGING_TEST_DIRECTORY";
|
||||
const CHILD_PROCESS_ENV: &str = "CPR_LOGGING_TEST_CHILD";
|
||||
const REQUEST_DUMP_ENABLED_ENV: &str = "CPR_LOGGING_TEST_REQUEST_DUMP_ENABLED";
|
||||
const APPLICATION_LOG_FILE_PREFIX: &str = "codex-proxy-rs.";
|
||||
const OAUTH_RECOVERY_LOG_FILE_PREFIX: &str = "codex-proxy-rs-oath.";
|
||||
const APPLICATION_LOG_FILE_PREFIX: &str = "codex-proxy-rs-application.";
|
||||
const OAUTH_RECOVERY_LOG_FILE_PREFIX: &str = "codex-proxy-rs-oauth-recovery.";
|
||||
const REQUEST_DUMP_LOG_FILE_PREFIX: &str = "codex-proxy-rs-request-dump.";
|
||||
const APPLICATION_LOG_TARGET: &str = "logging_test_application";
|
||||
const APPLICATION_LOG_MARKER: &str = "application-file-filter-test";
|
||||
@@ -37,9 +41,10 @@ fn logging_requires_at_least_one_sink() {
|
||||
directory: PathBuf::from("logs"),
|
||||
retention_days: 7,
|
||||
max_file_size_mb: 100,
|
||||
max_files: 30,
|
||||
},
|
||||
oauth_recovery: false,
|
||||
request_dump: false,
|
||||
request_dump_retention_days: 1,
|
||||
},
|
||||
system_update: SystemUpdateConfig::default(),
|
||||
drain_timeout_seconds: 30,
|
||||
@@ -143,8 +148,115 @@ fn sensitive_file_logging_is_separate_and_overrides_global_log_level() {
|
||||
assert!(!application_log.contains(REQUEST_DUMP_SECRET));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn oauth_recovery_switch_controls_only_its_dedicated_file() {
|
||||
const OAUTH_ENABLED_ENV: &str = "CPR_LOGGING_TEST_OAUTH_ENABLED";
|
||||
const APPLICATION_ENABLED_ENV: &str = "CPR_LOGGING_TEST_APPLICATION_ENABLED";
|
||||
const TOKEN: &str = "synthetic-oauth-token-must-stay-in-recovery-file";
|
||||
if env::var_os(CHILD_PROCESS_ENV).is_some() {
|
||||
let mut config =
|
||||
logging_config(PathBuf::from(env::var_os(LOG_DIRECTORY_ENV).unwrap()), true);
|
||||
config.logging.oauth_recovery = env::var(OAUTH_ENABLED_ENV).unwrap() == "true";
|
||||
config.logging.file.enabled = env::var(APPLICATION_ENABLED_ENV).unwrap() == "true";
|
||||
config.logging.stdout = true;
|
||||
with_logging(config, || {
|
||||
tracing::info!(target: APPLICATION_LOG_TARGET, marker = APPLICATION_LOG_MARKER);
|
||||
tracing::info!(target: OAUTH_RECOVERY_LOG_TARGET, access_token = TOKEN);
|
||||
tracing::info!(target: REQUEST_DUMP_LOG_TARGET, marker = REQUEST_DUMP_LOG_MARKER);
|
||||
});
|
||||
return;
|
||||
}
|
||||
for application_enabled in [false, true] {
|
||||
for oauth_enabled in [false, true] {
|
||||
let directory = tempfile::tempdir().unwrap();
|
||||
let output = Command::new(env::current_exe().unwrap())
|
||||
.args([
|
||||
"--exact",
|
||||
"logging::oauth_recovery_switch_controls_only_its_dedicated_file",
|
||||
"--nocapture",
|
||||
])
|
||||
.env(CHILD_PROCESS_ENV, "1")
|
||||
.env(LOG_DIRECTORY_ENV, directory.path())
|
||||
.env(OAUTH_ENABLED_ENV, oauth_enabled.to_string())
|
||||
.env(APPLICATION_ENABLED_ENV, application_enabled.to_string())
|
||||
.env("RUST_LOG", "info,oauth_recovery=trace,request_dump=trace")
|
||||
.output()
|
||||
.unwrap();
|
||||
assert!(
|
||||
output.status.success(),
|
||||
"{}",
|
||||
String::from_utf8_lossy(&output.stderr)
|
||||
);
|
||||
assert!(!String::from_utf8_lossy(&output.stdout).contains(TOKEN));
|
||||
assert!(!String::from_utf8_lossy(&output.stderr).contains(TOKEN));
|
||||
assert_eq!(
|
||||
log_file_set_exists(directory.path(), OAUTH_RECOVERY_LOG_FILE_PREFIX),
|
||||
oauth_enabled
|
||||
);
|
||||
assert_eq!(
|
||||
log_file_set_exists(directory.path(), APPLICATION_LOG_FILE_PREFIX),
|
||||
application_enabled
|
||||
);
|
||||
if oauth_enabled {
|
||||
assert!(
|
||||
read_log_file_set(directory.path(), OAUTH_RECOVERY_LOG_FILE_PREFIX)
|
||||
.contains(TOKEN)
|
||||
);
|
||||
}
|
||||
if application_enabled {
|
||||
let body = read_log_file_set(directory.path(), APPLICATION_LOG_FILE_PREFIX);
|
||||
assert!(body.contains(APPLICATION_LOG_MARKER));
|
||||
assert!(!body.contains(TOKEN));
|
||||
} else {
|
||||
assert!(String::from_utf8_lossy(&output.stdout).contains(APPLICATION_LOG_MARKER));
|
||||
}
|
||||
let dump = read_log_file_set(directory.path(), REQUEST_DUMP_LOG_FILE_PREFIX);
|
||||
assert!(dump.contains(REQUEST_DUMP_LOG_MARKER));
|
||||
assert!(!dump.contains(TOKEN));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn write_sensitive_logs(directory: PathBuf, request_dump: bool) {
|
||||
let config = HostConfig {
|
||||
with_file_logging(directory, request_dump, || {
|
||||
tracing::info!(
|
||||
target: APPLICATION_LOG_TARGET,
|
||||
marker = APPLICATION_LOG_MARKER,
|
||||
"application file test record"
|
||||
);
|
||||
tracing::info!(
|
||||
target: OAUTH_RECOVERY_LOG_TARGET,
|
||||
provider = OAUTH_RECOVERY_PROVIDER,
|
||||
marker = OAUTH_RECOVERY_LOG_MARKER,
|
||||
"OAuth recovery test record"
|
||||
);
|
||||
tracing::info!(
|
||||
target: REQUEST_DUMP_LOG_TARGET,
|
||||
marker = REQUEST_DUMP_LOG_MARKER,
|
||||
authorization = REQUEST_DUMP_SECRET,
|
||||
"request dump test record"
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
fn with_file_logging(directory: PathBuf, request_dump: bool, write: impl FnOnce()) {
|
||||
with_logging(logging_config(directory, request_dump), write);
|
||||
}
|
||||
|
||||
fn with_logging(config: HostConfig, write: impl FnOnce()) {
|
||||
let runtime = tokio::runtime::Builder::new_current_thread()
|
||||
.enable_all()
|
||||
.build()
|
||||
.expect("create logging runtime");
|
||||
let bundle = runtime
|
||||
.block_on(gateway_host::initialize(config))
|
||||
.expect("initialize logging");
|
||||
write();
|
||||
drop(bundle);
|
||||
}
|
||||
|
||||
fn logging_config(directory: PathBuf, request_dump: bool) -> HostConfig {
|
||||
HostConfig {
|
||||
listen: ListenConfig {
|
||||
host: "127.0.0.1".to_owned(),
|
||||
port: 8080,
|
||||
@@ -156,42 +268,17 @@ fn write_sensitive_logs(directory: PathBuf, request_dump: bool) {
|
||||
file: FileLoggingConfig {
|
||||
enabled: true,
|
||||
directory,
|
||||
retention_days: 1,
|
||||
retention_days: 7,
|
||||
max_file_size_mb: 1,
|
||||
max_files: 1,
|
||||
},
|
||||
oauth_recovery: true,
|
||||
request_dump,
|
||||
request_dump_retention_days: 1,
|
||||
},
|
||||
system_update: SystemUpdateConfig::default(),
|
||||
drain_timeout_seconds: 30,
|
||||
worker_shutdown_timeout_seconds: 30,
|
||||
};
|
||||
let runtime = tokio::runtime::Builder::new_current_thread()
|
||||
.enable_all()
|
||||
.build()
|
||||
.expect("create logging runtime");
|
||||
let bundle = runtime
|
||||
.block_on(gateway_host::initialize(config))
|
||||
.expect("initialize logging");
|
||||
|
||||
tracing::info!(
|
||||
target: APPLICATION_LOG_TARGET,
|
||||
marker = APPLICATION_LOG_MARKER,
|
||||
"application file test record"
|
||||
);
|
||||
tracing::info!(
|
||||
target: OAUTH_RECOVERY_LOG_TARGET,
|
||||
provider = OAUTH_RECOVERY_PROVIDER,
|
||||
marker = OAUTH_RECOVERY_LOG_MARKER,
|
||||
"OAuth recovery test record"
|
||||
);
|
||||
tracing::info!(
|
||||
target: REQUEST_DUMP_LOG_TARGET,
|
||||
marker = REQUEST_DUMP_LOG_MARKER,
|
||||
authorization = REQUEST_DUMP_SECRET,
|
||||
"request dump test record"
|
||||
);
|
||||
drop(bundle);
|
||||
}
|
||||
}
|
||||
|
||||
fn read_log_file_set(directory: &Path, file_prefix: &str) -> String {
|
||||
@@ -201,7 +288,10 @@ fn read_log_file_set(directory: &Path, file_prefix: &str) -> String {
|
||||
.filter(|path| {
|
||||
path.file_name()
|
||||
.and_then(|name| name.to_str())
|
||||
.is_some_and(|name| name.starts_with(file_prefix))
|
||||
.is_some_and(|name| {
|
||||
name.starts_with(file_prefix)
|
||||
&& (name.ends_with(".log") || name.ends_with(".log.gz"))
|
||||
})
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
files.sort();
|
||||
@@ -209,7 +299,17 @@ fn read_log_file_set(directory: &Path, file_prefix: &str) -> String {
|
||||
assert!(!files.is_empty(), "expected log file set {file_prefix}");
|
||||
files
|
||||
.into_iter()
|
||||
.map(|path| fs::read_to_string(path).expect("read log file"))
|
||||
.map(|path| {
|
||||
if path.extension().is_some_and(|extension| extension == "gz") {
|
||||
let mut body = String::new();
|
||||
flate2::read::GzDecoder::new(fs::File::open(path).expect("open archive"))
|
||||
.read_to_string(&mut body)
|
||||
.expect("read complete archive");
|
||||
body
|
||||
} else {
|
||||
fs::read_to_string(path).expect("read log file")
|
||||
}
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
@@ -218,10 +318,10 @@ fn log_file_set_exists(directory: &Path, file_prefix: &str) -> bool {
|
||||
.expect("read log directory")
|
||||
.filter_map(Result::ok)
|
||||
.any(|entry| {
|
||||
entry
|
||||
.file_name()
|
||||
.to_str()
|
||||
.is_some_and(|name| name.starts_with(file_prefix))
|
||||
entry.file_name().to_str().is_some_and(|name| {
|
||||
name.starts_with(file_prefix)
|
||||
&& (name.ends_with(".log") || name.ends_with(".log.gz"))
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
@@ -0,0 +1,108 @@
|
||||
use super::*;
|
||||
use gateway_core::health::HealthState;
|
||||
use std::time::Duration;
|
||||
|
||||
#[test]
|
||||
fn file_queue_drains_every_record_before_normal_shutdown() {
|
||||
if env::var_os(CHILD_PROCESS_ENV).is_some() {
|
||||
with_file_logging(
|
||||
PathBuf::from(env::var_os(LOG_DIRECTORY_ENV).unwrap()),
|
||||
true,
|
||||
|| {
|
||||
let payload = "x".repeat(16 * 1024);
|
||||
for sequence in 0..2000 {
|
||||
tracing::info!(target: REQUEST_DUMP_LOG_TARGET, sequence, payload, "queue record");
|
||||
}
|
||||
},
|
||||
);
|
||||
return;
|
||||
}
|
||||
let directory = tempfile::tempdir().unwrap();
|
||||
let output = Command::new(env::current_exe().unwrap())
|
||||
.args([
|
||||
"--exact",
|
||||
"logging::sink::file_queue_drains_every_record_before_normal_shutdown",
|
||||
])
|
||||
.env(CHILD_PROCESS_ENV, "1")
|
||||
.env(LOG_DIRECTORY_ENV, directory.path())
|
||||
.env("RUST_LOG", "off")
|
||||
.output()
|
||||
.unwrap();
|
||||
assert!(
|
||||
output.status.success(),
|
||||
"{}",
|
||||
String::from_utf8_lossy(&output.stderr)
|
||||
);
|
||||
let body = read_log_file_set(directory.path(), REQUEST_DUMP_LOG_FILE_PREFIX);
|
||||
let mut sequences = body
|
||||
.lines()
|
||||
.map(|line| {
|
||||
serde_json::from_str::<serde_json::Value>(line).unwrap()["fields"]["sequence"]
|
||||
.as_u64()
|
||||
.unwrap()
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
sequences.sort_unstable();
|
||||
assert_eq!(sequences, (0..2000).collect::<Vec<_>>());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn write_failure_marks_log_completeness_unhealthy_even_after_writes_recover() {
|
||||
if env::var_os(CHILD_PROCESS_ENV).is_some() {
|
||||
let directory = PathBuf::from(env::var_os(LOG_DIRECTORY_ENV).unwrap());
|
||||
let runtime = tokio::runtime::Builder::new_current_thread()
|
||||
.enable_all()
|
||||
.build()
|
||||
.unwrap();
|
||||
let bundle = runtime
|
||||
.block_on(gateway_host::initialize(logging_config(
|
||||
directory.clone(),
|
||||
true,
|
||||
)))
|
||||
.unwrap();
|
||||
let health = bundle.logging_health_probe();
|
||||
assert_eq!(runtime.block_on(health.check()), HealthState::Healthy);
|
||||
let blocked = directory.join(format!(
|
||||
"{REQUEST_DUMP_LOG_FILE_PREFIX}{}.1.log",
|
||||
chrono::Utc::now().date_naive()
|
||||
));
|
||||
fs::create_dir(&blocked).unwrap();
|
||||
let payload = "x".repeat(1024 * 1024);
|
||||
tracing::info!(target: REQUEST_DUMP_LOG_TARGET, payload, marker="before-failure");
|
||||
tracing::info!(target: REQUEST_DUMP_LOG_TARGET, payload, marker="cannot-write");
|
||||
runtime.block_on(async {
|
||||
tokio::time::timeout(Duration::from_secs(5), async {
|
||||
loop {
|
||||
if matches!(health.check().await, HealthState::Unhealthy(_)) {
|
||||
break;
|
||||
}
|
||||
tokio::time::sleep(Duration::from_millis(10)).await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
});
|
||||
fs::remove_dir(&blocked).unwrap();
|
||||
tracing::info!(target: REQUEST_DUMP_LOG_TARGET, marker="after-recovery");
|
||||
drop(bundle);
|
||||
assert!(matches!(
|
||||
runtime.block_on(health.check()),
|
||||
HealthState::Unhealthy(_)
|
||||
));
|
||||
return;
|
||||
}
|
||||
let directory = tempfile::tempdir().unwrap();
|
||||
let output = Command::new(env::current_exe().unwrap())
|
||||
.args(["--nocapture", "--exact", "logging::sink::write_failure_marks_log_completeness_unhealthy_even_after_writes_recover"])
|
||||
.env(CHILD_PROCESS_ENV, "1").env(LOG_DIRECTORY_ENV, directory.path())
|
||||
.env("RUST_LOG", "off").output().unwrap();
|
||||
assert!(
|
||||
output.status.success(),
|
||||
"{}",
|
||||
String::from_utf8_lossy(&output.stderr)
|
||||
);
|
||||
assert!(String::from_utf8_lossy(&output.stderr).contains("log completeness compromised"));
|
||||
let body = read_log_file_set(directory.path(), REQUEST_DUMP_LOG_FILE_PREFIX);
|
||||
assert!(body.contains("before-failure") && body.contains("after-recovery"));
|
||||
assert!(!body.contains("cannot-write"));
|
||||
}
|
||||
@@ -0,0 +1,118 @@
|
||||
use super::*;
|
||||
use std::time::SystemTime;
|
||||
|
||||
#[test]
|
||||
fn retention_preserves_complete_dates_across_compression_rotation_and_restart() {
|
||||
if env::var_os(CHILD_PROCESS_ENV).is_some() {
|
||||
with_file_logging(
|
||||
PathBuf::from(env::var_os(LOG_DIRECTORY_ENV).unwrap()),
|
||||
true,
|
||||
|| {
|
||||
let payload = "x".repeat(1024 * 1024);
|
||||
for sequence in 0..25 {
|
||||
tracing::info!(target: REQUEST_DUMP_LOG_TARGET, sequence, payload, "retention record");
|
||||
tracing::info!(target: APPLICATION_LOG_TARGET, sequence, payload, "retention record");
|
||||
tracing::info!(target: OAUTH_RECOVERY_LOG_TARGET, sequence, payload, "retention record");
|
||||
}
|
||||
},
|
||||
);
|
||||
return;
|
||||
}
|
||||
let directory = tempfile::tempdir().unwrap();
|
||||
let today = chrono::Utc::now().date_naive();
|
||||
let mut retained = Vec::new();
|
||||
let mut expired = Vec::new();
|
||||
for (prefix, days) in [
|
||||
(APPLICATION_LOG_FILE_PREFIX, 7),
|
||||
(OAUTH_RECOVERY_LOG_FILE_PREFIX, 7),
|
||||
(REQUEST_DUMP_LOG_FILE_PREFIX, 1),
|
||||
] {
|
||||
let boundary = today - chrono::Days::new(days);
|
||||
for segment in 1..=25 {
|
||||
let path = directory
|
||||
.path()
|
||||
.join(format!("{prefix}{boundary}.{segment}.log"));
|
||||
seed_log(&path, boundary, &format!("boundary-{segment}\n"));
|
||||
retained.push(path);
|
||||
}
|
||||
let old_date = boundary - chrono::Days::new(1);
|
||||
let path = directory.path().join(format!("{prefix}{old_date}.log"));
|
||||
seed_log(&path, old_date, "expired date\n");
|
||||
expired.push(path);
|
||||
// If any old segment was recently written, protect its whole UTC date.
|
||||
let restored_date = old_date - chrono::Days::new(1);
|
||||
for segment in 0..=1 {
|
||||
let path = directory
|
||||
.path()
|
||||
.join(format!("{prefix}{restored_date}.{segment}.log"));
|
||||
seed_log(
|
||||
&path,
|
||||
if segment == 0 { today } else { restored_date },
|
||||
"restored date\n",
|
||||
);
|
||||
retained.push(path);
|
||||
}
|
||||
}
|
||||
let unrelated = directory.path().join("unmanaged.log");
|
||||
fs::write(&unrelated, "unmanaged\n").unwrap();
|
||||
// A crash before archive publication leaves the original and possibly a partial temporary file.
|
||||
fs::write(
|
||||
retained[0].with_extension("log.gz.tmp"),
|
||||
"unfinished archive",
|
||||
)
|
||||
.unwrap();
|
||||
for run in 1..=2 {
|
||||
let output = Command::new(env::current_exe().unwrap())
|
||||
.args(["--exact", "logging::writer::retention_preserves_complete_dates_across_compression_rotation_and_restart"])
|
||||
.env(CHILD_PROCESS_ENV, "1").env(LOG_DIRECTORY_ENV, directory.path())
|
||||
.env("RUST_LOG", "off,logging_test_application=info").output().unwrap();
|
||||
assert!(
|
||||
output.status.success(),
|
||||
"{}",
|
||||
String::from_utf8_lossy(&output.stderr)
|
||||
);
|
||||
assert!(
|
||||
retained
|
||||
.iter()
|
||||
.all(|path| path.with_extension("log.gz").exists())
|
||||
);
|
||||
assert!(
|
||||
expired
|
||||
.iter()
|
||||
.all(|path| !path.exists() && !path.with_extension("log.gz").exists())
|
||||
);
|
||||
assert!(unrelated.exists());
|
||||
|
||||
for prefix in [
|
||||
APPLICATION_LOG_FILE_PREFIX,
|
||||
OAUTH_RECOVERY_LOG_FILE_PREFIX,
|
||||
REQUEST_DUMP_LOG_FILE_PREFIX,
|
||||
] {
|
||||
let body = read_log_file_set(directory.path(), prefix);
|
||||
for sequence in 0..25 {
|
||||
assert_eq!(
|
||||
body.matches(&format!(r#""sequence":{sequence},"#)).count(),
|
||||
run
|
||||
);
|
||||
}
|
||||
for segment in 1..=25 {
|
||||
assert!(
|
||||
body.contains(&format!("boundary-{segment}\n")),
|
||||
"whole boundary date must survive"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn seed_log(path: &Path, date: chrono::NaiveDate, body: &str) {
|
||||
fs::write(path, body).unwrap();
|
||||
let modified = chrono::DateTime::<chrono::Utc>::from_naive_utc_and_offset(
|
||||
date.and_hms_opt(0, 0, 0).unwrap(),
|
||||
chrono::Utc,
|
||||
);
|
||||
fs::File::open(path)
|
||||
.unwrap()
|
||||
.set_modified(SystemTime::from(modified))
|
||||
.unwrap();
|
||||
}
|
||||
@@ -65,7 +65,8 @@ pub fn extract_usage(body: &Value) -> Option<TokenUsage> {
|
||||
let image_output_tokens =
|
||||
nested_number_field(body, &["tool_usage", "image_gen", "output_tokens"])
|
||||
.unwrap_or_default();
|
||||
let total_tokens = number_field(usage, "total_tokens").unwrap_or(input_tokens + output_tokens);
|
||||
let total_tokens = number_field(usage, "total_tokens")
|
||||
.unwrap_or_else(|| input_tokens.saturating_add(output_tokens));
|
||||
|
||||
let has_usage = [
|
||||
"input_tokens",
|
||||
@@ -98,7 +99,7 @@ pub fn extract_usage(body: &Value) -> Option<TokenUsage> {
|
||||
})
|
||||
}
|
||||
|
||||
/// 校验计价所需的输入、输出和缓存读取字段,拒绝缺失、非法或不一致的数值。
|
||||
/// 校验计价所需的 token 字段,拒绝缺失、非法、溢出或不一致的数值。
|
||||
///
|
||||
/// 用量提取允许缺失字段补零;计价必须额外确认这些零值有明确的上游事实。
|
||||
#[must_use]
|
||||
@@ -122,11 +123,28 @@ pub fn billable_usage_is_complete(response: &Value, usage: TokenUsage) -> bool {
|
||||
Some(value) => value.as_u64(),
|
||||
None => Some(0),
|
||||
};
|
||||
let written = raw
|
||||
.pointer("/input_tokens_details/cache_write_tokens")
|
||||
.or_else(|| raw.pointer("/prompt_tokens_details/cache_write_tokens"))
|
||||
.or_else(|| raw.get("cache_write_tokens"));
|
||||
let written = match written {
|
||||
Some(value) => value.as_u64(),
|
||||
None => Some(0),
|
||||
};
|
||||
let total = usage.input_tokens.checked_add(usage.output_tokens);
|
||||
|
||||
input == Some(usage.input_tokens)
|
||||
&& output == Some(usage.output_tokens)
|
||||
&& cached == Some(usage.cached_tokens)
|
||||
&& usage.cached_tokens <= usage.input_tokens
|
||||
&& written == Some(usage.cache_write_tokens)
|
||||
&& usage
|
||||
.cached_tokens
|
||||
.checked_add(usage.cache_write_tokens)
|
||||
.is_some_and(|cached| cached <= usage.input_tokens)
|
||||
&& total == Some(usage.total_tokens)
|
||||
&& raw
|
||||
.get("total_tokens")
|
||||
.is_none_or(|value| value.as_u64() == total)
|
||||
}
|
||||
|
||||
/// 从完整 SSE 文本中提取最终可见用量。
|
||||
|
||||
@@ -5,6 +5,57 @@ use gateway_protocol::openai::events::{
|
||||
};
|
||||
use serde_json::json;
|
||||
|
||||
#[test]
|
||||
fn billable_usage_should_validate_cache_writes_totals_and_overflow() {
|
||||
for (raw, expected) in [
|
||||
(
|
||||
json!({"input_tokens": 100, "output_tokens": 10, "input_tokens_details": {"cached_tokens": 20, "cache_write_tokens": 80}, "total_tokens": 110}),
|
||||
true,
|
||||
),
|
||||
(
|
||||
json!({"prompt_tokens": 100, "completion_tokens": 10, "prompt_tokens_details": {"cached_tokens": 20, "cache_write_tokens": 80}}),
|
||||
true,
|
||||
),
|
||||
(
|
||||
json!({"input_tokens": 100, "output_tokens": 10, "cached_tokens": 20, "cache_write_tokens": 90}),
|
||||
false,
|
||||
),
|
||||
(
|
||||
json!({"input_tokens": 100, "output_tokens": 10, "cache_write_tokens": -1}),
|
||||
false,
|
||||
),
|
||||
(
|
||||
json!({"input_tokens": 100, "output_tokens": 10, "input_tokens_details": {"cache_write_tokens": "1"}, "cache_write_tokens": 1}),
|
||||
false,
|
||||
),
|
||||
(
|
||||
json!({"input_tokens": 100, "output_tokens": 10, "cache_write_tokens": 1.5}),
|
||||
false,
|
||||
),
|
||||
(
|
||||
json!({"input_tokens": 100, "output_tokens": 10, "total_tokens": 111}),
|
||||
false,
|
||||
),
|
||||
(
|
||||
json!({"input_tokens": 100, "output_tokens": 10, "total_tokens": "110"}),
|
||||
false,
|
||||
),
|
||||
(json!({"input_tokens": u64::MAX, "output_tokens": 1}), false),
|
||||
(
|
||||
json!({"input_tokens": u64::MAX, "output_tokens": 0, "cached_tokens": u64::MAX, "cache_write_tokens": 1}),
|
||||
false,
|
||||
),
|
||||
] {
|
||||
let response = json!({"usage": raw});
|
||||
let usage = extract_usage(&response).expect("提取用量不得因溢出而崩溃");
|
||||
assert_eq!(
|
||||
billable_usage_is_complete(&response, usage),
|
||||
expected,
|
||||
"{response}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn billable_usage_should_require_valid_input_output_and_cache_facts() {
|
||||
for (raw, expected) in [
|
||||
|
||||
@@ -13,12 +13,12 @@ const REQUEST_ERROR_SELECT: &str = "select 'model_request'::text as source,
|
||||
mr.provider_account_authentication_kind_snapshot
|
||||
as provider_account_authentication_kind,
|
||||
mr.upstream_model_id, mr.upstream_transport,
|
||||
coalesce(mr.error_kind, 'failed') as failure_kind,
|
||||
mr.error_kind as failure_kind,
|
||||
mr.upstream_send_state,
|
||||
mr.client_status_code, mr.upstream_status_code,
|
||||
mr.provider_error_code, mr.client_response_id, mr.upstream_request_id,
|
||||
mr.latency_ms,
|
||||
coalesce(mr.error_message, mr.error_kind, 'request failed') as message,
|
||||
coalesce(mr.error_message, mr.error_kind) as message,
|
||||
mr.raw_upstream_error,
|
||||
host(mr.client_ip) as client_ip, mr.user_agent,
|
||||
mr.reasoning_effort, mr.reasoning_preset, mr.request_kind,
|
||||
@@ -32,7 +32,7 @@ const REQUEST_ERROR_SELECT: &str = "select 'model_request'::text as source,
|
||||
mr.completed_at as occurred_at,
|
||||
'model_request:' || mr.id as stable_sort_id
|
||||
from model_requests mr
|
||||
where mr.outcome = 'failed'";
|
||||
where true";
|
||||
|
||||
const OPS_EVENT_SELECT: &str = "select 'ops_event'::text as source,
|
||||
oe.id as event_id, oe.model_request_id as request_id, oe.attempt_index,
|
||||
@@ -106,7 +106,7 @@ pub(crate) async fn count_ops_errors(
|
||||
filter: &OpsErrorFilter,
|
||||
) -> StoreResult<u64> {
|
||||
let mut statement = QueryBuilder::<Postgres>::new(
|
||||
"select coalesce(sum(source_count), 0)::bigint from (select count(*)::bigint as source_count from model_requests mr where mr.outcome = 'failed'",
|
||||
"select coalesce(sum(source_count), 0)::bigint from (select count(*)::bigint as source_count from model_requests mr where true",
|
||||
);
|
||||
push_request_error_predicates(&mut statement, range, filter);
|
||||
statement.push(
|
||||
@@ -127,6 +127,9 @@ fn push_request_error_predicates(
|
||||
range: ObservabilityRange,
|
||||
filter: &OpsErrorFilter,
|
||||
) {
|
||||
// 错误事实独立于请求结束状态;主动取消不属于需要排查的错误。
|
||||
// 列表和总数共用此条件,避免流式响应中的错误因 outcome 被漏掉。
|
||||
statement.push(" and mr.error_kind is not null and mr.error_kind <> 'cancelled'");
|
||||
push_range(statement, "mr.completed_at", range);
|
||||
for (column, value) in [
|
||||
("mr.client_api_key_ref", &filter.client_api_key_ref),
|
||||
|
||||
@@ -1111,21 +1111,36 @@ async fn diagnostic_trace_is_finalized_atomically_and_available_for_failed_reque
|
||||
let mut finalization = successful_core_finalization("req_diagnostic_failed");
|
||||
finalization.outcome = ExecutionOutcome::Failed;
|
||||
finalization.client_status_code = Some(502);
|
||||
finalization.error = Some(GatewayError::new(
|
||||
GatewayErrorKind::UpstreamUnavailable,
|
||||
"upstream closed",
|
||||
));
|
||||
let diagnostic = gateway_core::error::ProviderDiagnostic::new(
|
||||
"OpenAI WebSocket receive idle timeout after 300s",
|
||||
)
|
||||
.with_classification("receive", "receive_idle_timeout");
|
||||
let provider_error = gateway_core::error::ProviderError::new(
|
||||
gateway_core::error::ProviderErrorKind::Timeout,
|
||||
UpstreamSendState::Ambiguous,
|
||||
)
|
||||
.with_diagnostic(diagnostic);
|
||||
finalization.error = Some(GatewayError::from_provider(&provider_error));
|
||||
let trace = serde_json::json!({"schemaVersion": 1, "requestId": "req_diagnostic_failed", "events": [
|
||||
{"attemptIndex": 1, "stage": "upstream.close", "data": {"code": 1000}},
|
||||
{"attemptIndex": 1, "stage": "attempt.failed", "data": {"diagnostic": {
|
||||
"stage": "receive", "code": "receive_idle_timeout", "message": "OpenAI WebSocket receive idle timeout after 300s"
|
||||
}}},
|
||||
]});
|
||||
finalization.diagnostic_trace_json = Some(trace.to_string());
|
||||
ExecutionStore::finalize_model_request(&store, finalization)
|
||||
.await
|
||||
.unwrap();
|
||||
let persisted: (String, serde_json::Value) = sqlx::query_as(
|
||||
"select outcome, diagnostic_trace_json from model_requests where id = 'req_diagnostic_failed'"
|
||||
let persisted: (String, serde_json::Value, String) = sqlx::query_as(
|
||||
"select outcome, diagnostic_trace_json, error_message from model_requests where id = 'req_diagnostic_failed'"
|
||||
).fetch_one(&database.pool).await.unwrap();
|
||||
assert_eq!(persisted, ("failed".to_owned(), trace.clone()));
|
||||
assert_eq!(
|
||||
persisted,
|
||||
(
|
||||
"failed".to_owned(),
|
||||
trace.clone(),
|
||||
"OpenAI WebSocket receive idle timeout after 300s".to_owned()
|
||||
)
|
||||
);
|
||||
let repository = super::observability_repository(&database.pool);
|
||||
let detail = repository
|
||||
.usage_record_detail("req_diagnostic_failed")
|
||||
|
||||
@@ -55,10 +55,6 @@ pub(super) fn admin_account_store(pool: &PgPool) -> PgAdminAccountStore {
|
||||
|
||||
impl TestDatabase {
|
||||
pub(super) async fn create(label: &str) -> Option<Self> {
|
||||
Self::create_at(label, i64::MAX).await
|
||||
}
|
||||
|
||||
pub(super) async fn create_at(label: &str, version: i64) -> Option<Self> {
|
||||
let database_url = crate::support::test_env("CPR_TEST_DATABASE_URL")?;
|
||||
let schema = format!("cpr_store_{label}_{}", Uuid::new_v4().simple());
|
||||
let admin = PgPoolOptions::new()
|
||||
@@ -86,16 +82,10 @@ impl TestDatabase {
|
||||
.connect(&database_url)
|
||||
.await
|
||||
.expect("connect isolated test schema");
|
||||
sqlx::migrate::Migrator::with_migrations(
|
||||
TEST_MIGRATOR
|
||||
.iter()
|
||||
.filter(|migration| migration.version <= version)
|
||||
.cloned()
|
||||
.collect(),
|
||||
)
|
||||
.run(&pool)
|
||||
.await
|
||||
.expect("apply test migrations");
|
||||
TEST_MIGRATOR
|
||||
.run(&pool)
|
||||
.await
|
||||
.expect("apply test migrations");
|
||||
Some(Self {
|
||||
admin,
|
||||
pool,
|
||||
|
||||
@@ -258,6 +258,119 @@ async fn ops_search_should_treat_sql_wildcards_as_literals() {
|
||||
database.close().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn ops_should_include_incomplete_upstream_errors() {
|
||||
let Some(database) = TestDatabase::create("ops_incomplete_error").await else {
|
||||
return;
|
||||
};
|
||||
let now = Utc::now();
|
||||
seed_observability_facts(&database.pool, now).await.unwrap();
|
||||
let raw_error = r#"{"type":"error","status":400,"error":{"type":"invalid_request_error","message":"Invalid `previous_response_id`."}}"#;
|
||||
sqlx::query(
|
||||
"update model_requests
|
||||
set outcome = 'incomplete', error_kind = 'invalid_request',
|
||||
client_transport = 'websocket', upstream_transport = 'websocket',
|
||||
downstream_committed_at = completed_at,
|
||||
client_status_code = null, upstream_status_code = 400,
|
||||
provider_error_code = null, raw_upstream_error = $1
|
||||
where id = 'req_observe_failed'",
|
||||
)
|
||||
.bind(raw_error)
|
||||
.execute(&database.pool)
|
||||
.await
|
||||
.unwrap();
|
||||
let page = admin_observability_store(&database.pool)
|
||||
.list_ops_errors(admin_observability::OpsErrorQuery {
|
||||
range: admin_observability::TimeRange::new(
|
||||
now - TimeDelta::hours(1),
|
||||
now + TimeDelta::hours(1),
|
||||
)
|
||||
.unwrap(),
|
||||
filter: admin_observability::OpsErrorFilter {
|
||||
request_id: Some("req_observe_failed".to_owned()),
|
||||
status_code: Some(400),
|
||||
..admin_observability::OpsErrorFilter::default()
|
||||
},
|
||||
current_page: 1,
|
||||
page_size: PageSize::new(10).unwrap(),
|
||||
})
|
||||
.await
|
||||
.expect("incomplete request must be available in admin error troubleshooting");
|
||||
database.close().await;
|
||||
|
||||
assert_eq!(page.total, 1);
|
||||
assert_eq!(page.items.len(), 1);
|
||||
let error = &page.items[0];
|
||||
assert_eq!(error.source, "model_request");
|
||||
assert_eq!(error.failure_kind, "invalid_request");
|
||||
assert_eq!(error.upstream_status_code, Some(400));
|
||||
assert_eq!(error.client_status_code, None);
|
||||
assert_eq!(error.raw_upstream_error.as_deref(), Some(raw_error));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn ops_should_select_errors_independently_of_request_outcome() {
|
||||
let Some(database) = TestDatabase::create("ops_error_outcomes").await else {
|
||||
return;
|
||||
};
|
||||
let now = Utc::now();
|
||||
seed_observability_facts(&database.pool, now).await.unwrap();
|
||||
let range =
|
||||
ObservabilityRange::new(now - TimeDelta::hours(1), now + TimeDelta::hours(1)).unwrap();
|
||||
let repository = observability_repository(&database.pool);
|
||||
let mut results = Vec::new();
|
||||
for (outcome, error_kind, expected_count) in [
|
||||
("incomplete", Some("upstream_unavailable"), 1),
|
||||
("succeeded", Some("upstream_unavailable"), 1),
|
||||
("failed", Some("upstream_unavailable"), 1),
|
||||
("failed", None, 0),
|
||||
("incomplete", None, 0),
|
||||
("cancelled", Some("cancelled"), 0),
|
||||
("incomplete", Some("cancelled"), 0),
|
||||
("failed", Some("cancelled"), 0),
|
||||
("succeeded", None, 0),
|
||||
] {
|
||||
sqlx::query(
|
||||
"update model_requests set outcome = $1, error_kind = $2
|
||||
where id = 'req_observe_success'",
|
||||
)
|
||||
.bind(outcome)
|
||||
.bind(error_kind)
|
||||
.execute(&database.pool)
|
||||
.await
|
||||
.unwrap();
|
||||
let page = repository
|
||||
.list_ops_errors(OpsErrorQuery {
|
||||
range,
|
||||
filter: OpsErrorFilter {
|
||||
request_id: Some("req_observe_success".to_owned()),
|
||||
..OpsErrorFilter::default()
|
||||
},
|
||||
current_page: 1,
|
||||
page_size: ObservabilityPageSize::new(10).unwrap(),
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
results.push((outcome, error_kind, expected_count, page));
|
||||
}
|
||||
database.close().await;
|
||||
|
||||
for (outcome, error_kind, expected_count, page) in results {
|
||||
assert_eq!(
|
||||
page.total, expected_count,
|
||||
"total for outcome={outcome}, error_kind={error_kind:?}"
|
||||
);
|
||||
assert_eq!(
|
||||
page.items.len() as u64,
|
||||
expected_count,
|
||||
"items for outcome={outcome}, error_kind={error_kind:?}"
|
||||
);
|
||||
if let Some(error) = page.items.first() {
|
||||
assert_eq!(Some(error.failure_kind.as_str()), error_kind);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn recovered_continuation_failure_should_be_visible_in_ops_but_hidden_from_business_metrics()
|
||||
{
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
use super::{TEST_MIGRATOR, TestDatabase};
|
||||
use super::TestDatabase;
|
||||
|
||||
async fn seed_request(pool: &sqlx::PgPool) {
|
||||
sqlx::query(
|
||||
@@ -101,8 +101,8 @@ async fn account_snapshots_are_required_before_live_foreign_keys_can_be_cleared(
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn upgrade_normalizes_admin_identity_and_preserves_existing_event_rows() {
|
||||
let Some(db) = TestDatabase::create_at("audit_upgrade", 7).await else {
|
||||
async fn audit_requires_canonical_admin_identity_and_retains_it_after_admin_deletion() {
|
||||
let Some(db) = TestDatabase::create("audit_identity").await else {
|
||||
return;
|
||||
};
|
||||
sqlx::raw_sql(
|
||||
@@ -110,28 +110,12 @@ async fn upgrade_normalizes_admin_identity_and_preserves_existing_event_rows() {
|
||||
values ('admin_test', 'test_hash', now(), now());
|
||||
insert into admin_audit_events (id, actor_kind, actor_admin_user_id, actor_ref,
|
||||
action, entity_kind, entity_ref, created_at)
|
||||
values ('audit_before_upgrade', 'admin_session', 'admin_test', 'admin_test',
|
||||
'update', 'settings', '1', now());
|
||||
insert into ops_events (id, level, component, operation, failure_kind, message, created_at)
|
||||
values ('ops_before_upgrade', 'error', 'probe', 'probe', 'timeout', 'test', now());",
|
||||
values ('audit_identity', 'admin_session', 'admin_test', 'admin:admin_test',
|
||||
'update', 'settings', '1', now());",
|
||||
)
|
||||
.execute(&db.pool)
|
||||
.await
|
||||
.expect("seed data from pre-optimization schema");
|
||||
TEST_MIGRATOR
|
||||
.run(&db.pool)
|
||||
.await
|
||||
.expect("upgrade populated database");
|
||||
let identity: String = sqlx::query_scalar("select actor_ref from admin_audit_events")
|
||||
.fetch_one(&db.pool)
|
||||
.await
|
||||
.expect("read normalized identity");
|
||||
assert_eq!(identity, "admin:admin_test");
|
||||
let events: i64 = sqlx::query_scalar("select count(*) from ops_events")
|
||||
.fetch_one(&db.pool)
|
||||
.await
|
||||
.expect("read retained events");
|
||||
assert_eq!(events, 1);
|
||||
.expect("seed audit event with canonical admin identity");
|
||||
let error = sqlx::query("update admin_audit_events set actor_ref = 'admin_test'")
|
||||
.execute(&db.pool)
|
||||
.await
|
||||
|
||||
@@ -23,8 +23,8 @@ getrandom = "0.4.3"
|
||||
hex = "0.4.3"
|
||||
# TLS 依赖按官方 Core 锁文件固定,包含用于约束传递依赖的直接声明。
|
||||
# 升级前须重复采样 ClientHello,核对算法、扩展集合及排序约束、key_share 与 ALPN。
|
||||
aws-lc-rs = "=1.16.2"
|
||||
hyper = { version = "=1.8.1", default-features = false }
|
||||
aws-lc-rs = "=1.18.1"
|
||||
hyper = { version = "=1.11.1", default-features = false }
|
||||
hyper-rustls = "=0.27.7"
|
||||
# 与官方 Codex Desktop 一致:reqwest 不启用压缩特性,上游请求不带 accept-encoding。
|
||||
reqwest = { version = "=0.12.28", default-features = false, features = ["json", "stream", "cookies", "rustls-tls-native-roots", "http2"] }
|
||||
@@ -33,15 +33,15 @@ semver = "1.0.27"
|
||||
# 与官方 rustls 路径一致:aws-lc 支持 P-521,优先提供 X25519MLKEM768。
|
||||
# reqwest 的 roots feature 仍会编译 ring;实际 provider 由 TLS 初始化显式选择。
|
||||
rustls = { version = "=0.23.36", default-features = false, features = ["aws_lc_rs", "prefer-post-quantum", "std", "tls12"] }
|
||||
rustls-native-certs = "=0.8.3"
|
||||
rustls-pki-types = "=1.14.0"
|
||||
rustls-native-certs = "=0.8.4"
|
||||
rustls-pki-types = "=1.15.1"
|
||||
secrecy = "0.10.3"
|
||||
serde = { version = "1.0.228", features = ["derive"] }
|
||||
serde_json = { version = "1.0.150", features = ["arbitrary_precision", "preserve_order"] }
|
||||
sha2 = "0.11.0"
|
||||
subtle = "2.6.1"
|
||||
thiserror = "2.0.18"
|
||||
tokio = { version = "=1.52.3", default-features = false, features = ["fs", "macros", "rt", "sync", "time"] }
|
||||
tokio = { version = "=1.53.1", default-features = false, features = ["fs", "macros", "rt", "sync", "time"] }
|
||||
tokio-rustls = { version = "=0.26.4", default-features = false }
|
||||
tokio-util = { version = "0.7.18", features = ["rt"] }
|
||||
# 官方 Codex WebSocket 需要上游尚未发布的 proxy 与 permessage-deflate 实现,固定到已审计提交。
|
||||
|
||||
@@ -213,6 +213,7 @@ impl CodexWebSocketPoolSettings {
|
||||
#[derive(Debug, Clone, Deserialize, PartialEq, Eq)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct CodexQuotaSettings {
|
||||
/// 保留模型目录的刷新周期;额度独立每 30 秒检查周期复核和 reset 到期条件。
|
||||
pub refresh_interval_minutes: u64,
|
||||
}
|
||||
|
||||
|
||||
@@ -57,8 +57,8 @@ use snapshot::{
|
||||
const DEFAULT_RATE_LIMIT_COOLDOWN: Duration = Duration::from_secs(60);
|
||||
pub(crate) const QUOTA_SCHEDULING_TTL: Duration = Duration::from_secs(10 * 60);
|
||||
const QUOTA_HYDRATION_FAILURE_TTL: Duration = Duration::from_secs(5);
|
||||
const EXHAUSTED_QUOTA_FALLBACK_RECHECK_INTERVAL: Duration = Duration::from_secs(30 * 60);
|
||||
const EXHAUSTED_QUOTA_REFRESH_RETRY_INTERVAL: Duration = Duration::from_secs(30 * 60);
|
||||
const EXHAUSTED_QUOTA_RESET_GRACE: Duration = Duration::from_secs(2 * 60);
|
||||
/// 首次 OAuth 异步观察失败时,由既有 quota worker 兜底重试的单轮上限。
|
||||
const INITIAL_QUOTA_SYNC_BATCH: usize = 100;
|
||||
// 5xx 上游拒绝的短退避重试预算;指数退避 1s/2s,吞掉瞬时抖动。
|
||||
@@ -68,8 +68,8 @@ const QUOTA_FETCH_5XX_BASE_DELAY: Duration = Duration::from_secs(1);
|
||||
/// OpenAI Provider 主动额度刷新的调度策略。
|
||||
///
|
||||
/// 正常账号依赖请求响应的被动额度同步;周期 worker 仅复核已耗尽账号。
|
||||
/// 该策略只控制 Provider 自己的 quota worker 频率,避免把 OpenAI
|
||||
/// 额度语义泄漏到公共调度层。
|
||||
/// 该策略保留模型目录的周期刷新频率;额度到期检查使用独立的短周期,
|
||||
/// 避免到达 reset 后还要等待完整的目录刷新周期。
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
pub struct CodexQuotaRefreshPolicy {
|
||||
interval: Duration,
|
||||
@@ -204,7 +204,12 @@ struct CodexQuotaSchedulingProjection {
|
||||
struct CodexQuotaProjectionState {
|
||||
next_version: u64,
|
||||
entries: BTreeMap<ProviderAccountId, CodexQuotaSchedulingEntry>,
|
||||
last_periodic_refresh_at: BTreeMap<ProviderAccountId, Instant>,
|
||||
last_periodic_refresh_at: BTreeMap<ProviderAccountId, CodexQuotaRefreshAttempt>,
|
||||
}
|
||||
|
||||
struct CodexQuotaRefreshAttempt {
|
||||
monotonic_at: Instant,
|
||||
wall_at: SystemTime,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
@@ -398,16 +403,27 @@ impl CodexQuotaSchedulingProjection {
|
||||
.last_periodic_refresh_at
|
||||
.retain(|account_id, _| candidate_ids.contains(account_id));
|
||||
|
||||
// 正常账号只由真实请求的响应头和 `codex.rate_limits` 被动同步。定时器只复核
|
||||
// 已耗尽且已到 reset/fallback 时刻的账号;重复复核至少间隔 30 分钟。
|
||||
// 正常账号只由真实请求的响应头和 `codex.rate_limits` 被动同步。
|
||||
// 已耗尽账号每 30 分钟复核,不能等待旧 reset:官方活动可能提前重置额度。
|
||||
// reset + 2 分钟可提前触发一次复核,给上游重置留出传播时间。
|
||||
let mut reserved = Vec::new();
|
||||
for account in candidates {
|
||||
if !periodic_quota_refresh_due(&state, account.id(), refreshed_at) {
|
||||
if !periodic_quota_refresh_due(
|
||||
&state,
|
||||
account.id(),
|
||||
account.quota().reset_at(),
|
||||
now,
|
||||
refreshed_at,
|
||||
) {
|
||||
continue;
|
||||
}
|
||||
state
|
||||
.last_periodic_refresh_at
|
||||
.insert(account.id().clone(), refreshed_at);
|
||||
state.last_periodic_refresh_at.insert(
|
||||
account.id().clone(),
|
||||
CodexQuotaRefreshAttempt {
|
||||
monotonic_at: refreshed_at,
|
||||
wall_at: now,
|
||||
},
|
||||
);
|
||||
reserved.push(account);
|
||||
}
|
||||
reserved
|
||||
@@ -415,24 +431,27 @@ impl CodexQuotaSchedulingProjection {
|
||||
}
|
||||
|
||||
fn quota_refresh_candidate(account: ProviderAccount, now: SystemTime) -> Option<ProviderAccount> {
|
||||
// reset 到期是“应刷新”的信号,不是“已恢复”的证据。
|
||||
(eligible_periodic_quota_refresh(&account, now)
|
||||
&& account
|
||||
.quota()
|
||||
.exhaustion_refresh_due(now, EXHAUSTED_QUOTA_FALLBACK_RECHECK_INTERVAL))
|
||||
.then_some(account)
|
||||
(eligible_periodic_quota_refresh(&account, now) && account.quota().is_exhausted())
|
||||
.then_some(account)
|
||||
}
|
||||
|
||||
fn periodic_quota_refresh_due(
|
||||
state: &CodexQuotaProjectionState,
|
||||
account_id: &ProviderAccountId,
|
||||
now: Instant,
|
||||
reset_at: Option<SystemTime>,
|
||||
now: SystemTime,
|
||||
monotonic_now: Instant,
|
||||
) -> bool {
|
||||
state
|
||||
.last_periodic_refresh_at
|
||||
.get(account_id)
|
||||
.is_none_or(|last| {
|
||||
now.saturating_duration_since(*last) >= EXHAUSTED_QUOTA_REFRESH_RETRY_INTERVAL
|
||||
monotonic_now.saturating_duration_since(last.monotonic_at)
|
||||
>= EXHAUSTED_QUOTA_REFRESH_RETRY_INTERVAL
|
||||
|| reset_at
|
||||
.and_then(|reset| reset.checked_add(EXHAUSTED_QUOTA_RESET_GRACE))
|
||||
// 已在该边界之后复核过时回到周期重试,避免过期 reset 每轮触发。
|
||||
.is_some_and(|due_at| last.wall_at < due_at && due_at <= now)
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
//! 成功取得 OAuth token 后写入独立恢复日志的记录。
|
||||
//!
|
||||
//! 此日志会刻意保存原始 AT/RT,避免后续数据库写入或校验失败时,已成功
|
||||
//! 交换的账号无法恢复。记录独立落盘,但沿用普通结构化日志的滚动和清理策略。
|
||||
//! 交换的账号无法恢复。Host 在 oauth_recovery 配置开启时独立落盘,沿用文件日志的滚动和清理策略。
|
||||
|
||||
const OAUTH_RECOVERY_LOG_TARGET: &str = "oauth_recovery";
|
||||
const OAUTH_RECOVERY_PROVIDER: &str = "openai";
|
||||
@@ -36,8 +36,8 @@ impl CodexOAuthRecoveryOperation {
|
||||
/// 将已取得的 AT/RT 写入独立恢复结构化日志。
|
||||
///
|
||||
/// 调用方必须在任何可能拒绝响应的本地校验、画像补全或数据库写入之前调用。
|
||||
/// 该调用不执行额外 I/O,也不返回业务错误。Host 负责恢复日志文件的分割、清理和
|
||||
/// 输出;日志系统不可用时按其既有策略处理,不能阻断 OAuth 业务流程。
|
||||
/// 此处只发出结构化事件,不返回业务错误。Host 的 oauth_recovery 配置控制落盘,
|
||||
/// 并负责文件分割、清理和写入队列背压。
|
||||
pub(crate) fn record_oauth_recovery(
|
||||
operation: CodexOAuthRecoveryOperation,
|
||||
account_id: Option<&str>,
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
//! OpenAI attempt 的选择、发送与响应流执行。
|
||||
|
||||
use gateway_core::metering::{CalculatedCost, Usage};
|
||||
|
||||
use super::*;
|
||||
|
||||
impl CodexProvider {
|
||||
@@ -490,6 +492,14 @@ pub(super) fn cold_json_response_stream(request: ColdJsonResponse) -> EventStrea
|
||||
let response_meta =
|
||||
ResponseMeta::for_provider_endpoint(request.context.request_id().as_str());
|
||||
yield ProviderEvent::canonical(GatewayEvent::Started(response_meta.clone()));
|
||||
if matches!(request.endpoint_path, CODEX_IMAGE_GENERATIONS_PATH | CODEX_IMAGE_EDITS_PATH)
|
||||
&& let Some((usage, cost)) = image_response_metering(&request.body, &response.body)
|
||||
{
|
||||
yield ProviderEvent::canonical(GatewayEvent::Usage(usage));
|
||||
if let Some(cost) = cost {
|
||||
yield ProviderEvent::canonical(GatewayEvent::CalculatedCost(cost));
|
||||
}
|
||||
}
|
||||
let wire = ProtocolWireEvent::raw_json(PROVIDER_NAME, response.body).map_err(|_| {
|
||||
provider_error(ProviderErrorKind::Protocol, UpstreamSendState::Sent)
|
||||
})?;
|
||||
@@ -500,6 +510,37 @@ pub(super) fn cold_json_response_stream(request: ColdJsonResponse) -> EventStrea
|
||||
})
|
||||
}
|
||||
|
||||
fn image_response_metering(
|
||||
request_body: &[u8],
|
||||
body: &[u8],
|
||||
) -> Option<(Usage, Option<CalculatedCost>)> {
|
||||
// 只保留 usage,跳过通常很大的 base64 图片;原始响应仍按字节透传。
|
||||
#[derive(Deserialize)]
|
||||
struct ImageUsageEnvelope {
|
||||
usage: Option<Value>,
|
||||
}
|
||||
|
||||
let raw = serde_json::from_slice::<ImageUsageEnvelope>(body)
|
||||
.ok()?
|
||||
.usage?;
|
||||
let mut usage = Usage::new();
|
||||
usage.input_tokens = raw.get("input_tokens").and_then(Value::as_u64);
|
||||
usage.output_tokens = raw.get("output_tokens").and_then(Value::as_u64);
|
||||
usage.cached_tokens = raw
|
||||
.pointer("/input_tokens_details/cached_tokens")
|
||||
.and_then(Value::as_u64);
|
||||
usage.image_input_tokens = raw
|
||||
.pointer("/input_tokens_details/image_tokens")
|
||||
.and_then(Value::as_u64);
|
||||
usage.image_output_tokens = raw
|
||||
.pointer("/output_tokens_details/image_tokens")
|
||||
.and_then(Value::as_u64);
|
||||
// 总量是上游独立报告的事实;图片明细是总输入/输出的子集,不能再次相加。
|
||||
usage.total_tokens = raw.get("total_tokens").and_then(Value::as_u64);
|
||||
let cost = crate::transport::usage::image_calculated_cost(request_body, &raw);
|
||||
(usage != Usage::default()).then_some((usage, cost))
|
||||
}
|
||||
|
||||
pub(super) fn cold_response_stream(response: ColdResponse) -> EventStream {
|
||||
let ColdResponse {
|
||||
client,
|
||||
|
||||
@@ -703,10 +703,7 @@ pub(super) fn map_client_error(
|
||||
uncertain_state: UpstreamSendState,
|
||||
observe_transport: bool,
|
||||
) -> MappedProviderFailure {
|
||||
let websocket_diagnostic = match &error {
|
||||
CodexClientError::WebSocket(error) => Some(websocket_diagnostic(error)),
|
||||
_ => None,
|
||||
};
|
||||
let diagnostic = client_diagnostic(&error);
|
||||
let raw_upstream_error = match &error {
|
||||
CodexClientError::WebSocket(error) => websocket_raw_error(error),
|
||||
_ => None,
|
||||
@@ -838,7 +835,7 @@ pub(super) fn map_client_error(
|
||||
ContinuationRecoveryDisposition::ClientReplayRequired,
|
||||
);
|
||||
}
|
||||
if let Some(diagnostic) = websocket_diagnostic {
|
||||
if let Some(diagnostic) = diagnostic {
|
||||
failure.error = failure.error.with_diagnostic(diagnostic);
|
||||
}
|
||||
if let Some(raw) = raw_upstream_error {
|
||||
@@ -853,7 +850,119 @@ pub(super) fn map_client_error(
|
||||
failure
|
||||
}
|
||||
|
||||
/// 在消费 transport 错误前提取可持久化事实,不能使用可能携带 URL/凭据的 Display。
|
||||
fn client_diagnostic(error: &CodexClientError) -> Option<ProviderDiagnostic> {
|
||||
let (stage, code, message) = match error {
|
||||
CodexClientError::WebSocket(error) => return Some(websocket_diagnostic(error)),
|
||||
CodexClientError::Http(error) | CodexClientError::HttpJson(error) => {
|
||||
let (stage, code) = if error.is_builder() {
|
||||
("prepare", "http_request_build_failed")
|
||||
} else if error.is_connect() && error.is_timeout() {
|
||||
("connect", "http_connect_timeout")
|
||||
} else if error.is_connect() {
|
||||
("connect", "http_connect_failed")
|
||||
} else if error.is_timeout() {
|
||||
("exchange", "http_timeout")
|
||||
} else if error.is_decode() {
|
||||
("decode", "http_decode_failed")
|
||||
} else if error.is_body() {
|
||||
("receive", "http_body_failed")
|
||||
} else {
|
||||
("exchange", "http_transport_failed")
|
||||
};
|
||||
return Some(
|
||||
ProviderDiagnostic::new(format!(
|
||||
"OpenAI HTTP failure: stage={stage}, cause={code}"
|
||||
))
|
||||
.with_classification(stage, code)
|
||||
.with_io_cause(error),
|
||||
);
|
||||
}
|
||||
CodexClientError::StreamIdleTimeout { timeout } => (
|
||||
"receive",
|
||||
"receive_idle_timeout",
|
||||
format!("OpenAI HTTP response event idle timeout after {timeout:?}"),
|
||||
),
|
||||
CodexClientError::InvalidSse(_) => (
|
||||
"decode",
|
||||
"invalid_sse",
|
||||
"OpenAI HTTP returned an invalid Responses event stream".to_owned(),
|
||||
),
|
||||
CodexClientError::InvalidHeaderName(_) => (
|
||||
"prepare",
|
||||
"invalid_header_name",
|
||||
"OpenAI request header name is invalid".to_owned(),
|
||||
),
|
||||
CodexClientError::InvalidHeaderValue(_) => (
|
||||
"prepare",
|
||||
"invalid_header_value",
|
||||
"OpenAI request header value is invalid".to_owned(),
|
||||
),
|
||||
CodexClientError::WebSocketEncode(_) | CodexClientError::RequestBodyEncode(_) => (
|
||||
"prepare",
|
||||
"request_encode_failed",
|
||||
"OpenAI request encoding failed".to_owned(),
|
||||
),
|
||||
CodexClientError::RequestCompression(_) => (
|
||||
"prepare",
|
||||
"request_compression_failed",
|
||||
"OpenAI request compression failed".to_owned(),
|
||||
),
|
||||
CodexClientError::CustomCa(_) => (
|
||||
"prepare",
|
||||
"custom_ca_invalid",
|
||||
"OpenAI custom CA configuration is invalid".to_owned(),
|
||||
),
|
||||
CodexClientError::ModelCatalog(_) => (
|
||||
"prepare",
|
||||
"model_catalog_invalid",
|
||||
"OpenAI model catalog validation failed".to_owned(),
|
||||
),
|
||||
CodexClientError::Upstream { .. } => return None,
|
||||
};
|
||||
Some(ProviderDiagnostic::new(message).with_classification(stage, code))
|
||||
}
|
||||
|
||||
fn websocket_diagnostic(error: &CodexWebSocketExchangeError) -> ProviderDiagnostic {
|
||||
let (stage, code) = match error.diagnostic_cause() {
|
||||
CodexWebSocketExchangeError::Connect(_) => (
|
||||
"connect",
|
||||
error
|
||||
.transport_failure_reason()
|
||||
.unwrap_or("websocket_connect_failed"),
|
||||
),
|
||||
CodexWebSocketExchangeError::ConnectTimeout { .. } => ("connect", "connect_timeout"),
|
||||
CodexWebSocketExchangeError::SharedConnectFailed => ("connect", "shared_connect_failed"),
|
||||
CodexWebSocketExchangeError::OriginCircuitOpen => ("connect", "origin_circuit_open"),
|
||||
CodexWebSocketExchangeError::OriginHalfOpenBusy => ("connect", "origin_half_open_busy"),
|
||||
CodexWebSocketExchangeError::InvalidRequest(_) => ("prepare", "invalid_request"),
|
||||
CodexWebSocketExchangeError::SendTimeout { .. } => ("send", "send_timeout"),
|
||||
CodexWebSocketExchangeError::ReceiveIdleTimeout { .. } => {
|
||||
("receive", "receive_idle_timeout")
|
||||
}
|
||||
CodexWebSocketExchangeError::InvalidSse(_) => ("decode", "invalid_sse"),
|
||||
CodexWebSocketExchangeError::UnexpectedBinaryEvent => ("decode", "unexpected_binary_event"),
|
||||
CodexWebSocketExchangeError::ClosedBeforeTerminal(_) => {
|
||||
("receive", "closed_before_terminal")
|
||||
}
|
||||
CodexWebSocketExchangeError::ConnectionLimitReached => {
|
||||
("upstream", "connection_limit_reached")
|
||||
}
|
||||
CodexWebSocketExchangeError::ContinuationUnavailable { .. } => {
|
||||
("prepare", "continuation_unavailable")
|
||||
}
|
||||
CodexWebSocketExchangeError::Upstream(_) => ("connect", "websocket_opening_rejected"),
|
||||
_ => (
|
||||
"exchange",
|
||||
error
|
||||
.transport_failure_reason()
|
||||
.unwrap_or("websocket_transport_failed"),
|
||||
),
|
||||
};
|
||||
websocket_diagnostic_message(error).with_classification(stage, code)
|
||||
}
|
||||
|
||||
fn websocket_diagnostic_message(error: &CodexWebSocketExchangeError) -> ProviderDiagnostic {
|
||||
if let Some(close) = error.close_before_terminal() {
|
||||
let mut message = close.code().map_or_else(
|
||||
|| "OpenAI WebSocket closed before a terminal response".to_owned(),
|
||||
@@ -868,7 +977,25 @@ fn websocket_diagnostic(error: &CodexWebSocketExchangeError) -> ProviderDiagnost
|
||||
return ProviderDiagnostic::new(message);
|
||||
}
|
||||
|
||||
ProviderDiagnostic::new(match error.classified() {
|
||||
if let Some(reason) = error.transport_failure_reason() {
|
||||
if matches!(
|
||||
error.diagnostic_cause(),
|
||||
CodexWebSocketExchangeError::Connect(_)
|
||||
) {
|
||||
return ProviderDiagnostic::new(format!(
|
||||
"OpenAI WebSocket connection failed before payload send ({reason})"
|
||||
));
|
||||
}
|
||||
return ProviderDiagnostic::new(if reason == "reset_without_closing_handshake" {
|
||||
"OpenAI WebSocket disconnected without a closing handshake after payload send; result is ambiguous".to_owned()
|
||||
} else {
|
||||
format!(
|
||||
"OpenAI WebSocket transport failed after payload send ({reason}); result is ambiguous"
|
||||
)
|
||||
});
|
||||
}
|
||||
|
||||
ProviderDiagnostic::new(match error.diagnostic_cause() {
|
||||
CodexWebSocketExchangeError::InvalidRequest(_) => {
|
||||
"OpenAI WebSocket request could not be constructed".to_owned()
|
||||
}
|
||||
@@ -1041,10 +1168,13 @@ pub(super) fn map_upstream_failure(
|
||||
.upstream_code()
|
||||
.map_or_else(|| "none".to_owned(), |code| code.as_str().to_owned());
|
||||
let kind = error.kind().as_str();
|
||||
error = error.with_diagnostic(ProviderDiagnostic::new(format!(
|
||||
"OpenAI upstream failure: kind={}, status={status}, code={code}",
|
||||
kind
|
||||
)));
|
||||
error = error.with_diagnostic(
|
||||
ProviderDiagnostic::new(format!(
|
||||
"OpenAI upstream failure: kind={}, status={status}, code={code}",
|
||||
kind
|
||||
))
|
||||
.with_classification("upstream", "upstream_rejected"),
|
||||
);
|
||||
MappedProviderFailure {
|
||||
error,
|
||||
websocket_transport_retryable: false,
|
||||
|
||||
@@ -237,6 +237,15 @@ impl Provider for CodexProvider {
|
||||
_ => None,
|
||||
};
|
||||
return ProviderRequestObservation {
|
||||
requested_model: match operation {
|
||||
Operation::Search(request) => {
|
||||
observation::endpoint_requested_model(request.payload())
|
||||
}
|
||||
Operation::GenerateImage(request) => {
|
||||
observation::endpoint_requested_model(request.payload())
|
||||
}
|
||||
_ => None,
|
||||
},
|
||||
continuation: ContinuationRequestObservation {
|
||||
affinity_hash: affinity.map(|affinity| affinity.persistence_hash().to_owned()),
|
||||
..Default::default()
|
||||
@@ -259,6 +268,7 @@ impl Provider for CodexProvider {
|
||||
requested: previous_response_id.is_some(),
|
||||
};
|
||||
ProviderRequestObservation {
|
||||
requested_model: None,
|
||||
reasoning_effort,
|
||||
reasoning_preset: semantics.reasoning_preset.map(str::to_owned),
|
||||
request_kind: semantics.request_kind,
|
||||
|
||||
@@ -2,6 +2,23 @@
|
||||
|
||||
use super::*;
|
||||
|
||||
pub(super) fn endpoint_requested_model(
|
||||
payload: &gateway_core::operation::RawJsonPayload,
|
||||
) -> Option<gateway_core::routing::PublicModelId> {
|
||||
if payload.protocol() != PROVIDER_NAME {
|
||||
return None;
|
||||
}
|
||||
// 只读取 model,避免把编辑请求中的整张图片复制到观测数据。
|
||||
#[derive(Deserialize)]
|
||||
struct RequestModel {
|
||||
model: Option<Value>,
|
||||
}
|
||||
let model = serde_json::from_slice::<RequestModel>(payload.body())
|
||||
.ok()?
|
||||
.model?;
|
||||
gateway_core::routing::PublicModelId::new(model.as_str()?.to_owned()).ok()
|
||||
}
|
||||
|
||||
/// OpenAI Responses 的观测状态完全归 Provider 所有。
|
||||
///
|
||||
/// 每次原始 SSE/WS 事件推进状态后重新生成不可变 observation,Core 只负责携带、
|
||||
@@ -128,9 +145,9 @@ impl OpenAiResponseObservationState {
|
||||
}
|
||||
|
||||
pub(super) fn effective_service_tier(&self) -> Option<&str> {
|
||||
self.requested_service_tier
|
||||
self.upstream_service_tier
|
||||
.as_deref()
|
||||
.or(self.upstream_service_tier.as_deref())
|
||||
.or(self.requested_service_tier.as_deref())
|
||||
}
|
||||
|
||||
pub(super) fn merge_rate_limit_headers(&mut self, updates: &[(String, String)]) -> bool {
|
||||
|
||||
@@ -7,8 +7,10 @@ pub(super) const WORKER_MAXIMUM_BACKOFF: Duration = Duration::from_secs(60);
|
||||
pub(super) const WORKER_LEASE_TTL: Duration = Duration::from_secs(15 * 60);
|
||||
pub(super) const WORKER_LEASE_RENEWAL: Duration = Duration::from_secs(5 * 60);
|
||||
pub(super) const OAUTH_REFRESH_INTERVAL: Duration = Duration::from_secs(30);
|
||||
pub(super) const QUOTA_CHECK_INTERVAL: Duration = Duration::from_secs(30);
|
||||
pub(super) const DESKTOP_RELEASE_WORKER_OWNER: &str = "openai-desktop-release";
|
||||
pub(super) const MODEL_ETAG_WORKER_OWNER: &str = "openai-model-etag";
|
||||
pub(super) const MODEL_CATALOG_WORKER_OWNER: &str = "openai-model-catalog";
|
||||
|
||||
pub(crate) fn worker_contributions(
|
||||
refresh: Arc<CodexCredentialRefreshService>,
|
||||
@@ -20,6 +22,7 @@ pub(crate) fn worker_contributions(
|
||||
) -> Result<Vec<WorkerContribution>, WorkerDefinitionError> {
|
||||
let refresh_id = WorkerId::try_new(WorkerKind::OAuthRefresh, PROVIDER_NAME)?;
|
||||
let quota_id = WorkerId::try_new(WorkerKind::QuotaCatalogHealth, PROVIDER_NAME)?;
|
||||
let catalog_id = WorkerId::try_new(WorkerKind::QuotaCatalogHealth, MODEL_CATALOG_WORKER_OWNER)?;
|
||||
let etag_id = WorkerId::try_new(WorkerKind::QuotaCatalogHealth, MODEL_ETAG_WORKER_OWNER)?;
|
||||
let desktop_release_id =
|
||||
WorkerId::try_new(WorkerKind::QuotaCatalogHealth, DESKTOP_RELEASE_WORKER_OWNER)?;
|
||||
@@ -34,9 +37,13 @@ pub(crate) fn worker_contributions(
|
||||
contributions.extend([
|
||||
WorkerContribution::Registration(scheduled_registration(
|
||||
quota_id,
|
||||
QUOTA_CHECK_INTERVAL,
|
||||
Box::new(OpenAiQuotaTask { quota }),
|
||||
)?),
|
||||
WorkerContribution::Registration(scheduled_registration(
|
||||
catalog_id,
|
||||
quota_refresh_policy.interval(),
|
||||
Box::new(OpenAiQuotaTask {
|
||||
quota,
|
||||
Box::new(OpenAiCatalogTask {
|
||||
catalog: Arc::clone(&catalog),
|
||||
}),
|
||||
)?),
|
||||
@@ -159,6 +166,9 @@ impl ScheduledTask for OpenAiOAuthRefreshTask {
|
||||
|
||||
pub(super) struct OpenAiQuotaTask {
|
||||
quota: Arc<CodexCredentialQuotaService>,
|
||||
}
|
||||
|
||||
pub(super) struct OpenAiCatalogTask {
|
||||
catalog: Arc<CodexCredentialCatalogService>,
|
||||
}
|
||||
|
||||
@@ -195,7 +205,6 @@ impl ScheduledTask for OpenAiQuotaTask {
|
||||
if context.cancellation().is_cancelled() {
|
||||
return Ok(());
|
||||
}
|
||||
let mut failures = false;
|
||||
match self.quota.synchronize().await {
|
||||
Ok(summary) if summary.has_operational_failures() => {
|
||||
tracing::warn!(
|
||||
@@ -209,30 +218,33 @@ impl ScheduledTask for OpenAiQuotaTask {
|
||||
}
|
||||
Ok(_) => {}
|
||||
Err(error) => {
|
||||
failures = true;
|
||||
tracing::warn!(
|
||||
error = %error,
|
||||
"OpenAI quota synchronization failed"
|
||||
);
|
||||
return Err(WorkerTaskError::safe("OpenAI quota synchronization failed"));
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
impl ScheduledTask for OpenAiCatalogTask {
|
||||
fn run_cycle(&self, context: WorkerCycleContext) -> BoxFuture<'_, Result<(), WorkerTaskError>> {
|
||||
Box::pin(async move {
|
||||
if context.cancellation().is_cancelled() {
|
||||
return Ok(());
|
||||
}
|
||||
match self.catalog.refresh_catalogs().await {
|
||||
Ok(_) | Err(CodexCredentialCatalogError::NoEligibleCredential) => {}
|
||||
Ok(_) | Err(CodexCredentialCatalogError::NoEligibleCredential) => Ok(()),
|
||||
Err(error) => {
|
||||
failures = true;
|
||||
tracing::warn!(error = %error, "OpenAI model catalog refresh failed");
|
||||
Err(WorkerTaskError::safe(
|
||||
"OpenAI model catalog synchronization failed",
|
||||
))
|
||||
}
|
||||
}
|
||||
if failures {
|
||||
Err(WorkerTaskError::safe(
|
||||
"OpenAI quota or catalog synchronization failed",
|
||||
))
|
||||
} else {
|
||||
Ok(())
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -158,7 +158,7 @@ impl CodexCanonicalDecoder {
|
||||
self
|
||||
}
|
||||
|
||||
/// 请求显式指定的档位优先用于计费;响应档位只在请求未指定时兜底。
|
||||
/// 响应未报告实际档位时,才使用请求档位估算费用。
|
||||
#[must_use]
|
||||
pub fn with_requested_service_tier(mut self, service_tier: Option<&str>) -> Self {
|
||||
self.requested_service_tier = normalize_service_tier(service_tier);
|
||||
@@ -862,18 +862,19 @@ impl CodexCanonicalDecoder {
|
||||
.unwrap_or(&self.fallback_model)
|
||||
.to_owned();
|
||||
let service_tier = self
|
||||
.requested_service_tier
|
||||
.response_service_tier
|
||||
.as_deref()
|
||||
.or(self.response_service_tier.as_deref());
|
||||
let web_search_calls = web_search_call_count(response);
|
||||
.or(self.requested_service_tier.as_deref());
|
||||
let tool_calls = billable_tool_calls(response);
|
||||
if let Some(breakdown) = usage
|
||||
.filter(|usage| billable_usage_is_complete(response, *usage))
|
||||
.and_then(|usage| {
|
||||
let web_search_calls = web_search_calls?;
|
||||
let (web_search_calls, file_search_calls) = tool_calls?;
|
||||
openai_billing_breakdown(
|
||||
&model,
|
||||
OpenAiBillingUsage::from(usage)
|
||||
.with_web_search_calls(web_search_calls, self.web_search_pricing),
|
||||
.with_web_search_calls(web_search_calls, self.web_search_pricing)
|
||||
.with_file_search_calls(file_search_calls),
|
||||
service_tier,
|
||||
)
|
||||
})
|
||||
@@ -1000,26 +1001,43 @@ fn core_usage(usage: TokenUsage) -> Usage {
|
||||
normalized
|
||||
}
|
||||
|
||||
fn web_search_call_count(response: &Value) -> Option<u64> {
|
||||
let mut calls = 0_u64;
|
||||
for item in response
|
||||
.get("output")
|
||||
.and_then(Value::as_array)
|
||||
.into_iter()
|
||||
.flatten()
|
||||
.filter(|item| item.get("type").and_then(Value::as_str) == Some("web_search_call"))
|
||||
{
|
||||
match item
|
||||
.get("action")
|
||||
.and_then(|action| action.get("type"))
|
||||
.and_then(Value::as_str)
|
||||
{
|
||||
Some("search") => calls = calls.checked_add(1)?,
|
||||
Some("open_page" | "find_in_page") => {}
|
||||
Some(_) | None => return None,
|
||||
fn billable_tool_calls(response: &Value) -> Option<(u64, u64)> {
|
||||
let mut web = 0_u64;
|
||||
let mut file = 0_u64;
|
||||
let items = match response.get("output") {
|
||||
Some(value) => value.as_array()?.as_slice(),
|
||||
None => &[],
|
||||
};
|
||||
for item in items {
|
||||
match item.get("type").and_then(Value::as_str)? {
|
||||
"web_search_call" => match item.pointer("/action/type").and_then(Value::as_str) {
|
||||
Some("search") => web = web.checked_add(1)?,
|
||||
Some("open_page" | "find_in_page") => {}
|
||||
_ => return None,
|
||||
},
|
||||
"file_search_call"
|
||||
if item.get("status").and_then(Value::as_str) == Some("completed") =>
|
||||
{
|
||||
file = file.checked_add(1)?;
|
||||
}
|
||||
// 这些输出没有独立的 OpenAI 工具调用费。
|
||||
"message"
|
||||
| "reasoning"
|
||||
| "function_call"
|
||||
| "custom_tool_call"
|
||||
| "computer_call"
|
||||
| "local_shell_call"
|
||||
| "apply_patch_call"
|
||||
| "mcp_call"
|
||||
| "mcp_list_tools"
|
||||
| "mcp_approval_request"
|
||||
| "compaction" => {}
|
||||
// 容器会话、生图及未知工具需要额外计费事实,不能把 token 小计
|
||||
// 当作完整费用。
|
||||
_ => return None,
|
||||
}
|
||||
}
|
||||
Some(calls)
|
||||
Some((web, file))
|
||||
}
|
||||
|
||||
fn incomplete_finish_reason(response: &Value) -> FinishReason {
|
||||
|
||||
@@ -851,7 +851,9 @@ pub(super) fn websocket_success_decision(
|
||||
) -> CodexTransportDecision {
|
||||
match requirement {
|
||||
TransportRequirement::ExactWebSocketContinuation => CodexTransportDecision::ExactWebSocket,
|
||||
TransportRequirement::ExplicitWebSocketWarmup => CodexTransportDecision::RequiredWebSocket,
|
||||
TransportRequirement::ExplicitWebSocketWarmup | TransportRequirement::WebSocketNewChain => {
|
||||
CodexTransportDecision::RequiredWebSocket
|
||||
}
|
||||
_ if prepared.reused() => CodexTransportDecision::ReusedWebSocket,
|
||||
_ => CodexTransportDecision::ConnectedWebSocket,
|
||||
}
|
||||
|
||||
@@ -272,6 +272,7 @@ impl CodexBackendClient {
|
||||
Some(WEBSOCKET_FAST_PATH_BUDGET)
|
||||
}
|
||||
TransportRequirement::ExplicitWebSocketWarmup
|
||||
| TransportRequirement::WebSocketNewChain
|
||||
| TransportRequirement::ExactWebSocketContinuation
|
||||
| TransportRequirement::ExternalUnknown => None,
|
||||
TransportRequirement::HttpRequired => None,
|
||||
|
||||
@@ -121,6 +121,8 @@ pub enum TransportRequirement {
|
||||
HttpRequired,
|
||||
/// `generate=false + store=false` 预热必须保留在同一条 WebSocket。
|
||||
ExplicitWebSocketWarmup,
|
||||
/// 客户端 WebSocket 的非持久化新链,必须在池化连接上建立后续续接状态。
|
||||
WebSocketNewChain,
|
||||
/// 只能使用持有指定 connection-local response 的精确 WebSocket。
|
||||
ExactWebSocketContinuation,
|
||||
/// previous response 已持久化,允许 WebSocket 或 HTTP/2。
|
||||
@@ -136,7 +138,9 @@ impl TransportRequirement {
|
||||
pub fn requires_websocket(self) -> bool {
|
||||
matches!(
|
||||
self,
|
||||
Self::ExplicitWebSocketWarmup | Self::ExactWebSocketContinuation
|
||||
Self::ExplicitWebSocketWarmup
|
||||
| Self::WebSocketNewChain
|
||||
| Self::ExactWebSocketContinuation
|
||||
)
|
||||
}
|
||||
|
||||
@@ -150,7 +154,7 @@ impl TransportRequirement {
|
||||
|
||||
/// 无续接依赖的新请求可以在明确的连接级拒绝后重新建连。
|
||||
pub fn allows_connection_restart(self) -> bool {
|
||||
matches!(self, Self::NewChain)
|
||||
matches!(self, Self::NewChain | Self::WebSocketNewChain)
|
||||
}
|
||||
|
||||
/// 用于审计与遥测的稳定名称。
|
||||
@@ -158,6 +162,7 @@ impl TransportRequirement {
|
||||
match self {
|
||||
Self::HttpRequired => "http_required",
|
||||
Self::ExplicitWebSocketWarmup => "explicit_websocket_warmup",
|
||||
Self::WebSocketNewChain => "websocket_new_chain",
|
||||
Self::ExactWebSocketContinuation => "exact_websocket_continuation",
|
||||
Self::PersistedContinuation => "persisted_continuation",
|
||||
Self::ExternalUnknown => "external_unknown",
|
||||
@@ -184,6 +189,11 @@ pub fn transport_requirement(request: &CodexResponsesRequest) -> TransportRequir
|
||||
TransportRequirement::ExternalUnknown
|
||||
}
|
||||
},
|
||||
// 客户端会在下一轮提交 response ID;HTTP store=false 的成功响应无法
|
||||
// 在池化 WebSocket 上续接,所以首轮也不能按普通快路径降级到 HTTP。
|
||||
None if request.downstream_websocket_connection_id.is_some() && !request.store() => {
|
||||
TransportRequirement::WebSocketNewChain
|
||||
}
|
||||
None => TransportRequirement::NewChain,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
use gateway_core::metering::{
|
||||
CalculatedCostAmounts, CalculatedCostBreakdown, CalculatedCostRates, CurrencyCode, Decimal,
|
||||
Money,
|
||||
CalculatedCost, CalculatedCostAmounts, CalculatedCostBreakdown, CalculatedCostRates,
|
||||
CurrencyCode, Decimal, Money,
|
||||
};
|
||||
use gateway_protocol::openai::events::{TokenUsage, retry_after_seconds_from_body};
|
||||
use reqwest::StatusCode;
|
||||
@@ -16,6 +16,7 @@ use super::{
|
||||
const LONG_CONTEXT_THRESHOLD: u64 = 272_000;
|
||||
const WEB_SEARCH_CALL_TICKS: u128 = 100_000_000;
|
||||
const WEB_SEARCH_PREVIEW_NON_REASONING_CALL_TICKS: u128 = 250_000_000;
|
||||
const FILE_SEARCH_CALL_TICKS: u128 = 25_000_000;
|
||||
|
||||
/// OpenAI 公开 Token 价格计算所需的单次用量事实。
|
||||
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
|
||||
@@ -27,6 +28,7 @@ pub struct OpenAiBillingUsage {
|
||||
image_input_tokens: u64,
|
||||
image_output_tokens: u64,
|
||||
web_search_calls: u64,
|
||||
file_search_calls: u64,
|
||||
web_search_pricing: Option<WebSearchPricing>,
|
||||
}
|
||||
|
||||
@@ -47,6 +49,7 @@ impl OpenAiBillingUsage {
|
||||
image_input_tokens: 0,
|
||||
image_output_tokens: 0,
|
||||
web_search_calls: 0,
|
||||
file_search_calls: 0,
|
||||
web_search_pricing: None,
|
||||
}
|
||||
}
|
||||
@@ -60,6 +63,11 @@ impl OpenAiBillingUsage {
|
||||
self.web_search_pricing = pricing;
|
||||
self
|
||||
}
|
||||
|
||||
pub(crate) const fn with_file_search_calls(mut self, calls: u64) -> Self {
|
||||
self.file_search_calls = calls;
|
||||
self
|
||||
}
|
||||
}
|
||||
|
||||
impl From<TokenUsage> for OpenAiBillingUsage {
|
||||
@@ -72,6 +80,7 @@ impl From<TokenUsage> for OpenAiBillingUsage {
|
||||
image_input_tokens: usage.image_input_tokens,
|
||||
image_output_tokens: usage.image_output_tokens,
|
||||
web_search_calls: 0,
|
||||
file_search_calls: 0,
|
||||
web_search_pricing: None,
|
||||
}
|
||||
}
|
||||
@@ -125,6 +134,7 @@ struct ModelPricing {
|
||||
long_flex: TokenRates,
|
||||
long_fast: TokenRates,
|
||||
cache_write_percent: u32,
|
||||
unpriced_long_context: bool,
|
||||
}
|
||||
|
||||
impl ModelPricing {
|
||||
@@ -137,6 +147,7 @@ impl ModelPricing {
|
||||
long_flex: TokenRates::ZERO,
|
||||
long_fast: TokenRates::ZERO,
|
||||
cache_write_percent: 0,
|
||||
unpriced_long_context: false,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -170,7 +181,15 @@ impl ModelPricing {
|
||||
self
|
||||
}
|
||||
|
||||
const fn with_unpriced_long_context(mut self) -> Self {
|
||||
self.unpriced_long_context = true;
|
||||
self
|
||||
}
|
||||
|
||||
fn rates(self, tier: PricingTier, long_context: bool) -> Option<TokenRates> {
|
||||
if long_context && self.unpriced_long_context {
|
||||
return None;
|
||||
}
|
||||
// Only models with a published long-context column switch at the threshold.
|
||||
// A dash in that column is not a second, unavailable price tier for models
|
||||
// whose entire supported context is covered by the short-context price.
|
||||
@@ -200,10 +219,12 @@ struct PricingRule {
|
||||
pricing: ModelPricing,
|
||||
}
|
||||
|
||||
// Source: https://developers.openai.com/api/docs/pricing (verified 2026-08-14).
|
||||
// 价格来源:https://developers.openai.com/api/docs/pricing,核验日期 2026-09-09。
|
||||
// 使用常规价,不采用 Sol 的临时 $4/$20 优惠;缓存、Flex、Fast 和长上下文
|
||||
// 档位也统一按常规价计算。
|
||||
const PRICING_RULES: &[PricingRule] = &[
|
||||
// Astra: https://developers.openai.com/api/docs/models/gpt-6-astra
|
||||
// Rates verified against the pricing table on 2026-09-05.
|
||||
// Astra:https://developers.openai.com/api/docs/models/gpt-6-astra
|
||||
// 已于 2026-09-09 对照官方价目表核验。
|
||||
PricingRule {
|
||||
model: "gpt-6-astra",
|
||||
pricing: ModelPricing::new(100_000, 500_000, 10_000)
|
||||
@@ -405,10 +426,60 @@ const PRICING_RULES: &[PricingRule] = &[
|
||||
model: "gpt-3.5-turbo",
|
||||
pricing: ModelPricing::new(5_000, 15_000, 0),
|
||||
},
|
||||
// 各变体独立采用已公开的价格与档位,不能继承父型号的所有档位。
|
||||
// Cyber 长上下文的官方来源存在差异,该区间暂不估价。
|
||||
PricingRule {
|
||||
model: "gpt-5.1-codex-mini",
|
||||
pricing: ModelPricing::new(2_500, 20_000, 250),
|
||||
},
|
||||
PricingRule {
|
||||
model: "gpt-5.3-chat-latest",
|
||||
pricing: ModelPricing::new(17_500, 140_000, 1_750),
|
||||
},
|
||||
PricingRule {
|
||||
model: "gpt-5.6-cyber",
|
||||
pricing: ModelPricing::new(125_000, 750_000, 12_500)
|
||||
.with_cache_write(125)
|
||||
.with_unpriced_long_context(),
|
||||
},
|
||||
PricingRule {
|
||||
model: "gpt-5.5-cyber",
|
||||
pricing: ModelPricing::new(125_000, 750_000, 12_500).with_unpriced_long_context(),
|
||||
},
|
||||
PricingRule {
|
||||
model: "chat-latest",
|
||||
pricing: ModelPricing::new(50_000, 300_000, 5_000),
|
||||
},
|
||||
PricingRule {
|
||||
model: "gpt-5-codex",
|
||||
pricing: ModelPricing::new(12_500, 100_000, 1_250),
|
||||
},
|
||||
PricingRule {
|
||||
model: "gpt-5.1-codex",
|
||||
pricing: ModelPricing::new(12_500, 100_000, 1_250),
|
||||
},
|
||||
PricingRule {
|
||||
model: "gpt-5.1-codex-max",
|
||||
pricing: ModelPricing::new(12_500, 100_000, 1_250),
|
||||
},
|
||||
PricingRule {
|
||||
model: "gpt-5-chat-latest",
|
||||
pricing: ModelPricing::new(12_500, 100_000, 1_250),
|
||||
},
|
||||
PricingRule {
|
||||
model: "gpt-5.1-chat-latest",
|
||||
pricing: ModelPricing::new(12_500, 100_000, 1_250),
|
||||
},
|
||||
PricingRule {
|
||||
model: "gpt-5.2-codex",
|
||||
pricing: ModelPricing::new(17_500, 140_000, 1_750),
|
||||
},
|
||||
PricingRule {
|
||||
model: "gpt-5.2-chat-latest",
|
||||
pricing: ModelPricing::new(17_500, 140_000, 1_750),
|
||||
},
|
||||
];
|
||||
|
||||
const UNPRICED_MODELS: &[&str] = &["gpt-5.3-codex-spark"];
|
||||
|
||||
#[derive(Clone, Copy)]
|
||||
struct TokenAmounts {
|
||||
input_ticks: u128,
|
||||
@@ -439,13 +510,14 @@ fn openai_billing_breakdown_with_context(
|
||||
service_tier: Option<&str>,
|
||||
long_context: bool,
|
||||
) -> Option<CalculatedCostBreakdown> {
|
||||
if usage.cached_tokens > usage.input_tokens
|
||||
if usage.cached_tokens.checked_add(usage.cache_write_tokens)? > usage.input_tokens
|
||||
|| usage.image_input_tokens > 0
|
||||
|| usage.image_output_tokens > 0
|
||||
{
|
||||
return None;
|
||||
}
|
||||
let web_search_ticks = web_search_amount_ticks(usage)?;
|
||||
let tool_ticks = web_search_amount_ticks(usage)?
|
||||
.checked_add(u128::from(usage.file_search_calls).checked_mul(FILE_SEARCH_CALL_TICKS)?)?;
|
||||
let pricing = model_pricing(model)?;
|
||||
let normalized_tier = normalize_service_tier(service_tier);
|
||||
let tier = pricing_tier(normalized_tier.as_deref())?;
|
||||
@@ -467,8 +539,8 @@ fn openai_billing_breakdown_with_context(
|
||||
usage.cached_tokens,
|
||||
usage.cache_write_tokens,
|
||||
)?;
|
||||
standard.total_ticks = standard.total_ticks.checked_add(web_search_ticks)?;
|
||||
selected.total_ticks = selected.total_ticks.checked_add(web_search_ticks)?;
|
||||
standard.total_ticks = standard.total_ticks.checked_add(tool_ticks)?;
|
||||
selected.total_ticks = selected.total_ticks.checked_add(tool_ticks)?;
|
||||
let multiplier_percent =
|
||||
effective_multiplier_percent(selected.total_ticks, standard.total_ticks)?;
|
||||
let cache_write_rate = cache_write_rate(selected_rates, pricing.cache_write_percent)?;
|
||||
@@ -493,18 +565,88 @@ fn openai_billing_breakdown_with_context(
|
||||
))
|
||||
}
|
||||
|
||||
fn model_pricing(model: &str) -> Option<ModelPricing> {
|
||||
let normalized = normalize_model_name(model);
|
||||
if UNPRICED_MODELS
|
||||
.iter()
|
||||
.any(|rule| model_matches_rule(&normalized, rule))
|
||||
/// 独立 Images 端点按公开标准 API 单价估算;不是 ChatGPT 账号实际扣费。
|
||||
pub(crate) fn image_calculated_cost(request_body: &[u8], usage: &Value) -> Option<CalculatedCost> {
|
||||
#[derive(serde::Deserialize)]
|
||||
struct ImageModel {
|
||||
model: String,
|
||||
}
|
||||
// 只读取模型,跳过编辑请求中可能很大的 base64 图片。
|
||||
let request = serde_json::from_slice::<ImageModel>(request_body).ok()?;
|
||||
if !matches!(
|
||||
request.model.as_str(),
|
||||
"gpt-image-2"
|
||||
| "gpt-image-2-2026-04-21"
|
||||
| "gpt-image-2.5-sunburst"
|
||||
| "gpt-image-2.5-sunburst-2026-09-08"
|
||||
| "gpt-image-2.5-flare"
|
||||
| "gpt-image-2.5-flare-2026-09-08"
|
||||
) {
|
||||
return None;
|
||||
}
|
||||
let input = usage.get("input_tokens")?.as_u64()?;
|
||||
let output = usage.get("output_tokens")?.as_u64()?;
|
||||
let details = usage.get("input_tokens_details")?;
|
||||
let text_input = details.get("text_tokens")?.as_u64()?;
|
||||
let image_input = details.get("image_tokens")?.as_u64()?;
|
||||
if text_input.checked_add(image_input)? != input {
|
||||
return None;
|
||||
}
|
||||
if let Some(total) = usage.get("total_tokens")
|
||||
&& total.as_u64()? != input.checked_add(output)?
|
||||
{
|
||||
return None;
|
||||
}
|
||||
// 此模型仅输出图片;上游如报告其他输出模态,不能套用图片单价。
|
||||
if let Some(details) = usage.get("output_tokens_details")
|
||||
&& (details.get("image_tokens")?.as_u64()? != output
|
||||
|| details.get("text_tokens")?.as_u64()? != 0)
|
||||
{
|
||||
return None;
|
||||
}
|
||||
let cached = match details.get("cached_tokens") {
|
||||
None => 0,
|
||||
Some(value) => value.as_u64()?,
|
||||
};
|
||||
if cached > input {
|
||||
return None;
|
||||
}
|
||||
// 缓存只给总量时,混合输入无法判定应套用哪种缓存单价。
|
||||
let (cached_text, cached_image) = match (text_input, image_input, cached) {
|
||||
(_, _, 0) => (0, 0),
|
||||
(_, 0, cached) => (cached, 0),
|
||||
(0, _, cached) => (0, cached),
|
||||
(_, _, cached) if cached == input => (text_input, image_input),
|
||||
_ => return None,
|
||||
};
|
||||
// 价格来源:https://developers.openai.com/api/docs/pricing,核验日期 2026-09-09。
|
||||
// GPT Image 2 和两个 2.5 型号按相同的 token 单价计费,不使用按张估价。
|
||||
// 每百万 token 的美元单价:文本 5 / 缓存 1.25;图片 8 / 缓存 2 / 输出 30。
|
||||
let text = token_amounts(
|
||||
TokenRates::new(50_000, 0, 12_500),
|
||||
0,
|
||||
text_input,
|
||||
0,
|
||||
cached_text,
|
||||
0,
|
||||
)?;
|
||||
let image = token_amounts(
|
||||
TokenRates::new(80_000, 300_000, 20_000),
|
||||
0,
|
||||
image_input,
|
||||
output,
|
||||
cached_image,
|
||||
0,
|
||||
)?;
|
||||
CalculatedCost::from_usd_ticks(text.total_ticks.checked_add(image.total_ticks)?).ok()
|
||||
}
|
||||
|
||||
fn model_pricing(model: &str) -> Option<ModelPricing> {
|
||||
let normalized = normalize_model_name(model);
|
||||
let model = pricing_model_name(&normalized);
|
||||
PRICING_RULES
|
||||
.iter()
|
||||
.filter(|rule| model_matches_rule(&normalized, rule.model))
|
||||
.max_by_key(|rule| rule.model.len())
|
||||
.find(|rule| model == rule.model)
|
||||
.map(|rule| rule.pricing)
|
||||
}
|
||||
|
||||
@@ -543,16 +685,17 @@ fn fixed_block_web_search_model(model: &str) -> bool {
|
||||
|
||||
fn reasoning_model(model: &str) -> bool {
|
||||
let normalized = normalize_model_name(model);
|
||||
normalized.starts_with("gpt-5")
|
||||
|| model_matches_rule(&normalized, "gpt-6-astra")
|
||||
|| normalized.starts_with("o1")
|
||||
|| normalized.starts_with("o3")
|
||||
|| normalized.starts_with("o4")
|
||||
let model = pricing_model_name(&normalized);
|
||||
model.starts_with("gpt-5")
|
||||
|| model == "gpt-6-astra"
|
||||
|| model.starts_with("o1")
|
||||
|| model.starts_with("o3")
|
||||
|| model.starts_with("o4")
|
||||
}
|
||||
|
||||
fn pricing_tier(service_tier: Option<&str>) -> Option<PricingTier> {
|
||||
match service_tier {
|
||||
None | Some("auto" | "default" | "standard") => Some(PricingTier::Standard),
|
||||
None | Some("default" | "standard") => Some(PricingTier::Standard),
|
||||
Some("flex") => Some(PricingTier::Flex),
|
||||
Some("fast" | "priority") => Some(PricingTier::Fast),
|
||||
Some(_) => None,
|
||||
@@ -574,20 +717,23 @@ fn token_amounts(
|
||||
cached_tokens: u64,
|
||||
cache_write_tokens: u64,
|
||||
) -> Option<TokenAmounts> {
|
||||
if cached_tokens.checked_add(cache_write_tokens)? > input_tokens {
|
||||
return None;
|
||||
}
|
||||
let billed_cache_read = if rates.cache_read_ticks > 0 {
|
||||
cached_tokens.min(input_tokens)
|
||||
cached_tokens
|
||||
} else {
|
||||
0
|
||||
};
|
||||
let cache_write_rate = cache_write_rate(rates, cache_write_percent)?;
|
||||
let billed_cache_write = if cache_write_rate > 0 {
|
||||
cache_write_tokens.min(input_tokens.saturating_sub(billed_cache_read))
|
||||
cache_write_tokens
|
||||
} else {
|
||||
0
|
||||
};
|
||||
let uncached_input = input_tokens
|
||||
.saturating_sub(billed_cache_read)
|
||||
.saturating_sub(billed_cache_write);
|
||||
.checked_sub(billed_cache_read)?
|
||||
.checked_sub(billed_cache_write)?;
|
||||
let input_ticks = u128::from(uncached_input).checked_mul(rates.input_ticks)?;
|
||||
let output_ticks = u128::from(output_tokens).checked_mul(rates.output_ticks)?;
|
||||
let cache_read_ticks = u128::from(billed_cache_read).checked_mul(rates.cache_read_ticks)?;
|
||||
@@ -633,13 +779,47 @@ fn normalize_model_name(model: &str) -> String {
|
||||
.to_ascii_lowercase()
|
||||
}
|
||||
|
||||
fn model_matches_rule(model: &str, rule: &str) -> bool {
|
||||
if model == rule {
|
||||
return true;
|
||||
// 仅已核验的别名和快照可以共用价格;未知后缀、未来日期和微调模型
|
||||
// 不得直接继承父型号的价格。
|
||||
fn pricing_model_name(model: &str) -> &str {
|
||||
match model {
|
||||
"gpt-3.5-turbo-0125" => "gpt-3.5-turbo",
|
||||
"gpt-4-0314" => "gpt-4",
|
||||
"gpt-4-0613" => "gpt-4",
|
||||
"gpt-4-turbo-2024-04-09" => "gpt-4-turbo",
|
||||
"gpt-4.1-2025-04-14" => "gpt-4.1",
|
||||
"gpt-4.1-mini-2025-04-14" => "gpt-4.1-mini",
|
||||
"gpt-4.1-nano-2025-04-14" => "gpt-4.1-nano",
|
||||
"gpt-4o-2024-08-06" => "gpt-4o",
|
||||
"gpt-4o-2024-11-20" => "gpt-4o",
|
||||
"gpt-4o-mini-2024-07-18" => "gpt-4o-mini",
|
||||
"gpt-5-2025-08-07" => "gpt-5",
|
||||
"gpt-5-mini-2025-08-07" => "gpt-5-mini",
|
||||
"gpt-5-nano-2025-08-07" => "gpt-5-nano",
|
||||
"gpt-5-pro-2025-10-06" => "gpt-5-pro",
|
||||
"gpt-5.1-2025-11-13" => "gpt-5.1",
|
||||
"gpt-5.2-2025-12-11" => "gpt-5.2",
|
||||
"gpt-5.2-pro-2025-12-11" => "gpt-5.2-pro",
|
||||
"gpt-5.4-2026-03-05" => "gpt-5.4",
|
||||
"gpt-5.4-mini-2026-03-17" => "gpt-5.4-mini",
|
||||
"gpt-5.4-nano-2026-03-17" => "gpt-5.4-nano",
|
||||
"gpt-5.4-pro-2026-03-05" => "gpt-5.4-pro",
|
||||
"gpt-5.5-2026-04-23" => "gpt-5.5",
|
||||
"gpt-5.5-pro-2026-04-23" => "gpt-5.5-pro",
|
||||
"gpt-daybreak-blue-latest" => "gpt-5.6-sol",
|
||||
"gpt-daybreak-red-latest" => "gpt-5.6-cyber",
|
||||
"o1-2024-12-17" => "o1",
|
||||
"o1-pro-2025-03-19" => "o1-pro",
|
||||
"o3-2025-04-16" => "o3",
|
||||
"o3-mini-2025-01-31" => "o3-mini",
|
||||
"o3-pro-2025-06-10" => "o3-pro",
|
||||
"o4-mini-2025-04-16" => "o4-mini",
|
||||
_ => model,
|
||||
}
|
||||
model
|
||||
.strip_prefix(rule)
|
||||
.is_some_and(|suffix| matches!(suffix.as_bytes().first(), Some(b'-' | b'.' | b':')))
|
||||
}
|
||||
|
||||
fn model_matches_rule(model: &str, rule: &str) -> bool {
|
||||
pricing_model_name(model) == rule
|
||||
}
|
||||
|
||||
/// 规范化请求或响应携带的服务档位,供观测与计费共用。
|
||||
|
||||
@@ -597,13 +597,20 @@ pub(crate) async fn execute_prepared_response_create_request_stream(
|
||||
}));
|
||||
trace.capture("upstream.request.body", request.payload_text().as_bytes());
|
||||
if let Err(error) = send_websocket_request(&websocket, request.payload_text()).await {
|
||||
trace.record(
|
||||
"upstream.send.failed",
|
||||
serde_json::json!({"phase": "websocket_payload", "sendState": "ambiguous"}),
|
||||
);
|
||||
let observation = websocket
|
||||
.observation()
|
||||
.with_exit_reason("outbound_transport_error");
|
||||
trace.record(
|
||||
"upstream.send.failed",
|
||||
serde_json::json!({
|
||||
"phase": "websocket_payload", "sendState": "ambiguous",
|
||||
"failureReason": error.transport_failure_reason().unwrap_or(observation.exit_reason()),
|
||||
"connectionId": observation.connection_id().to_string(),
|
||||
"connectionAgeMs": observation.age_ms(),
|
||||
"connectionIdleMs": observation.idle_ms(),
|
||||
"reused": reused,
|
||||
}),
|
||||
);
|
||||
discard_after_send(websocket, lease, observation.clone()).await;
|
||||
return Err(post_send_ambiguous(
|
||||
error.with_connection_observation(observation),
|
||||
|
||||
@@ -223,6 +223,50 @@ impl fmt::Display for CodexWebSocketUpstreamError {
|
||||
}
|
||||
|
||||
impl CodexWebSocketExchangeError {
|
||||
/// 仅供诊断展示穿透发送状态与连接快照包装;重试分类仍使用 classified。
|
||||
pub(crate) fn diagnostic_cause(&self) -> &Self {
|
||||
match self {
|
||||
Self::PostSendAmbiguous {
|
||||
source: Some(source),
|
||||
..
|
||||
}
|
||||
| Self::ReusedConnectionDiedBeforeFirstEvent {
|
||||
source: Some(source),
|
||||
..
|
||||
}
|
||||
| Self::ConnectionObserved { source, .. } => source.diagnostic_cause(),
|
||||
_ => self,
|
||||
}
|
||||
}
|
||||
|
||||
/// 返回可持久化的传输错误分类,不包含底层错误中的地址、报文或凭据。
|
||||
/// 与已有指标粗分类分开:缺少关闭握手本身不能证明收到 TCP RST。
|
||||
pub(crate) fn transport_failure_reason(&self) -> Option<&'static str> {
|
||||
match self.diagnostic_cause() {
|
||||
Self::Transport(error) | Self::Connect(error) => Some(match error {
|
||||
tungstenite::Error::Protocol(
|
||||
tungstenite::error::ProtocolError::ResetWithoutClosingHandshake,
|
||||
) => "reset_without_closing_handshake",
|
||||
tungstenite::Error::Io(error) => match error.kind() {
|
||||
std::io::ErrorKind::ConnectionRefused => "connection_refused",
|
||||
std::io::ErrorKind::NetworkUnreachable => "network_unreachable",
|
||||
std::io::ErrorKind::HostUnreachable => "host_unreachable",
|
||||
std::io::ErrorKind::ConnectionReset => "tcp_reset",
|
||||
std::io::ErrorKind::ConnectionAborted => "connection_aborted",
|
||||
std::io::ErrorKind::BrokenPipe => "broken_pipe",
|
||||
std::io::ErrorKind::UnexpectedEof => "unexpected_eof",
|
||||
std::io::ErrorKind::TimedOut => "transport_timeout",
|
||||
_ => "io_error",
|
||||
},
|
||||
tungstenite::Error::Tls(_) => "tls_error",
|
||||
tungstenite::Error::Protocol(_) => "protocol_error",
|
||||
tungstenite::Error::Capacity(_) => "capacity_error",
|
||||
_ => "transport_error",
|
||||
}),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn closed_before_terminal_on(
|
||||
connection_id: Uuid,
|
||||
code: Option<u16>,
|
||||
|
||||
@@ -151,6 +151,9 @@ async fn forward_websocket_response_stream(state: WebSocketStreamForwardState) {
|
||||
.map(|pool_return| std::mem::take(&mut pool_return.continuation))
|
||||
.unwrap_or_default();
|
||||
let mut last_event_type = None;
|
||||
// 官方 run_websocket_response_stream 按响应消费 metadata 事件;它们不属于握手头。
|
||||
// 复用连接时恢复握手快照,避免上一轮模型信息及普通响应头随每次请求累积。
|
||||
let opening_response_metadata = metadata.response_metadata.clone();
|
||||
loop {
|
||||
let message = tokio::select! {
|
||||
biased;
|
||||
@@ -182,14 +185,19 @@ async fn forward_websocket_response_stream(state: WebSocketStreamForwardState) {
|
||||
let message = match message {
|
||||
Ok(message) => message,
|
||||
Err(error) => {
|
||||
let observation = connection_observation(&websocket, &error);
|
||||
trace.record(
|
||||
"upstream.read.failed",
|
||||
json!({
|
||||
"lastEventType": last_event_type,
|
||||
"failureReason": error.transport_failure_reason().unwrap_or_else(|| observation.exit_reason()),
|
||||
"connectionId": observation.connection_id().to_string(),
|
||||
"connectionAgeMs": observation.age_ms(),
|
||||
"connectionIdleMs": observation.idle_ms(),
|
||||
"reused": reused_connection,
|
||||
"error": diagnostic_json(&json!({"message": error.to_string()})),
|
||||
}),
|
||||
);
|
||||
let observation = connection_observation(&websocket, &error);
|
||||
discard_stream_websocket(
|
||||
websocket,
|
||||
pool_return,
|
||||
@@ -323,6 +331,7 @@ async fn forward_websocket_response_stream(state: WebSocketStreamForwardState) {
|
||||
);
|
||||
match terminal {
|
||||
WebSocketTerminalKind::Completed => {
|
||||
metadata.response_metadata = opening_response_metadata;
|
||||
finish_stream_websocket(websocket, metadata, continuation, pool_return.take())
|
||||
.await;
|
||||
}
|
||||
|
||||
@@ -36,7 +36,8 @@ const DEFAULT_MAX_CONNECTING: usize = 16;
|
||||
const DEFAULT_MAX_AGE: Duration = Duration::from_mins(55);
|
||||
const DEFAULT_MAINTENANCE_INTERVAL: Duration = Duration::from_secs(25);
|
||||
const DEFAULT_PING_INTERVAL: Duration = Duration::from_secs(25);
|
||||
const DEFAULT_PING_TIMEOUT: Duration = Duration::from_secs(5);
|
||||
// 心跳也覆盖正在生成的连接,给短时链路停顿留出恢复余量。
|
||||
const DEFAULT_PING_TIMEOUT: Duration = Duration::from_secs(30);
|
||||
pub(crate) const DEFAULT_STREAM_IDLE_TIMEOUT: Duration = Duration::from_secs(300);
|
||||
|
||||
/// WebSocket 连接池。
|
||||
@@ -67,9 +68,9 @@ pub struct CodexWebSocketPoolConfig {
|
||||
pub max_connecting: usize,
|
||||
/// 后台维护间隔;`None` 表示不启动后台任务。
|
||||
pub maintenance_interval: Option<Duration>,
|
||||
/// idle socket 探活 ping 间隔;`None` 表示维护时只做过期清理。
|
||||
/// 池化连接的探活 ping 间隔(包括正在生成的连接);`None` 表示不主动 ping。
|
||||
pub ping_interval: Option<Duration>,
|
||||
/// 发送 ping 后等待上游响应的超时时间;零值表示不校验 Pong deadline。
|
||||
/// 发送 ping 后等待任意入站帧的超时时间;零值表示不校验 Pong deadline。
|
||||
pub ping_timeout: Duration,
|
||||
/// idle socket 无活动多久后视为失活。
|
||||
pub liveness_timeout: Option<Duration>,
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
//!
|
||||
//! 每条上游 WebSocket 连接都由一个后台 pump 任务独占:
|
||||
//! - 持续读取 socket:一旦观察到 `Close` / EOF / 传输错误,立即把连接标记为 `closed`。
|
||||
//! - 自动回应上游 `Ping`,并按 `ping_interval` 主动 `Ping`;匹配的 `Pong` 必须在 deadline 内返回。
|
||||
//! - 自动回应上游 `Ping`,并按 `ping_interval` 主动 `Ping`;收到任意入站帧即解除本次心跳 deadline。
|
||||
//! - 可选 `liveness_timeout`:长时间无任何入站活动时判定连接失活并退出。
|
||||
//!
|
||||
//! 因此空闲连接的“死没死”在后台被实时感知;复用方只需零成本读取 [`PumpedWebSocket::is_closed`],
|
||||
@@ -57,7 +57,7 @@ impl PumpLogContext {
|
||||
pub(crate) struct PumpKeepalive {
|
||||
/// 主动 `Ping` 间隔;`None` 表示 pump 不主动 ping(仅被动读取 + 回应上游 ping)。
|
||||
pub(crate) ping_interval: Option<Duration>,
|
||||
/// 主动 `Ping` 发出后等待匹配 `Pong` 的 deadline;`None` 表示不校验响应。
|
||||
/// 主动 `Ping` 发出后等待任意入站帧的 deadline;`None` 表示不校验响应。
|
||||
pub(crate) ping_timeout: Option<Duration>,
|
||||
/// 无入站活动多久后判定失活;`None` 表示只靠显式 close/传输错误发现死亡。
|
||||
pub(crate) liveness_timeout: Option<Duration>,
|
||||
|
||||
@@ -74,7 +74,7 @@ async fn openai_bundle_exposes_one_core_provider_and_drains_worker_contributions
|
||||
assert_eq!(bundle.core_provider().name(), "openai");
|
||||
assert_eq!(bundle.admin_provider().provider_kind().as_str(), "openai");
|
||||
let contributions = bundle.take_worker_contributions();
|
||||
assert_eq!(contributions.len(), 4);
|
||||
assert_eq!(contributions.len(), 5);
|
||||
assert!(
|
||||
contributions
|
||||
.iter()
|
||||
@@ -101,6 +101,30 @@ async fn openai_bundle_exposes_one_core_provider_and_drains_worker_contributions
|
||||
panic!("Desktop release worker must be scheduled");
|
||||
};
|
||||
assert_eq!(schedule.interval(), APPCAST_POLL_INTERVAL);
|
||||
for (owner, interval) in [
|
||||
("openai", Duration::from_secs(30)),
|
||||
(
|
||||
"openai-model-catalog",
|
||||
config.config.quota_refresh_policy().interval(),
|
||||
),
|
||||
] {
|
||||
let schedule = contributions
|
||||
.iter()
|
||||
.find_map(|contribution| match contribution {
|
||||
WorkerContribution::Registration(registration)
|
||||
if registration.id.kind() == WorkerKind::QuotaCatalogHealth
|
||||
&& registration.id.owner() == owner =>
|
||||
{
|
||||
match ®istration.runnable {
|
||||
WorkerRunnable::Scheduled { schedule, .. } => Some(schedule),
|
||||
WorkerRunnable::Daemon { .. } => None,
|
||||
}
|
||||
}
|
||||
_ => None,
|
||||
})
|
||||
.expect("quota/catalog scheduled worker");
|
||||
assert_eq!(schedule.interval(), interval, "{owner}");
|
||||
}
|
||||
assert!(contributions.iter().any(|contribution| {
|
||||
matches!(
|
||||
contribution,
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
//! OpenAI 额度事实边界与展示快照回归。
|
||||
|
||||
mod recovery;
|
||||
mod refresh_timing;
|
||||
mod scheduling;
|
||||
mod slots;
|
||||
mod snapshot;
|
||||
|
||||
@@ -0,0 +1,130 @@
|
||||
//! 通过 quota 服务验证周期复核与 reset 宽限期,不依赖内部调度状态。
|
||||
|
||||
use std::time::Duration;
|
||||
|
||||
use super::*;
|
||||
|
||||
async fn mount_usage(server: &MockServer, value: serde_json::Value) {
|
||||
server.reset().await;
|
||||
Mock::given(method("GET"))
|
||||
.and(path("/api/codex/usage"))
|
||||
.respond_with(ResponseTemplate::new(200).set_body_json(value))
|
||||
.mount(server)
|
||||
.await;
|
||||
}
|
||||
|
||||
async fn wait_for_reset_grace(reset: i64) {
|
||||
let due = SystemTime::UNIX_EPOCH + Duration::from_secs((reset + 120) as u64);
|
||||
if let Ok(remaining) = due.duration_since(SystemTime::now()) {
|
||||
tokio::time::sleep(remaining + Duration::from_millis(50)).await;
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn reset_grace_bypasses_periodic_throttle_once_then_allows_the_next_window() {
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
create_account(&store, "acct_quota_timing").await;
|
||||
let account = store.account("acct_quota_timing").expect("account");
|
||||
let server = MockServer::start().await;
|
||||
let service = quota_service_with_base_url(&store, reqwest::Client::new(), server.uri());
|
||||
// 使用接近宽限期终点的真实窗口,在数秒内走完两个窗口的调度与落库路径。
|
||||
let short_reset = Utc::now().timestamp() - 115;
|
||||
let week_reset = short_reset + 3;
|
||||
let usage = |short_used, short_reset, week_used, week_reset| {
|
||||
json!({"rate_limit": {
|
||||
"allowed": false,
|
||||
"primary_window": {
|
||||
"used_percent": short_used, "reset_at": short_reset, "limit_window_seconds": 18_000,
|
||||
},
|
||||
"secondary_window": {
|
||||
"used_percent": week_used, "reset_at": week_reset, "limit_window_seconds": 604_800,
|
||||
},
|
||||
}})
|
||||
};
|
||||
mount_usage(&server, usage(100, short_reset, 100, week_reset)).await;
|
||||
service
|
||||
.refresh_account(account.id())
|
||||
.await
|
||||
.expect("seed exhaustion");
|
||||
service.synchronize().await.expect("initial periodic check");
|
||||
let requests = server.received_requests().await.expect("requests").len();
|
||||
|
||||
assert!(
|
||||
Utc::now().timestamp() < short_reset + 120,
|
||||
"fixture must precede grace deadline"
|
||||
);
|
||||
service.synchronize().await.expect("before grace deadline");
|
||||
assert_eq!(
|
||||
server.received_requests().await.expect("requests").len(),
|
||||
requests
|
||||
);
|
||||
|
||||
mount_usage(&server, usage(0, short_reset + 18_000, 100, week_reset)).await;
|
||||
wait_for_reset_grace(short_reset).await;
|
||||
let partial = service.synchronize().await.expect("short reset check");
|
||||
assert_eq!(partial.exhausted, 1);
|
||||
assert_eq!(
|
||||
store
|
||||
.account("acct_quota_timing")
|
||||
.expect("account")
|
||||
.quota()
|
||||
.reset_at(),
|
||||
Some(SystemTime::UNIX_EPOCH + Duration::from_secs(week_reset as u64))
|
||||
);
|
||||
service
|
||||
.synchronize()
|
||||
.await
|
||||
.expect("next scan before weekly grace deadline");
|
||||
assert_eq!(server.received_requests().await.expect("requests").len(), 1);
|
||||
|
||||
// 周窗口的到期复核仍未恢复时,也不能每轮扫描重复请求。
|
||||
wait_for_reset_grace(week_reset).await;
|
||||
assert_eq!(
|
||||
service
|
||||
.synchronize()
|
||||
.await
|
||||
.expect("weekly reset check")
|
||||
.exhausted,
|
||||
1
|
||||
);
|
||||
service
|
||||
.synchronize()
|
||||
.await
|
||||
.expect("repeat scan after weekly check");
|
||||
assert_eq!(server.received_requests().await.expect("requests").len(), 2);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn periodic_checks_continue_when_reset_is_unknown_or_far_in_the_future() {
|
||||
for reset in [None, Some(Utc::now().timestamp() + 604_800)] {
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
create_account(&store, "acct_quota_periodic").await;
|
||||
let account = store.account("acct_quota_periodic").expect("account");
|
||||
let server = MockServer::start().await;
|
||||
let service = quota_service_with_base_url(&store, reqwest::Client::new(), server.uri());
|
||||
let mut usage = json!({"rate_limit": {
|
||||
"allowed": false,
|
||||
"primary_window": {"used_percent": 100, "limit_window_seconds": 604_800},
|
||||
}});
|
||||
if let Some(reset) = reset {
|
||||
usage["rate_limit"]["primary_window"]["reset_at"] = json!(reset);
|
||||
}
|
||||
mount_usage(&server, usage.clone()).await;
|
||||
service
|
||||
.refresh_account(account.id())
|
||||
.await
|
||||
.expect("seed exhaustion");
|
||||
mount_usage(&server, usage).await;
|
||||
|
||||
assert_eq!(
|
||||
service
|
||||
.synchronize()
|
||||
.await
|
||||
.expect("periodic check")
|
||||
.exhausted,
|
||||
1
|
||||
);
|
||||
service.synchronize().await.expect("throttled repeat check");
|
||||
assert_eq!(server.received_requests().await.expect("requests").len(), 1);
|
||||
}
|
||||
}
|
||||
@@ -172,6 +172,99 @@ async fn recovery_checks_each_exhausted_slot_in_worker_and_manual_refresh() {
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn worker_detects_early_resets_without_unlocking_other_exhausted_windows() {
|
||||
let short_reset = Utc::now().timestamp() + 18_000;
|
||||
let week_reset = Utc::now().timestamp() + 604_800;
|
||||
for (name, old_short, old_week, new_short, new_week, recovered) in [
|
||||
("short_only", Some(100), None, Some(0), None, true),
|
||||
("weekly_only", None, Some(100), None, Some(0), true),
|
||||
(
|
||||
"short_with_weekly_available",
|
||||
Some(100),
|
||||
Some(45),
|
||||
Some(0),
|
||||
Some(45),
|
||||
true,
|
||||
),
|
||||
(
|
||||
"weekly_still_exhausted",
|
||||
Some(100),
|
||||
Some(100),
|
||||
Some(0),
|
||||
Some(100),
|
||||
false,
|
||||
),
|
||||
] {
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
let account_id = format!("acct_early_{name}");
|
||||
create_account(&store, &account_id).await;
|
||||
let account = store.account(&account_id).expect("account");
|
||||
let server = MockServer::start().await;
|
||||
let service = quota_service_with_base_url(&store, reqwest::Client::new(), server.uri());
|
||||
let window = |used: Option<u64>, reset, seconds| {
|
||||
used.map(|used| {
|
||||
json!({
|
||||
"used_percent": used,
|
||||
"reset_at": reset,
|
||||
"limit_window_seconds": seconds,
|
||||
})
|
||||
})
|
||||
};
|
||||
mount_usage(
|
||||
&server,
|
||||
json!({"rate_limit": {
|
||||
"allowed": false,
|
||||
"primary_window": window(old_short, short_reset, 18_000),
|
||||
"secondary_window": window(old_week, week_reset, 604_800),
|
||||
}}),
|
||||
)
|
||||
.await;
|
||||
let baseline = service
|
||||
.refresh_account(account.id())
|
||||
.await
|
||||
.expect("seed exhaustion");
|
||||
assert!(baseline.quota().reset_at().expect("future reset") > SystemTime::now());
|
||||
|
||||
mount_usage(
|
||||
&server,
|
||||
json!({"rate_limit": {
|
||||
"allowed": recovered,
|
||||
"primary_window": window(new_short, short_reset + 18_000, 18_000),
|
||||
"secondary_window": window(new_week, if new_week == Some(0) {
|
||||
week_reset + 604_800
|
||||
} else {
|
||||
week_reset
|
||||
}, 604_800),
|
||||
}}),
|
||||
)
|
||||
.await;
|
||||
let summary = service
|
||||
.synchronize()
|
||||
.await
|
||||
.expect("periodic early reset check");
|
||||
assert_eq!(summary.updated, u64::from(recovered), "{name}");
|
||||
assert_eq!(summary.exhausted, u64::from(!recovered), "{name}");
|
||||
assert_eq!(
|
||||
store
|
||||
.account(&account_id)
|
||||
.expect("updated account")
|
||||
.quota()
|
||||
.is_exhausted(),
|
||||
!recovered,
|
||||
"{name}"
|
||||
);
|
||||
|
||||
// 未恢复的周窗口仍在未来,也必须保留复核节流;恢复后则退出定时复核。
|
||||
service.synchronize().await.expect("next worker cycle");
|
||||
assert_eq!(
|
||||
server.received_requests().await.expect("requests").len(),
|
||||
1,
|
||||
"{name}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn partial_recovery_survives_restart_missing_windows_and_changed_roles() {
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
|
||||
@@ -26,6 +26,7 @@ use gateway_core::error::{
|
||||
};
|
||||
use gateway_core::event::{GatewayEvent, WebSocketPoolKind};
|
||||
use gateway_core::lifecycle::CancellationToken;
|
||||
use gateway_core::metering::Usage;
|
||||
use gateway_core::operation::{
|
||||
CapabilityRequirements, GenerateRequest, ImageRequest, ImageRequestKind, Operation,
|
||||
OperationKind, ProtocolPayload, ProviderSessionState, RawJsonPayload, StandaloneSearchRequest,
|
||||
@@ -422,6 +423,7 @@ fn planned_provider_endpoint_request(provider_name: &str, operation: Operation)
|
||||
fn contract_account_scope() -> Arc<FrozenAccountScope> {
|
||||
let provider = ProviderKind::new("openai").expect("provider");
|
||||
let accounts = [
|
||||
"acct_abrupt_disconnect",
|
||||
"acct_affinity",
|
||||
"acct_affinity_switch_a",
|
||||
"acct_affinity_switch_b",
|
||||
@@ -912,6 +914,11 @@ async fn truncated_http_stream_allows_account_rotation_only_before_client_delive
|
||||
|
||||
assert_eq!(error.kind(), ProviderErrorKind::Transport);
|
||||
assert_eq!(error.send_state(), UpstreamSendState::Sent);
|
||||
assert_eq!(
|
||||
error.diagnostic().and_then(|diagnostic| diagnostic.code()),
|
||||
Some("unexpected_eof")
|
||||
);
|
||||
assert!(!error.diagnostic().unwrap().as_str().contains("127.0.0.1"));
|
||||
assert!(error.allows_pre_delivery_retry());
|
||||
assert!(!error.replay_is_safe());
|
||||
}
|
||||
@@ -994,6 +1001,18 @@ async fn image_endpoints_bypass_only_the_text_catalog_and_preserve_the_current_c
|
||||
br#"{"model":"gpt-image-2","images":[{"image_url":"data:image/png;base64,AAEC"}],"prompt":"add fog","prompt":"duplicate remains opaque"}"#.as_slice(),
|
||||
br#"{"created":1787212801,"data":[{"b64_json":"AwQF"}],"quality":"high"}"#.as_slice(),
|
||||
),
|
||||
(
|
||||
ImageRequestKind::Generation,
|
||||
"/codex/images/generations",
|
||||
br#"{"model":"gpt-image-2.5-flare","prompt":"a lighthouse","quality":"xhigh"}"#.as_slice(),
|
||||
br#"{"created":1788900000,"data":[{"b64_json":"AAEC"}],"quality":"xhigh"}"#.as_slice(),
|
||||
),
|
||||
(
|
||||
ImageRequestKind::Edit,
|
||||
"/codex/images/edits",
|
||||
br#"{"model":"gpt-image-2.5-sunburst","images":[{"image_url":"data:image/png;base64,AAEC"}],"prompt":"add fog","quality":"max"}"#.as_slice(),
|
||||
br#"{"created":1788900001,"data":[{"b64_json":"AwQF"}],"quality":"max"}"#.as_slice(),
|
||||
),
|
||||
];
|
||||
for (_, endpoint, body, response_body) in &cases {
|
||||
Mock::given(method("POST"))
|
||||
@@ -1051,6 +1070,13 @@ async fn image_endpoints_bypass_only_the_text_catalog_and_preserve_the_current_c
|
||||
let mut observed_http_json = false;
|
||||
while let Some(event) = stream.next().await {
|
||||
let event = event.expect("image provider event");
|
||||
assert!(
|
||||
event
|
||||
.canonical_facts()
|
||||
.iter()
|
||||
.all(|fact| !matches!(fact, GatewayEvent::Usage(_))),
|
||||
"missing upstream usage must remain unknown"
|
||||
);
|
||||
completed |= event
|
||||
.canonical_facts()
|
||||
.iter()
|
||||
@@ -1075,6 +1101,300 @@ async fn image_endpoints_bypass_only_the_text_catalog_and_preserve_the_current_c
|
||||
server.verify().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn image_endpoints_should_report_usage_before_delivering_the_unchanged_body() {
|
||||
let response = br#"{ "created":1778832973,"data":[{"b64_json":"AAEC"}],"quality":"medium","size":"1024x1536","usage":{"input_tokens":1474,"input_tokens_details":{"image_tokens":1457,"text_tokens":17,"cached_tokens":100},"output_tokens":1372,"output_tokens_details":{"image_tokens":1372,"text_tokens":0},"total_tokens":2846},"future":9007199254740993 }"#;
|
||||
let expected = Usage {
|
||||
input_tokens: Some(1474),
|
||||
output_tokens: Some(1372),
|
||||
cached_tokens: Some(100),
|
||||
image_input_tokens: Some(1457),
|
||||
image_output_tokens: Some(1372),
|
||||
total_tokens: Some(2846),
|
||||
..Usage::default()
|
||||
};
|
||||
for kind in [ImageRequestKind::Generation, ImageRequestKind::Edit] {
|
||||
let events = image_usage_events(kind, response).await;
|
||||
let usage = events
|
||||
.iter()
|
||||
.flat_map(|event| event.canonical_facts())
|
||||
.filter_map(|fact| match fact {
|
||||
GatewayEvent::Usage(usage) => Some(usage.clone()),
|
||||
_ => None,
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
assert_eq!(usage.as_slice(), std::slice::from_ref(&expected));
|
||||
let usage_index = events
|
||||
.iter()
|
||||
.position(|event| {
|
||||
event
|
||||
.canonical_facts()
|
||||
.iter()
|
||||
.any(|fact| matches!(fact, GatewayEvent::Usage(_)))
|
||||
})
|
||||
.expect("usage event");
|
||||
let wire_index = events
|
||||
.iter()
|
||||
.position(|event| {
|
||||
event
|
||||
.wire_event()
|
||||
.and_then(|wire| wire.raw_json_body())
|
||||
.is_some()
|
||||
})
|
||||
.expect("raw image response");
|
||||
assert!(
|
||||
usage_index < wire_index,
|
||||
"usage must be observed before image delivery"
|
||||
);
|
||||
assert_eq!(
|
||||
events[wire_index]
|
||||
.wire_event()
|
||||
.and_then(|wire| wire.raw_json_body())
|
||||
.map(|body| body.as_ref()),
|
||||
Some(response.as_slice())
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn image_usage_should_preserve_unknown_fields_and_explicit_zero_counts() {
|
||||
let cases = [
|
||||
(json!(null), None),
|
||||
(json!({}), None),
|
||||
(json!([]), None),
|
||||
(
|
||||
json!({"input_tokens": -1, "output_tokens": "42", "total_tokens": 1.5}),
|
||||
None,
|
||||
),
|
||||
(
|
||||
json!({"input_tokens": 0, "output_tokens": 0, "total_tokens": 0, "input_tokens_details": {"image_tokens": 0, "cached_tokens": 0}, "output_tokens_details": {"image_tokens": 0}}),
|
||||
Some(Usage {
|
||||
input_tokens: Some(0),
|
||||
output_tokens: Some(0),
|
||||
total_tokens: Some(0),
|
||||
cached_tokens: Some(0),
|
||||
image_input_tokens: Some(0),
|
||||
image_output_tokens: Some(0),
|
||||
..Usage::default()
|
||||
}),
|
||||
),
|
||||
(
|
||||
json!({"input_tokens": 17, "output_tokens": 1372, "input_tokens_details": {"image_tokens": "invalid"}, "output_tokens_details": {"image_tokens": 1372}}),
|
||||
Some(Usage {
|
||||
input_tokens: Some(17),
|
||||
output_tokens: Some(1372),
|
||||
image_output_tokens: Some(1372),
|
||||
..Usage::default()
|
||||
}),
|
||||
),
|
||||
];
|
||||
for (usage, expected) in cases {
|
||||
let response = serde_json::to_vec(
|
||||
&json!({"created":1778832973,"data":[{"b64_json":"AAEC"}],"usage":usage}),
|
||||
)
|
||||
.expect("image response JSON");
|
||||
let events = image_usage_events(ImageRequestKind::Generation, &response).await;
|
||||
let actual = events
|
||||
.iter()
|
||||
.flat_map(|event| event.canonical_facts())
|
||||
.filter_map(|fact| match fact {
|
||||
GatewayEvent::Usage(usage) => Some(usage.clone()),
|
||||
_ => None,
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
assert_eq!(actual, expected.into_iter().collect::<Vec<_>>());
|
||||
let raw = events
|
||||
.iter()
|
||||
.find_map(|event| event.wire_event().and_then(|wire| wire.raw_json_body()));
|
||||
assert_eq!(raw.map(|body| body.as_ref()), Some(response.as_slice()));
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn image_prices_should_use_modality_rates_and_precede_delivery() {
|
||||
// First case is the real gpt-image-2 response verified on 2026-09-08.
|
||||
let cases = [
|
||||
(18, 0, 229, None, 69_600_000_u128),
|
||||
(17, 1457, 1372, None, 529_010_000),
|
||||
(100, 0, 10, Some(40), 6_500_000),
|
||||
(0, 100, 10, Some(40), 8_600_000),
|
||||
(20, 80, 10, Some(100), 4_850_000),
|
||||
(0, 0, 0, Some(0), 0),
|
||||
];
|
||||
for (text, image, output, cached, expected_ticks) in cases {
|
||||
let mut usage = json!({
|
||||
"input_tokens":text+image,
|
||||
"input_tokens_details":{"text_tokens":text,"image_tokens":image},
|
||||
"output_tokens":output,
|
||||
"output_tokens_details":{"text_tokens":0,"image_tokens":output},
|
||||
"total_tokens":text+image+output,
|
||||
});
|
||||
if let Some(cached) = cached {
|
||||
usage["input_tokens_details"]["cached_tokens"] = json!(cached);
|
||||
}
|
||||
let response = serde_json::to_vec(&json!({"data":[{"b64_json":"AAEC"}],"usage":usage}))
|
||||
.expect("response");
|
||||
for (kind, model) in [
|
||||
(ImageRequestKind::Generation, "gpt-image-2"),
|
||||
(ImageRequestKind::Edit, "gpt-image-2-2026-04-21"),
|
||||
(ImageRequestKind::Generation, "gpt-image-2.5-sunburst"),
|
||||
(ImageRequestKind::Edit, "gpt-image-2.5-sunburst-2026-09-08"),
|
||||
(ImageRequestKind::Generation, "gpt-image-2.5-flare"),
|
||||
(ImageRequestKind::Edit, "gpt-image-2.5-flare-2026-09-08"),
|
||||
] {
|
||||
let events = image_metering_events(kind, &response, model).await;
|
||||
let costs = events
|
||||
.iter()
|
||||
.enumerate()
|
||||
.flat_map(|(index, event)| {
|
||||
event
|
||||
.canonical_facts()
|
||||
.iter()
|
||||
.filter_map(move |fact| match fact {
|
||||
GatewayEvent::CalculatedCost(cost) => Some((index, *cost)),
|
||||
_ => None,
|
||||
})
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
assert_eq!(costs.len(), 1, "model={model}");
|
||||
let (cost_index, cost) = costs[0];
|
||||
assert_eq!(
|
||||
cost.total().amount().scaled(),
|
||||
expected_ticks,
|
||||
"model={model}"
|
||||
);
|
||||
assert_eq!(cost.into_estimate().source().as_str(), "calculated");
|
||||
let wire_index = events
|
||||
.iter()
|
||||
.position(|event| event.wire_event().is_some())
|
||||
.expect("wire event");
|
||||
assert!(cost_index < wire_index);
|
||||
assert_eq!(
|
||||
events[wire_index]
|
||||
.wire_event()
|
||||
.and_then(|wire| wire.raw_json_body())
|
||||
.map(|body| body.as_ref()),
|
||||
Some(response.as_slice())
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn image_prices_should_remain_unknown_when_modality_or_model_is_uncertain() {
|
||||
let valid = json!({
|
||||
"input_tokens":100,
|
||||
"input_tokens_details":{"text_tokens":20,"image_tokens":80},
|
||||
"output_tokens":10,
|
||||
"output_tokens_details":{"text_tokens":0,"image_tokens":10},
|
||||
"total_tokens":110,
|
||||
});
|
||||
let mut cases = vec![
|
||||
("gpt-image-future", valid.clone()),
|
||||
("gpt-image-2-future", valid.clone()),
|
||||
("gpt-image-2.5", valid.clone()),
|
||||
("gpt-image-2.5-sunburst-future", valid.clone()),
|
||||
("gpt-image-2.5-flare-future", valid.clone()),
|
||||
("gpt-image-2", json!(null)),
|
||||
(
|
||||
"gpt-image-2",
|
||||
json!({"input_tokens":100,"output_tokens":10}),
|
||||
),
|
||||
];
|
||||
for (pointer, value) in [
|
||||
("/input_tokens", json!(101)),
|
||||
("/input_tokens_details/text_tokens", json!(null)),
|
||||
("/input_tokens_details/image_tokens", json!("80")),
|
||||
("/output_tokens_details/text_tokens", json!(1)),
|
||||
("/output_tokens_details/image_tokens", json!(9)),
|
||||
("/output_tokens", json!(-1)),
|
||||
("/total_tokens", json!(111)),
|
||||
] {
|
||||
let mut usage = valid.clone();
|
||||
*usage.pointer_mut(pointer).expect("existing field") = value;
|
||||
cases.push(("gpt-image-2", usage));
|
||||
}
|
||||
for cached in [json!(40), json!(101), json!(null), json!("40")] {
|
||||
let mut usage = valid.clone();
|
||||
usage["input_tokens_details"]["cached_tokens"] = cached;
|
||||
cases.push(("gpt-image-2", usage));
|
||||
}
|
||||
cases.push(("gpt-image-2", json!({
|
||||
"input_tokens":u64::MAX,"input_tokens_details":{"text_tokens":u64::MAX,"image_tokens":0},
|
||||
"output_tokens":10,
|
||||
})));
|
||||
for (model, usage) in cases {
|
||||
let response = serde_json::to_vec(&json!({"data":[{"b64_json":"AAEC"}],"usage":usage}))
|
||||
.expect("response");
|
||||
let events = image_metering_events(ImageRequestKind::Generation, &response, model).await;
|
||||
assert!(
|
||||
events
|
||||
.iter()
|
||||
.flat_map(|event| event.canonical_facts())
|
||||
.all(|fact| !matches!(fact, GatewayEvent::CalculatedCost(_)))
|
||||
);
|
||||
assert_eq!(
|
||||
events
|
||||
.iter()
|
||||
.find_map(|event| event.wire_event().and_then(|wire| wire.raw_json_body()))
|
||||
.map(|body| body.as_ref()),
|
||||
Some(response.as_slice())
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
async fn image_usage_events(
|
||||
kind: ImageRequestKind,
|
||||
response: &[u8],
|
||||
) -> Vec<gateway_core::event::ProviderEvent> {
|
||||
image_metering_events(kind, response, "gpt-image-2").await
|
||||
}
|
||||
|
||||
async fn image_metering_events(
|
||||
kind: ImageRequestKind,
|
||||
response: &[u8],
|
||||
model: &str,
|
||||
) -> Vec<gateway_core::event::ProviderEvent> {
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
create_account(&store, "acct_provider_contract").await;
|
||||
let server = MockServer::start().await;
|
||||
let endpoint = match kind {
|
||||
ImageRequestKind::Generation => "/codex/images/generations",
|
||||
ImageRequestKind::Edit => "/codex/images/edits",
|
||||
};
|
||||
Mock::given(method("POST"))
|
||||
.and(path(endpoint))
|
||||
.respond_with(
|
||||
ResponseTemplate::new(200).set_body_raw(response.to_vec(), "application/json"),
|
||||
)
|
||||
.expect(1)
|
||||
.mount(&server)
|
||||
.await;
|
||||
let provider = provider_with_base_url(&store, server.uri());
|
||||
let payload = RawJsonPayload::new(
|
||||
"openai",
|
||||
Bytes::from(
|
||||
serde_json::to_vec(&json!({"model":model,"prompt":"draw a tree"}))
|
||||
.expect("request JSON"),
|
||||
),
|
||||
)
|
||||
.expect("image payload");
|
||||
let operation = Operation::GenerateImage(ImageRequest::from_raw_json(kind, payload));
|
||||
let mut stream = provider
|
||||
.execute(
|
||||
planned_provider_endpoint_request("openai", operation),
|
||||
context("req_image_usage", CancellationToken::new()),
|
||||
)
|
||||
.await
|
||||
.expect("prepare image stream");
|
||||
let mut events = Vec::new();
|
||||
while let Some(event) = stream.next().await {
|
||||
events.push(event.expect("image provider event"));
|
||||
}
|
||||
server.verify().await;
|
||||
events
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn image_endpoint_returns_the_exact_upstream_error_response() {
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
@@ -1920,6 +2240,168 @@ async fn websocket_fast_path_miss_uses_http_and_keeps_background_preconnect() {
|
||||
server.await.expect("HTTP and WebSocket upstream server");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn connection_failure_preserves_io_cause_without_claiming_payload_was_sent() {
|
||||
for websocket in [false, true] {
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
create_account(&store, "acct_abrupt_disconnect").await;
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let base_url = format!("http://{}", listener.local_addr().unwrap());
|
||||
drop(listener);
|
||||
let operation = if websocket {
|
||||
Operation::Generate(generate_with_persisted_session_context(
|
||||
"acct_abrupt_disconnect",
|
||||
"conversation-refused",
|
||||
"session-refused",
|
||||
"turn-refused",
|
||||
))
|
||||
} else {
|
||||
http_generate_operation()
|
||||
};
|
||||
let mut stream = provider_with_base_url(&store, base_url)
|
||||
.execute(
|
||||
planned_request("openai", operation),
|
||||
context("req_connect_refused", CancellationToken::new()),
|
||||
)
|
||||
.await
|
||||
.expect("prepare provider stream");
|
||||
let error = loop {
|
||||
match stream.next().await {
|
||||
Some(Ok(_)) => {}
|
||||
Some(Err(error)) => break error,
|
||||
None => panic!("closed listener must fail"),
|
||||
}
|
||||
};
|
||||
let diagnostic = error.diagnostic().expect("connection diagnosis");
|
||||
assert_eq!(diagnostic.code(), Some("connection_refused"));
|
||||
assert_eq!(diagnostic.stage(), Some("connect"));
|
||||
assert_eq!(error.send_state(), UpstreamSendState::NotSent);
|
||||
assert!(!diagnostic.as_str().contains("after payload send"));
|
||||
assert!(!diagnostic.as_str().contains("127.0.0.1"));
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn abrupt_websocket_disconnect_preserves_diagnosis_and_ambiguous_send_state() {
|
||||
const ACCOUNT_ID: &str = "acct_abrupt_disconnect";
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
create_account(&store, ACCOUNT_ID).await;
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let base_url = format!("http://{}", listener.local_addr().unwrap());
|
||||
let server = tokio::spawn(async move {
|
||||
let (stream, _) = listener.accept().await.unwrap();
|
||||
let mut websocket = accept_codex_test_websocket(stream).await;
|
||||
websocket.next().await.unwrap().unwrap();
|
||||
// The peer disappears after receiving the payload, without sending a Close frame.
|
||||
});
|
||||
let operation = Operation::Generate(generate_with_persisted_session_context(
|
||||
ACCOUNT_ID,
|
||||
"conversation-abrupt-disconnect",
|
||||
"session-abrupt-disconnect",
|
||||
"turn-abrupt-disconnect",
|
||||
));
|
||||
let mut stream = provider_with_base_url(&store, base_url)
|
||||
.execute(
|
||||
planned_request("openai", operation),
|
||||
context("req_abrupt_disconnect", CancellationToken::new()),
|
||||
)
|
||||
.await
|
||||
.expect("prepare WebSocket provider stream");
|
||||
let error = loop {
|
||||
match stream.next().await {
|
||||
Some(Ok(_)) => {}
|
||||
Some(Err(error)) => break error,
|
||||
None => panic!("abrupt disconnect must surface a provider error"),
|
||||
}
|
||||
};
|
||||
server.await.unwrap();
|
||||
assert_eq!(error.kind(), ProviderErrorKind::Transport);
|
||||
assert_eq!(error.send_state(), UpstreamSendState::Ambiguous);
|
||||
assert!(!error.replay_is_safe());
|
||||
assert_eq!(error.pre_delivery_retry(), None);
|
||||
assert_eq!(
|
||||
error.diagnostic().map(|diagnostic| diagnostic.as_str()),
|
||||
Some(
|
||||
"OpenAI WebSocket disconnected without a closing handshake after payload send; result is ambiguous"
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn websocket_idle_timeout_diagnosis_survives_ambiguous_send_wrapping() {
|
||||
const ACCOUNT_ID: &str = "acct_abrupt_disconnect";
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
create_account(&store, ACCOUNT_ID).await;
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let base_url = format!("http://{}", listener.local_addr().unwrap());
|
||||
let server =
|
||||
tokio::spawn(async move {
|
||||
let (stream, _) = listener.accept().await.unwrap();
|
||||
let mut websocket = accept_codex_test_websocket(stream).await;
|
||||
websocket.next().await.unwrap().unwrap();
|
||||
websocket.send(Message::Text(json!({
|
||||
"type": "response.created", "response": {"id": "resp_idle", "model": "gpt-5.4"},
|
||||
}).to_string().into())).await.unwrap();
|
||||
websocket
|
||||
.send(Message::Text(
|
||||
json!({"type": "response.output_text.delta", "delta": "partial"})
|
||||
.to_string()
|
||||
.into(),
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
futures::future::pending::<()>().await;
|
||||
});
|
||||
let operation = Operation::Generate(generate_with_persisted_session_context(
|
||||
ACCOUNT_ID,
|
||||
"conversation-idle-timeout",
|
||||
"session-idle-timeout",
|
||||
"turn-idle-timeout",
|
||||
));
|
||||
let mut stream = provider_with_base_url(&store, base_url)
|
||||
.execute(
|
||||
planned_request("openai", operation),
|
||||
context("req_idle_timeout", CancellationToken::new()),
|
||||
)
|
||||
.await
|
||||
.expect("prepare WebSocket provider stream");
|
||||
loop {
|
||||
let event = stream
|
||||
.next()
|
||||
.await
|
||||
.expect("partial response")
|
||||
.expect("valid partial response");
|
||||
if event.has_client_event() {
|
||||
break;
|
||||
}
|
||||
}
|
||||
tokio::time::pause();
|
||||
tokio::time::advance(Duration::from_secs(300)).await;
|
||||
let error = loop {
|
||||
match stream.next().await {
|
||||
Some(Ok(_)) => {}
|
||||
Some(Err(error)) => break error,
|
||||
None => panic!("idle timeout must surface a provider error"),
|
||||
}
|
||||
};
|
||||
server.abort();
|
||||
assert_eq!(error.send_state(), UpstreamSendState::Ambiguous);
|
||||
assert!(!error.replay_is_safe());
|
||||
assert_eq!(error.pre_delivery_retry(), None);
|
||||
assert_eq!(
|
||||
error.diagnostic().and_then(|diagnostic| diagnostic.code()),
|
||||
Some("receive_idle_timeout")
|
||||
);
|
||||
assert_eq!(
|
||||
error.diagnostic().and_then(|diagnostic| diagnostic.stage()),
|
||||
Some("receive")
|
||||
);
|
||||
assert_eq!(
|
||||
error.diagnostic().map(|diagnostic| diagnostic.as_str()),
|
||||
Some("OpenAI WebSocket receive idle timeout after 300s"),
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn ambiguous_websocket_close_reconnects_and_retains_native_continuation() {
|
||||
const ACCOUNT_ID: &str = "acct_websocket_close";
|
||||
@@ -2521,6 +3003,127 @@ async fn fallback_attempt_transport_forces_http_sse_for_a_websocket_request() {
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn downstream_websocket_new_chain_should_override_session_and_attempt_http_fallback() {
|
||||
const SESSION_ID: &str = "downstream-required-session";
|
||||
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
create_account(&store, "acct_http_sse_exhausted").await;
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let base_url = format!("http://{}", listener.local_addr().unwrap());
|
||||
let server = tokio::spawn(async move {
|
||||
let (mut opening, _) = listener.accept().await.unwrap();
|
||||
let request = capture_http_request(&mut opening).await;
|
||||
assert!(String::from_utf8_lossy(&request).starts_with("GET /codex/responses"));
|
||||
opening
|
||||
.write_all(
|
||||
b"HTTP/1.1 426 Upgrade Required\r\ncontent-length: 0\r\nconnection: close\r\n\r\n",
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let (stream, _) = listener.accept().await.unwrap();
|
||||
let mut websocket = accept_codex_test_websocket(stream).await;
|
||||
for id in ["resp_required_session", "resp_required_attempt"] {
|
||||
let request = websocket.next().await.unwrap().unwrap();
|
||||
let payload: Value = serde_json::from_str(request.to_text().unwrap()).unwrap();
|
||||
assert_eq!(payload["store"], false);
|
||||
websocket
|
||||
.send(Message::Text(
|
||||
json!({
|
||||
"type": "response.completed",
|
||||
"response": {
|
||||
"id": id,
|
||||
"model": "gpt-5.4",
|
||||
"status": "completed",
|
||||
"store": false,
|
||||
"output": [],
|
||||
"usage": {"input_tokens": 1, "output_tokens": 1, "total_tokens": 2}
|
||||
}
|
||||
})
|
||||
.to_string()
|
||||
.into(),
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
});
|
||||
let provider = provider_with_base_url(&store, base_url);
|
||||
let mut seed = provider
|
||||
.execute(
|
||||
planned_request(
|
||||
"openai",
|
||||
Operation::Generate(generate_with_session_context(SESSION_ID, None, None)),
|
||||
),
|
||||
context("req_disable_optional_websocket", CancellationToken::new()),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let error = loop {
|
||||
match seed.next().await {
|
||||
Some(Ok(_)) => {}
|
||||
Some(Err(error)) => break error,
|
||||
None => panic!("426 opening must enable session HTTP fallback"),
|
||||
}
|
||||
};
|
||||
assert_eq!(
|
||||
error.pre_delivery_retry(),
|
||||
Some(PreDeliveryRetry::SameAccountTransportFallback)
|
||||
);
|
||||
drop(seed);
|
||||
|
||||
for attempt in [
|
||||
context("req_required_session", CancellationToken::new()),
|
||||
fallback_transport_context("req_required_attempt"),
|
||||
] {
|
||||
let payload = ProtocolPayload::json_object(
|
||||
"openai",
|
||||
Map::from_iter([
|
||||
("model".to_owned(), json!("gpt-5.4")),
|
||||
("input".to_owned(), json!("hello")),
|
||||
("store".to_owned(), json!(false)),
|
||||
("session_id".to_owned(), json!(SESSION_ID)),
|
||||
]),
|
||||
)
|
||||
.unwrap()
|
||||
.with_context(Map::from_iter([(
|
||||
"downstream_websocket_connection_id".to_owned(),
|
||||
json!("ws_required_session"),
|
||||
)]));
|
||||
let request = GenerateRequest::from_protocol_payload(payload).with_provider_session_state(
|
||||
ProviderSessionState::new(
|
||||
"openai",
|
||||
Map::from_iter([
|
||||
("account_id".to_owned(), json!("acct_http_sse_exhausted")),
|
||||
("conversation_id".to_owned(), json!(SESSION_ID)),
|
||||
("continuation_scope".to_owned(), json!("persisted")),
|
||||
]),
|
||||
)
|
||||
.unwrap(),
|
||||
);
|
||||
let mut stream = provider
|
||||
.execute(
|
||||
planned_request("openai", Operation::Generate(request)),
|
||||
attempt,
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(stream.metadata().transport().as_str(), "websocket");
|
||||
let mut session = None;
|
||||
while let Some(event) = stream.next().await {
|
||||
let event = event.expect("required WebSocket response");
|
||||
if let Some(update) = event.session_update() {
|
||||
session = Some(update.clone());
|
||||
}
|
||||
}
|
||||
assert_eq!(
|
||||
session.unwrap().payload().get("continuation_scope"),
|
||||
Some(&json!("connection_local"))
|
||||
);
|
||||
}
|
||||
server.await.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn websocket_turn_state_metadata_is_exposed_through_response_observation() {
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
@@ -4240,7 +4843,7 @@ async fn completed_response_persists_session_affinity_before_stream_consumer_sto
|
||||
drop(stream);
|
||||
|
||||
assert_eq!(affinity.binding_count(), 1);
|
||||
assert_eq!(observed_service_tier.as_deref(), Some("priority"));
|
||||
assert_eq!(observed_service_tier.as_deref(), Some("default"));
|
||||
assert_eq!(upstream_service_tier.as_deref(), Some("default"));
|
||||
}
|
||||
|
||||
@@ -5510,6 +6113,52 @@ fn request_observation_reads_openai_metadata_without_changing_the_operation() {
|
||||
assert!(matches!(operation, Operation::Generate(_)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn endpoint_observation_should_read_models_without_rewriting_or_requiring_a_catalog() {
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
let provider = provider(&store);
|
||||
let client_key = ClientApiKeyId::new("key_endpoint_model").expect("client key");
|
||||
for model in [
|
||||
json!("gpt-image-future"),
|
||||
json!("gpt-5.6-sol"),
|
||||
json!(null),
|
||||
json!(42),
|
||||
json!(""),
|
||||
] {
|
||||
let body = serde_json::to_vec(&json!({"model":model,"future":9007199254740993_u64}))
|
||||
.expect("body");
|
||||
let payload =
|
||||
RawJsonPayload::new("openai", Bytes::copy_from_slice(&body)).expect("payload");
|
||||
let operations = [
|
||||
Operation::GenerateImage(ImageRequest::from_raw_json(
|
||||
ImageRequestKind::Generation,
|
||||
payload.clone(),
|
||||
)),
|
||||
Operation::GenerateImage(ImageRequest::from_raw_json(
|
||||
ImageRequestKind::Edit,
|
||||
payload.clone(),
|
||||
)),
|
||||
Operation::Search(StandaloneSearchRequest::from_raw_json(payload)),
|
||||
];
|
||||
for operation in operations {
|
||||
let observation = provider.request_observation(&operation, &client_key);
|
||||
assert_eq!(
|
||||
observation
|
||||
.requested_model
|
||||
.as_ref()
|
||||
.map(PublicModelId::as_str),
|
||||
model.as_str().filter(|model| !model.is_empty()),
|
||||
);
|
||||
let payload = match &operation {
|
||||
Operation::GenerateImage(request) => request.payload(),
|
||||
Operation::Search(request) => request.payload(),
|
||||
_ => unreachable!(),
|
||||
};
|
||||
assert_eq!(payload.body().as_ref(), body.as_slice());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn request_observation_preserves_the_raw_reasoning_effort() {
|
||||
let store = Arc::new(MemoryAccountStore::default());
|
||||
|
||||
@@ -219,7 +219,7 @@ fn decoder_should_emit_calculated_cost_for_complete_known_model_usage() {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn decoder_should_prefer_requested_service_tier_over_response_tier_for_billing() {
|
||||
fn decoder_should_prefer_response_service_tier_over_requested_tier_for_billing() {
|
||||
let body = concat!(
|
||||
"event: response.created\n",
|
||||
"data: {\"type\":\"response.created\",\"response\":{\"id\":\"resp_fast_cost\",\"model\":\"gpt-5.4\",\"service_tier\":\"default\"}}\n\n",
|
||||
@@ -236,7 +236,7 @@ fn decoder_should_prefer_requested_service_tier_over_response_tier_for_billing()
|
||||
assert!(canonical_facts(&events).into_iter().any(|event| matches!(
|
||||
event,
|
||||
GatewayEvent::CalculatedCost(cost)
|
||||
if cost.total().amount().scaled() == 6_875_000
|
||||
if cost.total().amount().scaled() == 3_437_500
|
||||
)));
|
||||
}
|
||||
|
||||
@@ -346,7 +346,7 @@ fn decoder_should_fail_closed_for_fixed_block_web_search_content() {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn websocket_decoder_should_prefer_requested_service_tier_over_response_tier_for_billing() {
|
||||
fn websocket_decoder_should_prefer_response_service_tier_over_requested_tier_for_billing() {
|
||||
let created = websocket_event_to_sse_frame(
|
||||
r#"{"type":"response.created","response":{"id":"resp_ws_fast_cost","model":"gpt-5.4","service_tier":"default"}}"#,
|
||||
)
|
||||
@@ -366,7 +366,7 @@ fn websocket_decoder_should_prefer_requested_service_tier_over_response_tier_for
|
||||
assert!(canonical_facts(&events).into_iter().any(|event| matches!(
|
||||
event,
|
||||
GatewayEvent::CalculatedCost(cost)
|
||||
if cost.total().amount().scaled() == 6_875_000
|
||||
if cost.total().amount().scaled() == 3_437_500
|
||||
)));
|
||||
}
|
||||
|
||||
@@ -936,3 +936,96 @@ fn canonical_facts(events: &[ProviderEvent]) -> Vec<&GatewayEvent> {
|
||||
.flat_map(ProviderEvent::canonical_facts)
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn pricing_response_cost(
|
||||
requested: Option<&str>,
|
||||
actual: Option<&str>,
|
||||
output: serde_json::Value,
|
||||
tools: &[serde_json::Value],
|
||||
) -> Option<u128> {
|
||||
let mut response = json!({
|
||||
"id": "resp_pricing", "model": "gpt-6-astra", "status": "completed",
|
||||
"output": output, "usage": {"input_tokens": 10_000, "output_tokens": 0, "total_tokens": 10_000}
|
||||
});
|
||||
if let Some(tier) = actual {
|
||||
response["service_tier"] = json!(tier);
|
||||
}
|
||||
let created = json!({"type": "response.created", "response": {"id": "resp_pricing", "model": "gpt-6-astra"}});
|
||||
let completed = json!({"type": "response.completed", "response": response});
|
||||
let body = format!(
|
||||
"event: response.created\ndata: {created}\n\nevent: response.completed\ndata: {completed}\n\n"
|
||||
);
|
||||
let events = CodexCanonicalDecoder::new("gpt-6-astra")
|
||||
.with_requested_service_tier(requested)
|
||||
.with_request_tool_pricing("gpt-6-astra", Some(tools))
|
||||
.with_raw_sse_passthrough()
|
||||
.push(body.as_bytes())
|
||||
.expect("计费不确定时仍须正常透传响应");
|
||||
assert!(
|
||||
canonical_facts(&events)
|
||||
.iter()
|
||||
.any(|event| matches!(event, GatewayEvent::Completed(_)))
|
||||
);
|
||||
canonical_facts(&events)
|
||||
.into_iter()
|
||||
.find_map(|event| match event {
|
||||
GatewayEvent::CalculatedCost(cost) => Some(cost.total().amount().scaled()),
|
||||
_ => None,
|
||||
})
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn billing_should_follow_actual_tier_and_only_fall_back_when_it_is_absent() {
|
||||
for (requested, actual, expected) in [
|
||||
(Some("fast"), Some("default"), Some(1_000_000_000)),
|
||||
(Some("auto"), Some("fast"), Some(2_000_000_000)),
|
||||
(Some("default"), Some("flex"), Some(500_000_000)),
|
||||
(Some("fast"), None, Some(2_000_000_000)),
|
||||
(Some("auto"), None, None),
|
||||
(Some("default"), Some("future"), None),
|
||||
] {
|
||||
assert_eq!(
|
||||
pricing_response_cost(requested, actual, json!([]), &[]),
|
||||
expected,
|
||||
"{requested:?} -> {actual:?}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn billing_should_add_file_and_web_search_fees_without_tier_markup() {
|
||||
let output = json!([
|
||||
{"type": "file_search_call", "id": "fs_1", "status": "completed"},
|
||||
{"type": "file_search_call", "id": "fs_2", "status": "completed"},
|
||||
{"type": "web_search_call", "id": "ws_1", "status": "completed", "action": {"type": "search"}}
|
||||
]);
|
||||
for (tier, expected) in [("default", 1_150_000_000), ("fast", 2_150_000_000)] {
|
||||
assert_eq!(
|
||||
pricing_response_cost(
|
||||
None,
|
||||
Some(tier),
|
||||
output.clone(),
|
||||
&[json!({"type": "web_search"})]
|
||||
),
|
||||
Some(expected)
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn billing_should_not_emit_partial_totals_for_unpriced_tool_outputs() {
|
||||
for output in [
|
||||
json!([{"type": "code_interpreter_call", "id": "ci_1", "status": "completed"}]),
|
||||
json!([{"type": "shell_call", "id": "sh_1", "status": "completed"}]),
|
||||
json!([{"type": "image_generation_call", "id": "ig_1", "status": "completed"}]),
|
||||
json!([{"type": "future_paid_tool_call", "id": "tool_1"}]),
|
||||
json!([{"type": "file_search_call", "id": "fs_1", "status": "failed"}]),
|
||||
json!({"malformed": true}),
|
||||
] {
|
||||
assert_eq!(
|
||||
pricing_response_cost(None, None, output.clone(), &[]),
|
||||
None,
|
||||
"{output}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -115,6 +115,79 @@ async fn cold_websocket_should_fall_back_without_recording_a_successful_connect(
|
||||
pool.shutdown().await;
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn downstream_websocket_new_chain_should_preserve_continuation_after_slow_opening() {
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let addr = listener.local_addr().unwrap();
|
||||
let server = tokio::spawn(async move {
|
||||
let (stream, _) = listener.accept().await.unwrap();
|
||||
tokio::select! {
|
||||
() = tokio::time::sleep(Duration::from_secs(1)) => {}
|
||||
accepted = listener.accept() => {
|
||||
let (mut http, _) = accepted.unwrap();
|
||||
let request = read_http_request(&mut http).await;
|
||||
assert!(request.starts_with("POST /codex/responses HTTP/1.1"));
|
||||
write_completed_sse_response(&mut http).await;
|
||||
return;
|
||||
}
|
||||
}
|
||||
let mut websocket = accept_codex_test_websocket(stream).await;
|
||||
let _first = websocket.next().await.unwrap().unwrap();
|
||||
websocket
|
||||
.send(Message::Text(
|
||||
completed_websocket_response("resp_slow_seed", 2, 1).into(),
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let second = websocket.next().await.unwrap().unwrap();
|
||||
let second: Value = serde_json::from_str(second.to_text().unwrap()).unwrap();
|
||||
assert_eq!(second["previous_response_id"], "resp_slow_seed");
|
||||
websocket
|
||||
.send(Message::Text(
|
||||
completed_websocket_response("resp_slow_continuation", 2, 1).into(),
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
});
|
||||
let pool = Arc::new(CodexWebSocketPool::default());
|
||||
let backend = CodexBackendClient::new(
|
||||
reqwest::Client::builder().no_proxy().build().unwrap(),
|
||||
format!("http://{addr}"),
|
||||
test_wire_profile(),
|
||||
)
|
||||
.with_websocket_pool(Arc::clone(&pool));
|
||||
let mut request = new_chain_request("conversation-downstream-slow");
|
||||
request.downstream_websocket_connection_id = Some("ws_downstream_slow".to_owned());
|
||||
|
||||
let first = backend
|
||||
.create_response(
|
||||
&request,
|
||||
request_context("req_slow_seed", Some("chatgpt-account")),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(first.transport, CodexBackendTransport::WebSocket);
|
||||
assert!(first.connection_local_continuation);
|
||||
|
||||
request.set_previous_response_id(Some("resp_slow_seed".to_owned()));
|
||||
request.previous_response_scope = Some(PreviousResponseScope::ConnectionLocal);
|
||||
let second = backend
|
||||
.create_response(
|
||||
&request,
|
||||
request_context("req_slow_continuation", Some("chatgpt-account")),
|
||||
)
|
||||
.await
|
||||
.expect("the next turn must use the socket that generated the previous response");
|
||||
assert_eq!(
|
||||
second.transport_metrics.decision,
|
||||
Some(CodexTransportDecision::ExactWebSocket)
|
||||
);
|
||||
assert!(second.body.contains("resp_slow_continuation"));
|
||||
server.await.unwrap();
|
||||
pool.shutdown().await;
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn external_continuation_should_wait_for_a_cold_websocket() {
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
|
||||
@@ -273,6 +273,34 @@ fn transport_should_prefer_websocket_when_requested_without_history() {
|
||||
assert!(transport_requirement(&request).allows_pre_send_http_fallback());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn downstream_websocket_new_chain_should_require_pooled_websocket_without_storage() {
|
||||
for store in [None, Some(false)] {
|
||||
let mut body = serde_json::Map::new();
|
||||
if let Some(store) = store {
|
||||
body.insert("store".to_owned(), json!(store));
|
||||
}
|
||||
let mut request = CodexResponsesRequest::from_body(body);
|
||||
request.downstream_websocket_connection_id = Some("ws_downstream".to_owned());
|
||||
|
||||
let requirement = transport_requirement(&request);
|
||||
assert!(requirement.requires_websocket());
|
||||
assert!(!requirement.allows_pre_send_http_fallback());
|
||||
assert!(requirement.allows_connection_restart());
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn downstream_websocket_new_chain_should_allow_http_when_response_is_stored() {
|
||||
let mut request = CodexResponsesRequest::from_body(serde_json::Map::from_iter([(
|
||||
"store".to_owned(),
|
||||
json!(true),
|
||||
)]));
|
||||
request.downstream_websocket_connection_id = Some("ws_downstream".to_owned());
|
||||
|
||||
assert!(transport_requirement(&request).allows_pre_send_http_fallback());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unspecified_transport_without_history_should_still_be_a_new_chain() {
|
||||
let request = codex_request("gpt-test", "be brief", Vec::new());
|
||||
|
||||
@@ -239,6 +239,90 @@ async fn exact_limit_success_body_should_be_accepted() {
|
||||
assert!(usage["rate_limit"].is_object());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn billing_should_use_explicit_variant_prices_and_verified_snapshots() {
|
||||
// 每个用例包含一个普通输入 token 和一个输出 token,金额单位为 USD ticks。
|
||||
for (model, expected) in [
|
||||
("gpt-5.1-codex-mini", 22_500),
|
||||
("gpt-5.3-chat-latest", 157_500),
|
||||
("gpt-5.6-cyber", 875_000),
|
||||
("gpt-5.5-cyber", 875_000),
|
||||
("gpt-daybreak-red-latest", 875_000),
|
||||
("gpt-daybreak-blue-latest", 350_000),
|
||||
("chat-latest", 350_000),
|
||||
("gpt-5.1-codex-max", 112_500),
|
||||
("gpt-5.2-codex", 157_500),
|
||||
("gpt-5.4-2026-03-05", 175_000),
|
||||
(" OpenAI/GPT-4o-2024-08-06 ", 125_000),
|
||||
("gpt-4o-2024-05-13", 200_000),
|
||||
("gpt-3.5-turbo-0125", 20_000),
|
||||
("gpt-3.5-turbo-1106", 30_000),
|
||||
] {
|
||||
let billing =
|
||||
openai_billing_breakdown(model, billing_usage(1, 1, 0, 0), None).expect(model);
|
||||
assert_eq!(
|
||||
billing.total_amount().amount().scaled(),
|
||||
expected,
|
||||
"{model}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn billing_should_not_inherit_prices_for_unknown_models_or_tiers() {
|
||||
for model in [
|
||||
"gpt-6-astra-future",
|
||||
"gpt-6-astra-2099-01-01",
|
||||
"gpt-5.6-sol-wm",
|
||||
"gpt-5.6-cyber-future",
|
||||
"gpt-5.4-cyber",
|
||||
"gpt-5.3-codex-spark",
|
||||
"gpt-4.5-preview",
|
||||
"gpt-4-32k",
|
||||
"gpt-5.4:custom",
|
||||
"gpt-5.4.1",
|
||||
] {
|
||||
assert!(
|
||||
openai_billing_breakdown(model, billing_usage(1, 1, 0, 0), None).is_none(),
|
||||
"{model}"
|
||||
);
|
||||
}
|
||||
for (model, input, tier) in [
|
||||
("gpt-5.6-cyber", 272_001, None),
|
||||
("gpt-5.5-cyber", 272_001, None),
|
||||
("gpt-5.6-cyber", 1, Some("fast")),
|
||||
("gpt-5.1-codex-mini", 1, Some("flex")),
|
||||
("gpt-6-astra", 1, Some("auto")),
|
||||
("gpt-6-astra", 1, Some("ultrafast")),
|
||||
] {
|
||||
assert!(
|
||||
openai_billing_breakdown(model, billing_usage(input, 1, 0, 0), tier).is_none(),
|
||||
"{model} {tier:?}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn billing_should_reject_overlapping_or_overflowing_cache_counts() {
|
||||
for (input, cached, written) in [(100, 20, 90), (100, 0, 101), (u64::MAX, u64::MAX, 1)] {
|
||||
assert!(
|
||||
openai_billing_breakdown(
|
||||
"gpt-6-astra",
|
||||
billing_usage(input, 0, cached, written),
|
||||
None
|
||||
)
|
||||
.is_none()
|
||||
);
|
||||
}
|
||||
let cyber = openai_billing_breakdown("gpt-5.6-cyber", billing_usage(100, 10, 20, 10), None)
|
||||
.expect("已公开的 Cyber 缓存写价格");
|
||||
assert_eq!(
|
||||
cyber.cache_write_price_per_million().amount().to_string(),
|
||||
"15.6250000000"
|
||||
);
|
||||
assert_eq!(cyber.total_amount().amount().scaled(), 18_062_500);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn fetch_should_use_wham_usage_headers_only() {
|
||||
let server = MockServer::start().await;
|
||||
|
||||
@@ -0,0 +1,107 @@
|
||||
use super::*;
|
||||
use gateway_core::diagnostics::TraceContext;
|
||||
|
||||
#[tokio::test]
|
||||
async fn diagnostics_preserve_abrupt_disconnect_facts_without_response_content() {
|
||||
for (reuse, send_event) in [(false, false), (false, true), (true, false), (true, true)] {
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let addr = listener.local_addr().unwrap();
|
||||
let server = tokio::spawn(async move {
|
||||
let (stream, _) = listener.accept().await.unwrap();
|
||||
let mut websocket = accept_codex_test_websocket(stream).await;
|
||||
websocket.next().await.unwrap().unwrap();
|
||||
if reuse {
|
||||
websocket
|
||||
.send(Message::Text(
|
||||
completed_websocket_response("resp_warmup", 1, 1).into(),
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
websocket.next().await.unwrap().unwrap();
|
||||
}
|
||||
if send_event {
|
||||
websocket
|
||||
.send(Message::Text(
|
||||
json!({"type": "codex.response.metadata", "private": "private-response-body"})
|
||||
.to_string()
|
||||
.into(),
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
// Drop the transport without the WebSocket close handshake.
|
||||
});
|
||||
let trace = TraceContext::new("req_abrupt_disconnect");
|
||||
let attempt = trace.attempt(1);
|
||||
let mut request = codex_request("gpt-5.5", "private-request-body", Vec::new());
|
||||
request.set_previous_response_id(Some("resp_previous".to_owned()));
|
||||
request.previous_response_scope = Some(PreviousResponseScope::Persisted);
|
||||
request.force_http_sse = false;
|
||||
request.local_conversation_id = Some("diagnostics-conversation".to_owned());
|
||||
let backend = CodexBackendClient::new(
|
||||
reqwest::Client::builder().no_proxy().build().unwrap(),
|
||||
format!("http://{addr}"),
|
||||
test_wire_profile(),
|
||||
)
|
||||
.with_websocket_pool(Arc::new(CodexWebSocketPool::new(Duration::from_mins(1))));
|
||||
if reuse {
|
||||
backend
|
||||
.create_response(
|
||||
&request,
|
||||
request_context("req_warmup", Some("chatgpt-account")),
|
||||
)
|
||||
.await
|
||||
.expect("warmup response must complete");
|
||||
}
|
||||
let result = backend
|
||||
.create_response_stream(
|
||||
&request,
|
||||
request_context("req_abrupt_disconnect", Some("chatgpt-account"))
|
||||
.with_trace(&attempt),
|
||||
)
|
||||
.await;
|
||||
let mut failed = result.is_err();
|
||||
if let Ok(response) = result {
|
||||
let mut body = response.body;
|
||||
while let Some(chunk) = body.next().await {
|
||||
if chunk.is_err() {
|
||||
failed = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
assert!(failed, "an abrupt disconnect must fail the exchange");
|
||||
server.await.unwrap();
|
||||
let snapshot = trace.snapshot().unwrap();
|
||||
let events = snapshot["events"].as_array().unwrap();
|
||||
let connection = events
|
||||
.iter()
|
||||
.find(|event| event["stage"] == "upstream.connection")
|
||||
.unwrap();
|
||||
let failure = events
|
||||
.iter()
|
||||
.find(|event| event["stage"] == "upstream.read.failed")
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
failure["data"]["failureReason"],
|
||||
"reset_without_closing_handshake"
|
||||
);
|
||||
assert_eq!(
|
||||
failure["data"]["connectionId"],
|
||||
connection["data"]["connectionId"]
|
||||
);
|
||||
assert_eq!(failure["data"]["reused"], reuse);
|
||||
assert_eq!(
|
||||
failure["data"]["lastEventType"],
|
||||
if send_event {
|
||||
json!("codex.response.metadata")
|
||||
} else {
|
||||
Value::Null
|
||||
}
|
||||
);
|
||||
assert!(failure["data"]["connectionAgeMs"].is_u64());
|
||||
assert!(failure["data"]["connectionIdleMs"].is_u64());
|
||||
assert!(!snapshot.to_string().contains("private-response-body"));
|
||||
assert!(!snapshot.to_string().contains("private-request-body"));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,92 @@
|
||||
use super::*;
|
||||
|
||||
#[tokio::test]
|
||||
async fn reused_websocket_should_keep_response_metadata_scoped_to_each_exchange() {
|
||||
const ROUNDS: usize = 40;
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let addr = listener.local_addr().unwrap();
|
||||
let server = tokio::spawn(async move {
|
||||
let (stream, _) = listener.accept().await.unwrap();
|
||||
let mut websocket = accept_codex_test_websocket_with(stream, |_, response| {
|
||||
for value in ["first", "second"] {
|
||||
response
|
||||
.headers_mut()
|
||||
.append("x-opening-multi", value.parse().unwrap());
|
||||
}
|
||||
})
|
||||
.await;
|
||||
for round in 0..ROUNDS {
|
||||
websocket.next().await.unwrap().unwrap();
|
||||
websocket
|
||||
.send(Message::Text(
|
||||
json!({
|
||||
"type": "codex.response.metadata",
|
||||
"headers": {
|
||||
"x-models-etag": format!("etag-{round}"),
|
||||
"openai-model": format!("model-{round}"),
|
||||
"x-reasoning-included": "true",
|
||||
"x-codex-safety-buffering-enabled": "true",
|
||||
"x-codex-safety-buffering-faster-model": "faster-model",
|
||||
"x-future-response-metadata": format!("future-{round}"),
|
||||
"x-codex-turn-state": format!("turn-{round}")
|
||||
}
|
||||
})
|
||||
.to_string()
|
||||
.into(),
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
websocket
|
||||
.send(Message::Text(
|
||||
completed_websocket_response(&format!("resp_metadata_{round}"), 1, 1).into(),
|
||||
))
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
});
|
||||
let pool = Arc::new(CodexWebSocketPool::new(Duration::from_mins(1)));
|
||||
let backend = CodexBackendClient::new(
|
||||
reqwest::Client::builder().no_proxy().build().unwrap(),
|
||||
format!("http://{addr}"),
|
||||
test_wire_profile(),
|
||||
)
|
||||
.with_websocket_pool(pool);
|
||||
let request = pooled_websocket_request("conversation-response-metadata");
|
||||
let expected = CodexResponseMetadata {
|
||||
client_headers: vec![
|
||||
(
|
||||
"x-opening-multi".to_owned(),
|
||||
bytes::Bytes::from_static(b"first"),
|
||||
),
|
||||
(
|
||||
"x-opening-multi".to_owned(),
|
||||
bytes::Bytes::from_static(b"second"),
|
||||
),
|
||||
],
|
||||
..CodexResponseMetadata::default()
|
||||
};
|
||||
|
||||
for round in 0..ROUNDS {
|
||||
let response = timeout(
|
||||
Duration::from_secs(10),
|
||||
backend.create_response(
|
||||
&request,
|
||||
request_context("req_response_metadata", Some("chatgpt-account")),
|
||||
),
|
||||
)
|
||||
.await
|
||||
.expect("response within timeout")
|
||||
.expect("response should complete on the same socket");
|
||||
assert_eq!(response.response_metadata, expected, "exchange {round}");
|
||||
assert!(response.body.contains(&format!("etag-{round}")));
|
||||
assert_eq!(response.turn_state, Some(format!("turn-{round}")));
|
||||
if round > 0 {
|
||||
assert!(
|
||||
response
|
||||
.websocket_pool_decision
|
||||
.is_some_and(WebSocketPoolDecision::is_reuse)
|
||||
);
|
||||
}
|
||||
}
|
||||
server.await.unwrap();
|
||||
}
|
||||
@@ -4,6 +4,9 @@ use provider_openai::transport::{
|
||||
websocket::write_websocket_audit_artifact_for_dir,
|
||||
};
|
||||
|
||||
mod diagnostics;
|
||||
mod metadata;
|
||||
|
||||
fn rate_limit_event(used_percent: u64) -> String {
|
||||
json!({
|
||||
"type": "codex.rate_limits",
|
||||
|
||||
@@ -56,7 +56,7 @@ use crate::credential::{
|
||||
RotateManagedGrokCredential, SecretValue, VerifiedGrokAccount, VerifiedTokenSet,
|
||||
};
|
||||
use crate::transport::profile::{GrokCliReleaseSnapshot, GrokCliReleaseStatus};
|
||||
use crate::transport::{GROK_CLI_BASE_URL, XAI_PROVIDER_NAME, grok_billing_breakdown};
|
||||
use crate::transport::{GROK_CLI_BASE_URL, XAI_PROVIDER_NAME, grok_billing_breakdown_with_tier};
|
||||
|
||||
const PENDING_SCHEMA_VERSION: u64 = 3;
|
||||
const PENDING_TTL: TimeDelta = TimeDelta::minutes(30);
|
||||
@@ -390,11 +390,12 @@ impl ProviderAdmin for XaiAdminProvider {
|
||||
else {
|
||||
return Ok(None);
|
||||
};
|
||||
let Some(breakdown) = grok_billing_breakdown(
|
||||
let Some(breakdown) = grok_billing_breakdown_with_tier(
|
||||
&input.upstream_model_id,
|
||||
input_tokens,
|
||||
output_tokens,
|
||||
input.cached_tokens.unwrap_or_default(),
|
||||
input.service_tier.as_deref(),
|
||||
) else {
|
||||
return Ok(None);
|
||||
};
|
||||
|
||||
@@ -79,7 +79,7 @@ pub use transport::{
|
||||
MAX_GROK_BILLING_BYTES, MAX_GROK_MODEL_CATALOG_BYTES, OfficialGrokEndpointPolicy,
|
||||
ReqwestGrokInferenceTransport, ReqwestGrokModelCatalogTransport, ReqwestOAuthTransport,
|
||||
SelectedGrokSession, XAI_PROVIDER_NAME, build_grok_headers, grok_billing_breakdown,
|
||||
parse_grok_billing, parse_grok_model_catalog,
|
||||
grok_billing_breakdown_with_tier, parse_grok_billing, parse_grok_model_catalog,
|
||||
};
|
||||
|
||||
/// xAI 初始化后交给组装根的最小能力集。
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
//! xAI 上游失败分类、账号反馈与恢复决策。
|
||||
|
||||
use super::*;
|
||||
use gateway_core::error::ProviderDiagnostic;
|
||||
|
||||
pub(super) fn is_invalid_encrypted_content_failure(error: &GrokInferenceTransportError) -> bool {
|
||||
error.kind() == GrokInferenceTransportErrorKind::InvalidRequest
|
||||
@@ -509,6 +510,17 @@ pub(super) fn map_transport_error_with_state(
|
||||
if let Some(detail) = error.client_visible_upstream_error().cloned() {
|
||||
mapped = mapped.with_client_visible_upstream_error(detail);
|
||||
}
|
||||
let diagnostic = error.diagnostic().cloned().unwrap_or_else(|| {
|
||||
let status = error
|
||||
.status()
|
||||
.map_or_else(|| "none".to_owned(), |value| value.to_string());
|
||||
ProviderDiagnostic::new(format!(
|
||||
"xAI upstream failure: kind={}, status={status}",
|
||||
kind.as_str()
|
||||
))
|
||||
.with_classification("upstream", "upstream_failure")
|
||||
});
|
||||
mapped = mapped.with_diagnostic(diagnostic);
|
||||
if error.requires_credential_recovery() {
|
||||
mapped = mapped.with_credential_recovery().with_replay_safe();
|
||||
}
|
||||
|
||||
@@ -152,6 +152,7 @@ impl Provider for GrokBuildProvider {
|
||||
}
|
||||
let semantics = codex_responses_request_semantics(payload.body(), payload.context());
|
||||
ProviderRequestObservation {
|
||||
requested_model: None,
|
||||
reasoning_effort: semantics.reasoning_effort,
|
||||
reasoning_preset: semantics.reasoning_preset.map(str::to_owned),
|
||||
request_kind: semantics.request_kind,
|
||||
|
||||
@@ -185,7 +185,8 @@ pub(super) fn cold_compaction_http_sse_stream(
|
||||
}
|
||||
}
|
||||
};
|
||||
yield ProviderEvent::observation(accepted.observation);
|
||||
let mut observation = accepted.observation;
|
||||
yield ProviderEvent::observation(observation.clone());
|
||||
|
||||
let mut body = accepted.response.into_body();
|
||||
let mut canonical = GrokCanonicalDecoder::new(upstream_model.as_str());
|
||||
@@ -251,6 +252,10 @@ pub(super) fn cold_compaction_http_sse_stream(
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(tier) = canonical.response_service_tier() {
|
||||
observation = observation.with_service_tier_if_valid(tier.to_owned());
|
||||
yield ProviderEvent::observation(observation);
|
||||
}
|
||||
let started = facts
|
||||
.started
|
||||
.ok_or_else(|| mark_transient_compaction_failure(protocol_sent()))?;
|
||||
@@ -460,7 +465,7 @@ pub(super) fn cold_http_sse_stream(
|
||||
}
|
||||
};
|
||||
|
||||
let observation = xai_response_observation(&response)?;
|
||||
let mut observation = xai_response_observation(&response)?;
|
||||
let base_timings = observation.timings();
|
||||
let mut first_token_ms: Option<u64> = None;
|
||||
yield ProviderEvent::observation(observation.clone());
|
||||
@@ -529,6 +534,14 @@ pub(super) fn cold_http_sse_stream(
|
||||
}),
|
||||
);
|
||||
}
|
||||
if let Some(tier) = decoder.response_service_tier()
|
||||
&& observation.service_tier() != Some(tier)
|
||||
{
|
||||
observation = observation.with_service_tier_if_valid(tier.to_owned());
|
||||
yield ProviderEvent::observation(observation.clone().with_timings(
|
||||
ProviderResponseTimings { first_token_ms, ..base_timings },
|
||||
));
|
||||
}
|
||||
let completed = events
|
||||
.iter()
|
||||
.flat_map(ProviderEvent::canonical_facts)
|
||||
@@ -581,6 +594,14 @@ pub(super) fn cold_http_sse_stream(
|
||||
}),
|
||||
);
|
||||
}
|
||||
if let Some(tier) = decoder.response_service_tier()
|
||||
&& observation.service_tier() != Some(tier)
|
||||
{
|
||||
observation = observation.with_service_tier_if_valid(tier.to_owned());
|
||||
yield ProviderEvent::observation(observation.clone().with_timings(
|
||||
ProviderResponseTimings { first_token_ms, ..base_timings },
|
||||
));
|
||||
}
|
||||
attach_xai_session_update(&mut final_events, &mut session_capture)?;
|
||||
if let Some(capture) = reasoning_replay_capture.as_mut() {
|
||||
capture.observe(&final_events);
|
||||
|
||||
@@ -38,6 +38,20 @@ struct ModelPricing {
|
||||
long: TokenRates,
|
||||
}
|
||||
|
||||
// 价格来源:https://docs.x.ai/developers/pricing,核验日期 2026-09-09。
|
||||
const GROK_46_PRICING: ModelPricing = ModelPricing {
|
||||
short: TokenRates {
|
||||
input_ticks: 20_000,
|
||||
cached_input_ticks: 5_000,
|
||||
output_ticks: 60_000,
|
||||
},
|
||||
long: TokenRates {
|
||||
input_ticks: 40_000,
|
||||
cached_input_ticks: 10_000,
|
||||
output_ticks: 120_000,
|
||||
},
|
||||
};
|
||||
|
||||
const GROK_45_PRICING: ModelPricing = ModelPricing {
|
||||
short: TokenRates {
|
||||
input_ticks: 20_000,
|
||||
@@ -85,6 +99,23 @@ pub fn grok_billing_breakdown(
|
||||
output_tokens: u64,
|
||||
cached_tokens: u64,
|
||||
) -> Option<CalculatedCostBreakdown> {
|
||||
grok_billing_breakdown_with_tier(model, input_tokens, output_tokens, cached_tokens, None)
|
||||
}
|
||||
|
||||
/// 按响应确认的实际档位计算 xAI Token 费用;未知档位不估算。
|
||||
#[must_use]
|
||||
pub fn grok_billing_breakdown_with_tier(
|
||||
model: &str,
|
||||
input_tokens: u64,
|
||||
output_tokens: u64,
|
||||
cached_tokens: u64,
|
||||
service_tier: Option<&str>,
|
||||
) -> Option<CalculatedCostBreakdown> {
|
||||
let (tier, multiplier) = match service_tier.map(str::trim) {
|
||||
None | Some("default" | "standard") => ("default", 1_u128),
|
||||
Some("priority") => ("priority", 2),
|
||||
Some(_) => return None,
|
||||
};
|
||||
let pricing = model_pricing(model)?;
|
||||
let rates = if input_tokens >= LONG_CONTEXT_THRESHOLD {
|
||||
pricing.long
|
||||
@@ -101,21 +132,21 @@ pub fn grok_billing_breakdown(
|
||||
.checked_add(output_amount_ticks)?;
|
||||
Some(CalculatedCostBreakdown::new(
|
||||
CalculatedCostAmounts::new(
|
||||
usd_money(input_amount_ticks)?,
|
||||
usd_money(output_amount_ticks)?,
|
||||
usd_money(cache_read_amount_ticks)?,
|
||||
usd_money(input_amount_ticks.checked_mul(multiplier)?)?,
|
||||
usd_money(output_amount_ticks.checked_mul(multiplier)?)?,
|
||||
usd_money(cache_read_amount_ticks.checked_mul(multiplier)?)?,
|
||||
usd_money(0)?,
|
||||
usd_money(standard_amount_ticks)?,
|
||||
usd_money(standard_amount_ticks)?,
|
||||
usd_money(standard_amount_ticks.checked_mul(multiplier)?)?,
|
||||
),
|
||||
CalculatedCostRates::new(
|
||||
usd_price_per_million(rates.input_ticks)?,
|
||||
usd_price_per_million(rates.output_ticks)?,
|
||||
usd_price_per_million(rates.cached_input_ticks)?,
|
||||
usd_price_per_million(rates.input_ticks.checked_mul(multiplier)?)?,
|
||||
usd_price_per_million(rates.output_ticks.checked_mul(multiplier)?)?,
|
||||
usd_price_per_million(rates.cached_input_ticks.checked_mul(multiplier)?)?,
|
||||
usd_money(0)?,
|
||||
),
|
||||
Some("default".to_owned()),
|
||||
100,
|
||||
Some(tier.to_owned()),
|
||||
if multiplier == 2 { 200 } else { 100 },
|
||||
))
|
||||
}
|
||||
|
||||
@@ -144,6 +175,8 @@ pub struct GrokCanonicalDecoder {
|
||||
tool_arguments_seen: BTreeSet<u32>,
|
||||
usage_emitted: bool,
|
||||
output_start_seen: bool,
|
||||
response_service_tier: Option<String>,
|
||||
requires_provider_cost: bool,
|
||||
}
|
||||
|
||||
impl GrokCanonicalDecoder {
|
||||
@@ -159,6 +192,8 @@ impl GrokCanonicalDecoder {
|
||||
tool_arguments_seen: BTreeSet::new(),
|
||||
usage_emitted: false,
|
||||
output_start_seen: false,
|
||||
response_service_tier: None,
|
||||
requires_provider_cost: false,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -168,10 +203,30 @@ impl GrokCanonicalDecoder {
|
||||
pub fn for_request(fallback_model: impl Into<String>, request: &GrokResponsesRequest) -> Self {
|
||||
Self {
|
||||
response_transform: request.response_transform(),
|
||||
requires_provider_cost: request.body().get("tool_choice").and_then(Value::as_str)
|
||||
!= Some("none")
|
||||
&& request
|
||||
.body()
|
||||
.get("tools")
|
||||
.and_then(Value::as_array)
|
||||
.is_some_and(|tools| {
|
||||
tools.iter().any(|tool| {
|
||||
!matches!(
|
||||
tool.get("type").and_then(Value::as_str),
|
||||
Some("function" | "custom" | "mcp")
|
||||
)
|
||||
})
|
||||
}),
|
||||
..Self::new(fallback_model)
|
||||
}
|
||||
}
|
||||
|
||||
/// 上游报告的实际服务档位,供费用与持久观测共用。
|
||||
#[must_use]
|
||||
pub fn response_service_tier(&self) -> Option<&str> {
|
||||
self.response_service_tier.as_deref()
|
||||
}
|
||||
|
||||
pub fn push(&mut self, chunk: &[u8]) -> Result<Vec<ProviderEvent>, ProviderError> {
|
||||
let events = self.decoder.push(chunk).map_err(protocol_error)?;
|
||||
self.decode(events)
|
||||
@@ -259,6 +314,16 @@ impl GrokCanonicalDecoder {
|
||||
continue;
|
||||
};
|
||||
let event_type = event_type.to_owned();
|
||||
// 工具转换可能隐藏注入的调用,计费事实必须从转换前的上游事件读取。
|
||||
if let Some(response) = value.get("response") {
|
||||
if let Some(tier) = response.get("service_tier").and_then(Value::as_str) {
|
||||
self.response_service_tier = Some(tier.trim().to_owned());
|
||||
}
|
||||
self.requires_provider_cost |= !token_only_response(response);
|
||||
}
|
||||
if let Some(item) = value.get("item") {
|
||||
self.requires_provider_cost |= !token_only_output(item);
|
||||
}
|
||||
// 无法转换的上游帧(多为畸形帧)直接丢弃,保持已开始的客户端流不中断。
|
||||
let Ok(transformed) = self
|
||||
.response_transform
|
||||
@@ -590,7 +655,10 @@ impl GrokCanonicalDecoder {
|
||||
.to_owned();
|
||||
if let Some(cost) = provider_reported_cost(response)? {
|
||||
output.push(GatewayEvent::ProviderCost(cost));
|
||||
} else if let Some(cost) = usage.and_then(|usage| calculated_cost(response, &model, usage))
|
||||
} else if !self.requires_provider_cost
|
||||
&& let Some(cost) = usage.and_then(|usage| {
|
||||
calculated_cost(response, &model, usage, self.response_service_tier())
|
||||
})
|
||||
{
|
||||
output.push(GatewayEvent::CalculatedCost(cost));
|
||||
}
|
||||
@@ -729,29 +797,71 @@ fn provider_reported_cost(response: &Value) -> Result<Option<ProviderReportedCos
|
||||
return Ok(None);
|
||||
};
|
||||
let ticks = value.as_u64().ok_or_else(protocol_error_marker)?;
|
||||
if ticks == 0 {
|
||||
return Ok(None);
|
||||
}
|
||||
// 明确返回的零费用也是上游账单事实,不能当作缺失而改用本地估算。
|
||||
ProviderReportedCost::from_usd_ticks(u128::from(ticks))
|
||||
.map(Some)
|
||||
.map_err(protocol_error)
|
||||
}
|
||||
|
||||
fn calculated_cost(response: &Value, model: &str, usage: TokenUsage) -> Option<CalculatedCost> {
|
||||
if !billable_usage_is_complete(response, usage) {
|
||||
// cost_in_usd_ticks 已包含服务端工具费;缺少该字段时,只有全部按 token
|
||||
// 计费的响应才能确定总价。
|
||||
fn token_only_output(item: &Value) -> bool {
|
||||
matches!(
|
||||
item.get("type").and_then(Value::as_str),
|
||||
Some(
|
||||
"message"
|
||||
| "reasoning"
|
||||
| "function_call"
|
||||
| "custom_tool_call"
|
||||
| "mcp_call"
|
||||
| "mcp_list_tools"
|
||||
| "mcp_approval_request"
|
||||
| "compaction"
|
||||
)
|
||||
)
|
||||
}
|
||||
|
||||
fn token_only_response(response: &Value) -> bool {
|
||||
let empty = |value: &Value| value.as_object().is_some_and(serde_json::Map::is_empty);
|
||||
response.get("tool_usage").is_none_or(empty)
|
||||
&& response
|
||||
.pointer("/usage/server_side_tool_usage")
|
||||
.is_none_or(empty)
|
||||
&& response
|
||||
.pointer("/usage/num_server_side_tool_calls")
|
||||
.is_none_or(|value| value.as_u64() == Some(0))
|
||||
&& response.get("output").is_none_or(|value| {
|
||||
value
|
||||
.as_array()
|
||||
.is_some_and(|items| items.iter().all(token_only_output))
|
||||
})
|
||||
}
|
||||
|
||||
fn calculated_cost(
|
||||
response: &Value,
|
||||
model: &str,
|
||||
usage: TokenUsage,
|
||||
service_tier: Option<&str>,
|
||||
) -> Option<CalculatedCost> {
|
||||
if !billable_usage_is_complete(response, usage)
|
||||
|| usage.image_input_tokens > 0
|
||||
|| usage.image_output_tokens > 0
|
||||
{
|
||||
return None;
|
||||
}
|
||||
let breakdown = grok_billing_breakdown(
|
||||
let breakdown = grok_billing_breakdown_with_tier(
|
||||
model,
|
||||
usage.input_tokens,
|
||||
usage.output_tokens,
|
||||
usage.cached_tokens,
|
||||
service_tier,
|
||||
)?;
|
||||
Some(breakdown.calculated_cost())
|
||||
}
|
||||
|
||||
fn model_pricing(model: &str) -> Option<ModelPricing> {
|
||||
match model {
|
||||
"grok-4.6" | "grok-4.6-latest" => Some(GROK_46_PRICING),
|
||||
"grok-4.5" | "grok-4.5-latest" | "grok-4.5-build-free" | "grok-build-latest" => {
|
||||
Some(GROK_45_PRICING)
|
||||
}
|
||||
|
||||
@@ -4,7 +4,7 @@ use std::pin::Pin;
|
||||
use std::time::Duration;
|
||||
|
||||
use futures::Stream;
|
||||
use gateway_core::error::{ClientVisibleUpstreamError, OpaqueUpstreamValue};
|
||||
use gateway_core::error::{ClientVisibleUpstreamError, OpaqueUpstreamValue, ProviderDiagnostic};
|
||||
use gateway_core::event::UpstreamHttpVersion;
|
||||
use gateway_core::upstream::UpstreamSendState;
|
||||
use url::Url;
|
||||
@@ -318,6 +318,7 @@ pub enum GrokInferenceTransportErrorKind {
|
||||
#[derive(Clone, PartialEq, Eq)]
|
||||
pub struct GrokInferenceTransportError {
|
||||
kind: GrokInferenceTransportErrorKind,
|
||||
diagnostic: Option<Box<ProviderDiagnostic>>,
|
||||
send_state: UpstreamSendState,
|
||||
status: Option<u16>,
|
||||
retry_after: Option<Duration>,
|
||||
@@ -336,6 +337,7 @@ impl GrokInferenceTransportError {
|
||||
pub const fn new(kind: GrokInferenceTransportErrorKind, send_state: UpstreamSendState) -> Self {
|
||||
Self {
|
||||
kind,
|
||||
diagnostic: None,
|
||||
send_state,
|
||||
status: None,
|
||||
retry_after: None,
|
||||
@@ -353,6 +355,18 @@ impl GrokInferenceTransportError {
|
||||
}
|
||||
}
|
||||
|
||||
/// 保存 transport 提取的安全诊断,避免 Provider 映射时丢失原因。
|
||||
#[must_use]
|
||||
pub fn with_diagnostic(mut self, diagnostic: ProviderDiagnostic) -> Self {
|
||||
self.diagnostic = Some(Box::new(diagnostic));
|
||||
self
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn diagnostic(&self) -> Option<&ProviderDiagnostic> {
|
||||
self.diagnostic.as_deref()
|
||||
}
|
||||
|
||||
/// 附着合法的 HTTP 状态码。
|
||||
#[must_use]
|
||||
pub fn with_status(mut self, status: u16) -> Self {
|
||||
@@ -482,6 +496,7 @@ impl fmt::Debug for GrokInferenceTransportError {
|
||||
formatter
|
||||
.debug_struct("GrokInferenceTransportError")
|
||||
.field("kind", &self.kind)
|
||||
.field("diagnostic", &self.diagnostic)
|
||||
.field("send_state", &self.send_state)
|
||||
.field("status", &self.status)
|
||||
.field("retry_after", &self.retry_after)
|
||||
|
||||
@@ -11,7 +11,9 @@ pub(crate) mod profile;
|
||||
mod request;
|
||||
mod session;
|
||||
|
||||
pub use canonical::{GrokCanonicalDecoder, grok_billing_breakdown};
|
||||
pub use canonical::{
|
||||
GrokCanonicalDecoder, grok_billing_breakdown, grok_billing_breakdown_with_tier,
|
||||
};
|
||||
pub use catalog::{
|
||||
GROK_BILLING_URL, GROK_MODEL_CATALOG_URL, GrokBillingClient, GrokBillingError,
|
||||
GrokBillingRequest, GrokBillingSnapshot, GrokBillingTransport, GrokBillingTransportError,
|
||||
|
||||
@@ -30,7 +30,7 @@ use crate::{
|
||||
HttpMethod, MAX_GROK_BILLING_BYTES, MAX_GROK_MODEL_CATALOG_BYTES, OAuthHttpRequest,
|
||||
OAuthHttpResponse, OAuthHttpTransport, TransportFailure, TransportFailureKind, TransportFuture,
|
||||
};
|
||||
use gateway_core::error::{ClientVisibleUpstreamError, OpaqueUpstreamValue};
|
||||
use gateway_core::error::{ClientVisibleUpstreamError, OpaqueUpstreamValue, ProviderDiagnostic};
|
||||
use gateway_core::event::UpstreamHttpVersion;
|
||||
use gateway_core::upstream::UpstreamSendState;
|
||||
|
||||
@@ -954,6 +954,27 @@ fn classify_oauth_reqwest_error(error: reqwest::Error) -> TransportFailure {
|
||||
TransportFailure::new(kind)
|
||||
}
|
||||
|
||||
fn inference_http_diagnostic(error: &reqwest::Error) -> ProviderDiagnostic {
|
||||
let (stage, code) = if error.is_builder() {
|
||||
("prepare", "http_request_build_failed")
|
||||
} else if error.is_connect() && error.is_timeout() {
|
||||
("connect", "http_connect_timeout")
|
||||
} else if error.is_connect() {
|
||||
("connect", "http_connect_failed")
|
||||
} else if error.is_timeout() {
|
||||
("exchange", "http_timeout")
|
||||
} else if error.is_decode() {
|
||||
("decode", "http_decode_failed")
|
||||
} else if error.is_body() {
|
||||
("receive", "http_body_failed")
|
||||
} else {
|
||||
("exchange", "http_transport_failed")
|
||||
};
|
||||
ProviderDiagnostic::new(format!("xAI HTTP failure: stage={stage}, cause={code}"))
|
||||
.with_classification(stage, code)
|
||||
.with_io_cause(error)
|
||||
}
|
||||
|
||||
fn classify_inference_reqwest_error(error: reqwest::Error) -> GrokInferenceTransportError {
|
||||
let (kind, send_state) = if error.is_builder() {
|
||||
(
|
||||
@@ -977,6 +998,7 @@ fn classify_inference_reqwest_error(error: reqwest::Error) -> GrokInferenceTrans
|
||||
)
|
||||
};
|
||||
GrokInferenceTransportError::new(kind, send_state)
|
||||
.with_diagnostic(inference_http_diagnostic(&error))
|
||||
}
|
||||
|
||||
fn classify_model_catalog_reqwest_error(error: reqwest::Error) -> GrokModelCatalogTransportError {
|
||||
@@ -1010,6 +1032,7 @@ fn classify_inference_stream_error(error: &reqwest::Error) -> GrokInferenceTrans
|
||||
},
|
||||
UpstreamSendState::Sent,
|
||||
)
|
||||
.with_diagnostic(inference_http_diagnostic(error))
|
||||
}
|
||||
|
||||
async fn classify_inference_status(
|
||||
|
||||
@@ -158,6 +158,26 @@ async fn xai_admin_provider_validates_known_billing_breakdown() {
|
||||
assert_eq!(billing.total_amount.amount.as_str(), "0.0002175");
|
||||
assert_eq!(billing.input_price_per_million.amount.as_str(), "2");
|
||||
assert_eq!(billing.output_price_per_million.amount.as_str(), "6");
|
||||
|
||||
let priority = admin
|
||||
.calculated_billing(&ProviderBillingInput {
|
||||
upstream_model_id: "grok-4.6".to_owned(),
|
||||
service_tier: Some("priority".to_owned()),
|
||||
input_tokens: Some(100),
|
||||
output_tokens: Some(10),
|
||||
cached_tokens: Some(20),
|
||||
cache_write_tokens: Some(0),
|
||||
total: CurrencyCost {
|
||||
currency: "USD".to_owned(),
|
||||
amount: "0.00046".parse().expect("amount"),
|
||||
},
|
||||
})
|
||||
.expect("计费明细")
|
||||
.expect("已知的 Priority 价格");
|
||||
assert_eq!(priority.service_tier.as_deref(), Some("priority"));
|
||||
assert_eq!(priority.standard_amount.amount.as_str(), "0.00023");
|
||||
assert_eq!(priority.input_price_per_million.amount.as_str(), "4");
|
||||
assert_eq!(priority.multiplier_percent, 200);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
|
||||
@@ -1239,6 +1239,56 @@ async fn transport_error_observation_should_retain_available_metrics() {
|
||||
assert_eq!(error.kind(), ProviderErrorKind::Unavailable);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn billing_observation_should_persist_the_final_response_tier_before_completion() {
|
||||
for (actual, expected_cost) in [("priority", 400_000_000), ("default", 200_000_000)] {
|
||||
let body = format!(
|
||||
concat!(
|
||||
"event: response.created\ndata: {{\"type\":\"response.created\",\"response\":{{\"id\":\"resp_tier\",\"model\":\"grok-4.6\",\"service_tier\":\"priority\"}}}}\n\n",
|
||||
"event: response.completed\ndata: {{\"type\":\"response.completed\",\"response\":{{\"id\":\"resp_tier\",\"model\":\"grok-4.6\",\"service_tier\":\"{}\",\"status\":\"completed\",\"output\":[],\"usage\":{{\"input_tokens\":10000,\"output_tokens\":0}}}}}}\n\n"
|
||||
),
|
||||
actual
|
||||
);
|
||||
let transport =
|
||||
StubInferenceTransport::sequence([InferenceMode::SuccessBody(body.into_bytes())]);
|
||||
let provider = provider(StubSelector::success(), transport).await;
|
||||
let mut stream = provider
|
||||
.execute(
|
||||
provider_request("xai"),
|
||||
context(CancellationToken::new(), None),
|
||||
)
|
||||
.await
|
||||
.expect("provider stream");
|
||||
let mut tier = None;
|
||||
let mut cost = None;
|
||||
let mut completed = false;
|
||||
while let Some(event) = stream.next().await {
|
||||
let event = event.expect("provider event");
|
||||
if let Some(value) = event
|
||||
.response_observation()
|
||||
.and_then(|value| value.service_tier())
|
||||
{
|
||||
tier = Some(value.to_owned());
|
||||
}
|
||||
for fact in event.canonical_facts() {
|
||||
match fact {
|
||||
GatewayEvent::CalculatedCost(value) => {
|
||||
cost = Some(value.total().amount().scaled())
|
||||
}
|
||||
GatewayEvent::Completed(_) => completed = true,
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
if completed {
|
||||
break;
|
||||
}
|
||||
}
|
||||
assert!(completed);
|
||||
assert_eq!(tier.as_deref(), Some(actual));
|
||||
assert_eq!(cost, Some(expected_cost));
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn compaction_stream_should_emit_openai_wire_and_keep_only_metering_canonical() {
|
||||
let summary = valid_compaction_summary("validated summary");
|
||||
@@ -1704,6 +1754,35 @@ async fn inference_request_uses_oauth_headers_and_no_api_key() {
|
||||
drop(provider);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn transport_diagnostic_survives_provider_mapping_without_enabling_replay() {
|
||||
let transport = StubInferenceTransport::stream_error(
|
||||
GrokInferenceTransportError::new(
|
||||
GrokInferenceTransportErrorKind::Transport,
|
||||
UpstreamSendState::Sent,
|
||||
)
|
||||
.with_diagnostic(
|
||||
gateway_core::error::ProviderDiagnostic::new("xAI HTTP body read failed")
|
||||
.with_classification("receive", "http_body_failed"),
|
||||
),
|
||||
);
|
||||
let provider = provider(StubSelector::success(), transport).await;
|
||||
let mut stream = provider
|
||||
.execute(
|
||||
provider_request("xai"),
|
||||
context(CancellationToken::new(), None),
|
||||
)
|
||||
.await
|
||||
.expect("stream");
|
||||
let error = next_provider_error(&mut stream).await;
|
||||
let diagnostic = error.diagnostic().expect("preserved transport diagnosis");
|
||||
assert_eq!(diagnostic.code(), Some("http_body_failed"));
|
||||
assert_eq!(diagnostic.stage(), Some("receive"));
|
||||
assert_eq!(diagnostic.as_str(), "xAI HTTP body read failed");
|
||||
assert_eq!(error.send_state(), UpstreamSendState::Sent);
|
||||
assert!(!error.replay_is_safe());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn unauthorized_transport_feedback_is_bound_to_selected_account() {
|
||||
let selector = StubSelector::success();
|
||||
|
||||
@@ -5,7 +5,10 @@ use gateway_core::policy::ClientApiKeyId;
|
||||
use gateway_protocol::openai::sse::{encode_sse_event, encode_sse_event_with_metadata};
|
||||
use serde_json::Value;
|
||||
|
||||
use provider_xai::{GrokCanonicalDecoder, GrokResponsesRequest, grok_billing_breakdown};
|
||||
use provider_xai::{
|
||||
GrokCanonicalDecoder, GrokResponsesRequest, grok_billing_breakdown,
|
||||
grok_billing_breakdown_with_tier,
|
||||
};
|
||||
|
||||
fn terminal_cost_events(
|
||||
model: &str,
|
||||
@@ -231,17 +234,139 @@ fn billing_breakdown_should_match_the_calculated_terminal_cost() {
|
||||
|
||||
#[test]
|
||||
fn decoder_should_price_official_grok_45_build_free_variant() {
|
||||
let events = terminal_cost_events("grok-4.5-build-free", 201, 0, 76, Some(0));
|
||||
let events = terminal_cost_events("grok-4.5-build-free", 201, 0, 76, None);
|
||||
|
||||
assert_eq!(calculated_cost_ticks(&events), Some(8_580_000));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn zero_provider_cost_should_fall_back_to_calculated_cost() {
|
||||
fn zero_provider_cost_should_remain_an_authoritative_zero() {
|
||||
let events = terminal_cost_events("grok-4.5", 1, 0, 1, Some(0));
|
||||
|
||||
assert_eq!(calculated_cost_ticks(&events), Some(80_000));
|
||||
assert_eq!(provider_cost_ticks(&events), None);
|
||||
assert_eq!(calculated_cost_ticks(&events), None);
|
||||
assert_eq!(provider_cost_ticks(&events), Some(0));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn billing_should_price_grok_46_cache_and_priority_at_the_context_boundary() {
|
||||
// 短、长上下文分别检查普通输入、缓存、输出的单价,Priority 只采用已确认档位。
|
||||
for (input, tier, expected) in [
|
||||
(199_999, None, ["2", "0.5", "6"]),
|
||||
(200_000, None, ["4", "1", "12"]),
|
||||
(199_999, Some("priority"), ["4", "1", "12"]),
|
||||
(200_000, Some("priority"), ["8", "2", "24"]),
|
||||
] {
|
||||
let billing = grok_billing_breakdown_with_tier("grok-4.6", input, 10, 20, tier)
|
||||
.expect("Grok 4.6 价格");
|
||||
let rates = [
|
||||
billing.input_price_per_million(),
|
||||
billing.cache_read_price_per_million(),
|
||||
billing.output_price_per_million(),
|
||||
]
|
||||
.map(|money| {
|
||||
money
|
||||
.amount()
|
||||
.to_string()
|
||||
.trim_end_matches('0')
|
||||
.trim_end_matches('.')
|
||||
.to_owned()
|
||||
});
|
||||
assert_eq!(rates, expected);
|
||||
}
|
||||
assert!(grok_billing_breakdown_with_tier("grok-4.6", 10, 1, 0, Some("future")).is_none());
|
||||
let priority = grok_billing_breakdown_with_tier("grok-4.6", 100, 10, 20, Some("priority"))
|
||||
.expect("Priority 价格");
|
||||
assert_eq!(priority.standard_amount().amount().scaled(), 2_300_000);
|
||||
assert_eq!(priority.total_amount().amount().scaled(), 4_600_000);
|
||||
assert_eq!(priority.multiplier_percent(), 200);
|
||||
}
|
||||
|
||||
fn pricing_events(
|
||||
mut response: Value,
|
||||
request: Option<&GrokResponsesRequest>,
|
||||
) -> Vec<GatewayEvent> {
|
||||
response["id"] = serde_json::json!("resp_pricing");
|
||||
response["model"] = serde_json::json!("grok-4.6");
|
||||
response["status"] = serde_json::json!("completed");
|
||||
let created = serde_json::json!({"type": "response.created", "response": {"id": "resp_pricing", "model": "grok-4.6", "service_tier": "priority"}});
|
||||
let completed = serde_json::json!({"type": "response.completed", "response": response});
|
||||
let body = format!(
|
||||
"event: response.created\ndata: {created}\n\nevent: response.completed\ndata: {completed}\n\n"
|
||||
);
|
||||
let mut decoder = request.map_or_else(
|
||||
|| GrokCanonicalDecoder::new("grok-4.6"),
|
||||
|request| GrokCanonicalDecoder::for_request("grok-4.6", request),
|
||||
);
|
||||
let events = decoder
|
||||
.push(body.as_bytes())
|
||||
.expect("计费未知时仍应交付响应");
|
||||
let facts: Vec<_> = events
|
||||
.into_iter()
|
||||
.flat_map(|event| event.into_parts().0)
|
||||
.collect();
|
||||
assert!(
|
||||
facts
|
||||
.iter()
|
||||
.any(|event| matches!(event, GatewayEvent::Completed(_)))
|
||||
);
|
||||
facts
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn billing_should_use_the_final_response_tier_and_preserve_provider_cost() {
|
||||
for (tier, ticks, calculated, provider) in [
|
||||
("priority", None, Some(400_000_000), None),
|
||||
("default", None, Some(200_000_000), None),
|
||||
("future", None, None, None),
|
||||
("priority", Some(0), None, Some(0)),
|
||||
("priority", Some(123), None, Some(123)),
|
||||
] {
|
||||
let mut response = serde_json::json!({"service_tier": tier, "output": [], "usage": {"input_tokens": 10_000, "output_tokens": 0}});
|
||||
if let Some(ticks) = ticks {
|
||||
response["usage"]["cost_in_usd_ticks"] = serde_json::json!(ticks);
|
||||
}
|
||||
let events = pricing_events(response, None);
|
||||
assert_eq!(
|
||||
(calculated_cost_ticks(&events), provider_cost_ticks(&events)),
|
||||
(calculated, provider),
|
||||
"{tier}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn billing_should_require_provider_totals_for_server_tools() {
|
||||
for kind in [
|
||||
"web_search_call",
|
||||
"x_search_call",
|
||||
"code_interpreter_call",
|
||||
"file_search_call",
|
||||
"image_generation_call",
|
||||
"future_tool_call",
|
||||
] {
|
||||
let response = serde_json::json!({"output": [{"type": kind, "id": "tool_1", "status": "completed"}], "usage": {"input_tokens": 100, "output_tokens": 1}});
|
||||
assert_eq!(
|
||||
calculated_cost_ticks(&pricing_events(response, None)),
|
||||
None,
|
||||
"{kind}"
|
||||
);
|
||||
}
|
||||
let request =
|
||||
tool_request(serde_json::json!({"input": "查资料", "tools": [{"type": "web_search"}]}));
|
||||
// 工具结果可能被上游省略,允许调用收费工具时不能仅靠空 output 认定没有附加费。
|
||||
let mut response =
|
||||
serde_json::json!({"output": [], "usage": {"input_tokens": 100, "output_tokens": 1}});
|
||||
assert_eq!(
|
||||
calculated_cost_ticks(&pricing_events(response.clone(), Some(&request))),
|
||||
None
|
||||
);
|
||||
response["usage"]["cost_in_usd_ticks"] = serde_json::json!(0);
|
||||
assert_eq!(
|
||||
provider_cost_ticks(&pricing_events(response, Some(&request))),
|
||||
Some(0)
|
||||
);
|
||||
let response = serde_json::json!({"output": [], "usage": {"input_tokens": 100, "output_tokens": 1, "num_server_side_tool_calls": 1}});
|
||||
assert_eq!(calculated_cost_ticks(&pricing_events(response, None)), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -252,6 +252,22 @@ async fn inference_transport_should_build_distinct_cold_account_clients_concurre
|
||||
assert!(concurrency.max_active.load(Ordering::SeqCst) >= 2);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn inference_connect_failure_keeps_safe_diagnosis_without_endpoint_or_payload() {
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let origin = Url::parse(&format!("http://{}", listener.local_addr().unwrap())).unwrap();
|
||||
drop(listener);
|
||||
let error = inference_transport(&origin)
|
||||
.execute(inference_request(&origin))
|
||||
.await
|
||||
.expect_err("closed listener");
|
||||
let diagnostic = error.diagnostic().expect("transport cause");
|
||||
assert_eq!(diagnostic.stage(), Some("connect"));
|
||||
assert_eq!(diagnostic.code(), Some("connection_refused"));
|
||||
assert!(!diagnostic.as_str().contains("127.0.0.1"));
|
||||
assert_eq!(error.send_state(), UpstreamSendState::NotSent);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn inference_transport_should_classify_http_failures_without_retaining_bodies() {
|
||||
let cases = [
|
||||
|
||||
@@ -1,10 +1 @@
|
||||
60bcece284f2d6a2cd3e8dea67833dfc1c7da625cdd5f0b63884c725fa00a3d4 0001_initial.sql
|
||||
93219760c733616e7bba9a8933c207b26752b2ebf7e6edfe1ef42ba227c50a7d 0002_runtime_pressure_observability.sql
|
||||
1b6ffab434c754814aebb0eee3b650bd9770010059472bccf5bc856b319880ca 0003_client_min_versions.sql
|
||||
22f912ab3e196de7983ef50a1dba5aee340bbea82bdd15d05863e87012000c83 0004_ops_error_send_state.sql
|
||||
194d7b4bedd83f0d76428a21699b0e946449bf2ba61a0fdf8710ebfd421af8b2 0005_openai_continuation_recovery_observability.sql
|
||||
3b4bfd5e3f5135fa2fad1088b0c2a287861e1250d66af31dbd40dc890751b567 0006_openai_session_transport_recovery.sql
|
||||
8c0ec7fab80d95dfd5a370c70706c36ac5907bb5afea0cf1886196f21fd8317d 0007_request_diagnostics.sql
|
||||
374f0d8fa8b671ca845d085f7d173b2aa57320e7e5c41e0398329f0bdd89506e 0008_persisted_fact_integrity.sql
|
||||
de20c9c9c57fcf0371a2245a3de9a57d72ab75d1c08ffff92f76b2b270d6a72c 0009_trim_observability_indexes.sql
|
||||
661aba0469d4b9e4626e9e9331015bb2120cc948e8d15b1ca83a394a5007e11a 0010_remove_unused_event_aggregation.sql
|
||||
b7d3df7b48d835a00a3b74cbefc958a039d38fea5529cdad04bfbc703dca72aa 0001_initial.sql
|
||||
|
||||
@@ -33,18 +33,18 @@ create table admin_audit_events (
|
||||
constraint admin_audit_events_actor_fk foreign key (actor_admin_user_id)
|
||||
references admin_users (id)
|
||||
on update restrict
|
||||
on delete set null
|
||||
on delete set null,
|
||||
constraint admin_audit_events_actor_identity_ck check (
|
||||
actor_admin_user_id is null
|
||||
or (actor_kind = 'admin_session' and actor_ref = 'admin:' || actor_admin_user_id)
|
||||
)
|
||||
);
|
||||
|
||||
create index admin_audit_events_actor_idx
|
||||
on admin_audit_events (actor_admin_user_id, created_at desc, id desc)
|
||||
on admin_audit_events (actor_admin_user_id)
|
||||
where actor_admin_user_id is not null;
|
||||
create index admin_audit_events_created_idx
|
||||
on admin_audit_events (created_at desc, id desc);
|
||||
create index admin_audit_events_entity_idx
|
||||
on admin_audit_events (entity_kind, entity_ref, created_at desc, id desc);
|
||||
create index admin_audit_events_actor_ref_idx
|
||||
on admin_audit_events (actor_ref, created_at desc, id desc);
|
||||
|
||||
-- 客户端 API 密钥、鉴权范围与限额。
|
||||
create table client_api_keys (
|
||||
@@ -70,8 +70,6 @@ create table client_api_keys (
|
||||
|
||||
create index client_api_keys_created_idx
|
||||
on client_api_keys (created_at desc, id desc);
|
||||
create index client_api_keys_last_used_idx
|
||||
on client_api_keys (last_used_at desc, id desc);
|
||||
|
||||
-- 全局运行参数与配置版本。
|
||||
create table runtime_settings (
|
||||
@@ -88,6 +86,8 @@ create table runtime_settings (
|
||||
ops_event_retention_days bigint not null default 30,
|
||||
audit_retention_days bigint not null default 90,
|
||||
updated_at timestamptz not null,
|
||||
min_codex_desktop_version text,
|
||||
min_codex_cli_version text,
|
||||
constraint runtime_settings_singleton_ck check (id = 1),
|
||||
constraint runtime_settings_revision_ck check (config_revision > 0),
|
||||
constraint runtime_settings_refresh_ck check (
|
||||
@@ -107,6 +107,24 @@ create table runtime_settings (
|
||||
usage_retention_days >= 31
|
||||
and ops_event_retention_days > 0
|
||||
and audit_retention_days > 0
|
||||
),
|
||||
constraint runtime_settings_client_versions_ck check (
|
||||
(
|
||||
min_codex_desktop_version is null
|
||||
or (
|
||||
octet_length(min_codex_desktop_version) between 1 and 64
|
||||
and min_codex_desktop_version = btrim(min_codex_desktop_version)
|
||||
and min_codex_desktop_version !~ '[[:cntrl:]]'
|
||||
)
|
||||
)
|
||||
and (
|
||||
min_codex_cli_version is null
|
||||
or (
|
||||
octet_length(min_codex_cli_version) between 1 and 64
|
||||
and min_codex_cli_version = btrim(min_codex_cli_version)
|
||||
and min_codex_cli_version !~ '[[:cntrl:]]'
|
||||
)
|
||||
)
|
||||
)
|
||||
);
|
||||
|
||||
@@ -224,13 +242,8 @@ create index provider_accounts_runtime_idx
|
||||
on provider_accounts (provider_kind, enabled, id);
|
||||
create index provider_accounts_credential_state_idx
|
||||
on provider_accounts (credential_state, id);
|
||||
create index provider_accounts_quota_access_idx
|
||||
on provider_accounts (quota_access_state, quota_reset_at, id);
|
||||
create index provider_accounts_access_expiry_idx
|
||||
on provider_accounts (access_token_expires_at, id);
|
||||
create index provider_accounts_refresh_due_idx
|
||||
on provider_accounts (next_refresh_at, id)
|
||||
where enabled and has_refresh_token and next_refresh_at is not null;
|
||||
create index provider_accounts_email_idx
|
||||
on provider_accounts (provider_kind, lower(email))
|
||||
where email is not null;
|
||||
@@ -266,8 +279,6 @@ create table account_groups (
|
||||
|
||||
create unique index account_groups_name_uq
|
||||
on account_groups (lower(name));
|
||||
create index account_groups_list_idx
|
||||
on account_groups (enabled, created_at desc, id desc);
|
||||
|
||||
create table account_group_accounts (
|
||||
account_group_id text not null,
|
||||
@@ -376,6 +387,25 @@ create table model_requests (
|
||||
routing_scope text not null,
|
||||
routing_group_refs text[] not null default '{}',
|
||||
routing_group_names_snapshot jsonb not null default '[]'::jsonb,
|
||||
admission_decision_ms bigint,
|
||||
account_selection_wait_ms bigint,
|
||||
capacity_used_slots bigint,
|
||||
capacity_total_slots bigint,
|
||||
raw_upstream_error text,
|
||||
continuation_affinity_hash text,
|
||||
continuation_previous_response_id_hash text,
|
||||
continuation_requested boolean not null default false,
|
||||
continuation_unavailable_reason text,
|
||||
upstream_connection_id text,
|
||||
upstream_connection_exit_reason text,
|
||||
upstream_connection_age_ms bigint,
|
||||
upstream_connection_idle_ms bigint,
|
||||
recovery_request_id text,
|
||||
recovered_at timestamptz,
|
||||
recovery_attempt_count integer not null default 0,
|
||||
recovery_retry_delay_ms bigint,
|
||||
recovery_total_latency_ms bigint,
|
||||
diagnostic_trace_json jsonb,
|
||||
constraint model_requests_client_ref_ck check (
|
||||
client_api_key_id is null or client_api_key_id = client_api_key_ref
|
||||
),
|
||||
@@ -523,18 +553,97 @@ create table model_requests (
|
||||
constraint model_requests_account_fk foreign key (provider_account_id)
|
||||
references provider_accounts (id)
|
||||
on update restrict
|
||||
on delete set null
|
||||
on delete set null,
|
||||
constraint model_requests_runtime_pressure_ck check (
|
||||
(admission_decision_ms is null or admission_decision_ms >= 0)
|
||||
and (account_selection_wait_ms is null or account_selection_wait_ms >= 0)
|
||||
and (
|
||||
(capacity_used_slots is null and capacity_total_slots is null)
|
||||
or (
|
||||
capacity_used_slots >= 0
|
||||
and capacity_total_slots > 0
|
||||
and capacity_used_slots <= capacity_total_slots
|
||||
)
|
||||
)
|
||||
),
|
||||
constraint model_requests_continuation_hashes_ck check (
|
||||
(continuation_affinity_hash is null
|
||||
or continuation_affinity_hash ~ '^[0-9a-f]{64}$')
|
||||
and (continuation_previous_response_id_hash is null
|
||||
or continuation_previous_response_id_hash ~ '^[0-9a-f]{64}$')
|
||||
and (continuation_requested
|
||||
or continuation_previous_response_id_hash is null)
|
||||
),
|
||||
constraint model_requests_continuation_reason_ck check (
|
||||
continuation_unavailable_reason is null
|
||||
or (
|
||||
octet_length(continuation_unavailable_reason) between 1 and 64
|
||||
and continuation_unavailable_reason ~ '^[a-z][a-z0-9_]*$'
|
||||
)
|
||||
),
|
||||
constraint model_requests_upstream_connection_ck check (
|
||||
(
|
||||
upstream_connection_id is null
|
||||
and upstream_connection_exit_reason is null
|
||||
and upstream_connection_age_ms is null
|
||||
and upstream_connection_idle_ms is null
|
||||
)
|
||||
or (
|
||||
octet_length(upstream_connection_id) between 1 and 128
|
||||
and upstream_connection_id !~ '[[:cntrl:]]'
|
||||
and octet_length(upstream_connection_exit_reason) between 1 and 64
|
||||
and upstream_connection_exit_reason ~ '^[a-z][a-z0-9_]*$'
|
||||
and upstream_connection_age_ms >= 0
|
||||
and upstream_connection_idle_ms between 0 and upstream_connection_age_ms
|
||||
)
|
||||
),
|
||||
constraint model_requests_recovery_ck check (
|
||||
recovery_attempt_count >= 0
|
||||
and (
|
||||
(
|
||||
recovered_at is null
|
||||
and recovery_request_id is null
|
||||
and recovery_retry_delay_ms is null
|
||||
and recovery_total_latency_ms is null
|
||||
)
|
||||
or (
|
||||
recovered_at is not null
|
||||
and recovered_at >= completed_at
|
||||
and recovery_request_id is not null
|
||||
and octet_length(recovery_request_id) between 1 and 128
|
||||
and recovery_attempt_count > 0
|
||||
and recovery_retry_delay_ms >= 0
|
||||
and recovery_total_latency_ms >= recovery_retry_delay_ms
|
||||
)
|
||||
)
|
||||
),
|
||||
constraint model_requests_diagnostic_trace_object
|
||||
check (diagnostic_trace_json is null or jsonb_typeof(diagnostic_trace_json) = 'object'),
|
||||
constraint model_requests_fact_completeness_ck check (
|
||||
(provider_account_id is null or provider_account_ref is not null)
|
||||
and (cost_source = 'unavailable' or cost_currency is not null)
|
||||
and num_nonnulls(capacity_used_slots, capacity_total_slots) in (0, 2)
|
||||
and num_nonnulls(
|
||||
upstream_connection_id, upstream_connection_exit_reason,
|
||||
upstream_connection_age_ms, upstream_connection_idle_ms
|
||||
) in (0, 4)
|
||||
and (
|
||||
recovered_at is null
|
||||
or (
|
||||
completed_at is not null
|
||||
and recovery_retry_delay_ms is not null
|
||||
and recovery_total_latency_ms is not null
|
||||
)
|
||||
)
|
||||
)
|
||||
);
|
||||
|
||||
create index model_requests_client_idx
|
||||
on model_requests (client_api_key_id, started_at desc, id desc);
|
||||
on model_requests (client_api_key_id)
|
||||
where client_api_key_id is not null;
|
||||
create index model_requests_account_idx
|
||||
on model_requests (
|
||||
provider_account_id,
|
||||
provider_kind,
|
||||
started_at desc,
|
||||
id desc
|
||||
);
|
||||
on model_requests (provider_account_id)
|
||||
where provider_account_id is not null;
|
||||
create index model_requests_started_idx
|
||||
on model_requests (started_at desc, id desc);
|
||||
create index model_requests_client_ref_idx
|
||||
@@ -561,6 +670,37 @@ create index model_requests_retention_idx
|
||||
create index model_requests_routing_group_refs_idx
|
||||
on model_requests using gin (routing_group_refs);
|
||||
|
||||
create index model_requests_pending_continuation_recovery_idx
|
||||
on model_requests (
|
||||
client_api_key_ref,
|
||||
continuation_affinity_hash,
|
||||
completed_at desc,
|
||||
id desc
|
||||
)
|
||||
where error_kind = 'continuation_recovery_required'
|
||||
and recovered_at is null
|
||||
and continuation_affinity_hash is not null;
|
||||
|
||||
create index model_requests_recovery_request_idx
|
||||
on model_requests (recovery_request_id)
|
||||
where recovery_request_id is not null;
|
||||
|
||||
create index model_requests_pending_ws_transport_recovery_idx
|
||||
on model_requests (
|
||||
client_api_key_ref,
|
||||
continuation_affinity_hash,
|
||||
completed_at desc,
|
||||
id desc
|
||||
)
|
||||
where provider_kind = 'openai'
|
||||
and outcome = 'failed'
|
||||
and error_kind = 'upstream_unavailable'
|
||||
and upstream_transport = 'websocket'
|
||||
and upstream_send_state = 'ambiguous'
|
||||
and downstream_committed_at is null
|
||||
and recovered_at is null
|
||||
and continuation_affinity_hash is not null;
|
||||
|
||||
-- 运行期警告与错误事件。
|
||||
create table ops_events (
|
||||
id text primary key,
|
||||
@@ -583,8 +723,9 @@ create table ops_events (
|
||||
upstream_request_id text,
|
||||
latency_ms bigint,
|
||||
message text not null,
|
||||
occurrence_count integer not null default 1,
|
||||
created_at timestamptz not null,
|
||||
upstream_send_state text,
|
||||
raw_upstream_error text,
|
||||
constraint ops_events_request_attempt_ck check (
|
||||
(
|
||||
model_request_id is null
|
||||
@@ -601,7 +742,6 @@ create table ops_events (
|
||||
(status_code is null or status_code between 100 and 599)
|
||||
and (retry_after_ms is null or retry_after_ms >= 0)
|
||||
and (latency_ms is null or latency_ms >= 0)
|
||||
and occurrence_count > 0
|
||||
),
|
||||
constraint ops_events_account_ref_ck check (
|
||||
provider_account_id is null or provider_account_id = provider_account_ref
|
||||
@@ -613,24 +753,23 @@ create table ops_events (
|
||||
constraint ops_events_account_fk foreign key (provider_account_id)
|
||||
references provider_accounts (id)
|
||||
on update restrict
|
||||
on delete set null
|
||||
on delete set null,
|
||||
constraint ops_events_upstream_send_state_ck check (
|
||||
upstream_send_state is null
|
||||
or upstream_send_state in ('not_sent', 'sent', 'ambiguous')
|
||||
),
|
||||
constraint ops_events_account_snapshot_present_ck check (
|
||||
provider_account_id is null or provider_account_ref is not null
|
||||
)
|
||||
);
|
||||
|
||||
create index ops_events_request_idx
|
||||
on ops_events (model_request_id, attempt_index, id);
|
||||
create index ops_events_account_idx
|
||||
on ops_events (
|
||||
provider_account_id,
|
||||
provider_kind,
|
||||
created_at desc,
|
||||
id desc
|
||||
);
|
||||
on ops_events (provider_account_id)
|
||||
where provider_account_id is not null;
|
||||
create index ops_events_created_idx
|
||||
on ops_events (created_at desc, id desc);
|
||||
create index ops_events_component_idx
|
||||
on ops_events (component, created_at desc, id desc);
|
||||
create index ops_events_failure_idx
|
||||
on ops_events (failure_kind, created_at desc, id desc);
|
||||
create index ops_events_account_ref_idx
|
||||
on ops_events (provider_account_ref, created_at desc, id desc)
|
||||
where provider_account_ref is not null;
|
||||
@@ -798,6 +937,9 @@ create table backup_records (
|
||||
-- expires_at 为空,或不早于创建时间(创建时 now + 天数 恒晚于 created_at)。
|
||||
constraint backup_records_expiry_ck check (
|
||||
expires_at is null or expires_at >= created_at
|
||||
),
|
||||
constraint backup_records_completion_order_ck check (
|
||||
completed_at is null or completed_at >= started_at
|
||||
)
|
||||
);
|
||||
|
||||
|
||||
@@ -1,20 +0,0 @@
|
||||
-- 请求热路径的准入判定、账号选择等待与账号池容量观测。
|
||||
alter table model_requests
|
||||
add column admission_decision_ms bigint,
|
||||
add column account_selection_wait_ms bigint,
|
||||
add column capacity_used_slots bigint,
|
||||
add column capacity_total_slots bigint;
|
||||
|
||||
alter table model_requests
|
||||
add constraint model_requests_runtime_pressure_ck check (
|
||||
(admission_decision_ms is null or admission_decision_ms >= 0)
|
||||
and (account_selection_wait_ms is null or account_selection_wait_ms >= 0)
|
||||
and (
|
||||
(capacity_used_slots is null and capacity_total_slots is null)
|
||||
or (
|
||||
capacity_used_slots >= 0
|
||||
and capacity_total_slots > 0
|
||||
and capacity_used_slots <= capacity_total_slots
|
||||
)
|
||||
)
|
||||
);
|
||||
@@ -1,21 +0,0 @@
|
||||
alter table runtime_settings
|
||||
add column min_codex_desktop_version text,
|
||||
add column min_codex_cli_version text,
|
||||
add constraint runtime_settings_client_versions_ck check (
|
||||
(
|
||||
min_codex_desktop_version is null
|
||||
or (
|
||||
octet_length(min_codex_desktop_version) between 1 and 64
|
||||
and min_codex_desktop_version = btrim(min_codex_desktop_version)
|
||||
and min_codex_desktop_version !~ '[[:cntrl:]]'
|
||||
)
|
||||
)
|
||||
and (
|
||||
min_codex_cli_version is null
|
||||
or (
|
||||
octet_length(min_codex_cli_version) between 1 and 64
|
||||
and min_codex_cli_version = btrim(min_codex_cli_version)
|
||||
and min_codex_cli_version !~ '[[:cntrl:]]'
|
||||
)
|
||||
)
|
||||
);
|
||||
@@ -1,11 +0,0 @@
|
||||
-- 为运维错误保留 payload 发送边界与上游错误原文。
|
||||
alter table model_requests
|
||||
add column raw_upstream_error text;
|
||||
|
||||
alter table ops_events
|
||||
add column upstream_send_state text,
|
||||
add column raw_upstream_error text,
|
||||
add constraint ops_events_upstream_send_state_ck check (
|
||||
upstream_send_state is null
|
||||
or upstream_send_state in ('not_sent', 'sent', 'ambiguous')
|
||||
);
|
||||
@@ -1,80 +0,0 @@
|
||||
alter table model_requests
|
||||
add column continuation_affinity_hash text,
|
||||
add column continuation_previous_response_id_hash text,
|
||||
add column continuation_requested boolean not null default false,
|
||||
add column continuation_unavailable_reason text,
|
||||
add column upstream_connection_id text,
|
||||
add column upstream_connection_exit_reason text,
|
||||
add column upstream_connection_age_ms bigint,
|
||||
add column upstream_connection_idle_ms bigint,
|
||||
add column recovery_request_id text,
|
||||
add column recovered_at timestamptz,
|
||||
add column recovery_attempt_count integer not null default 0,
|
||||
add column recovery_retry_delay_ms bigint,
|
||||
add column recovery_total_latency_ms bigint,
|
||||
add constraint model_requests_continuation_hashes_ck check (
|
||||
(continuation_affinity_hash is null
|
||||
or continuation_affinity_hash ~ '^[0-9a-f]{64}$')
|
||||
and (continuation_previous_response_id_hash is null
|
||||
or continuation_previous_response_id_hash ~ '^[0-9a-f]{64}$')
|
||||
and (continuation_requested
|
||||
or continuation_previous_response_id_hash is null)
|
||||
),
|
||||
add constraint model_requests_continuation_reason_ck check (
|
||||
continuation_unavailable_reason is null
|
||||
or (
|
||||
octet_length(continuation_unavailable_reason) between 1 and 64
|
||||
and continuation_unavailable_reason ~ '^[a-z][a-z0-9_]*$'
|
||||
)
|
||||
),
|
||||
add constraint model_requests_upstream_connection_ck check (
|
||||
(
|
||||
upstream_connection_id is null
|
||||
and upstream_connection_exit_reason is null
|
||||
and upstream_connection_age_ms is null
|
||||
and upstream_connection_idle_ms is null
|
||||
)
|
||||
or (
|
||||
octet_length(upstream_connection_id) between 1 and 128
|
||||
and upstream_connection_id !~ '[[:cntrl:]]'
|
||||
and octet_length(upstream_connection_exit_reason) between 1 and 64
|
||||
and upstream_connection_exit_reason ~ '^[a-z][a-z0-9_]*$'
|
||||
and upstream_connection_age_ms >= 0
|
||||
and upstream_connection_idle_ms between 0 and upstream_connection_age_ms
|
||||
)
|
||||
),
|
||||
add constraint model_requests_recovery_ck check (
|
||||
recovery_attempt_count >= 0
|
||||
and (
|
||||
(
|
||||
recovered_at is null
|
||||
and recovery_request_id is null
|
||||
and recovery_retry_delay_ms is null
|
||||
and recovery_total_latency_ms is null
|
||||
)
|
||||
or (
|
||||
recovered_at is not null
|
||||
and recovered_at >= completed_at
|
||||
and recovery_request_id is not null
|
||||
and octet_length(recovery_request_id) between 1 and 128
|
||||
and recovery_attempt_count > 0
|
||||
and recovery_retry_delay_ms >= 0
|
||||
and recovery_total_latency_ms >= recovery_retry_delay_ms
|
||||
)
|
||||
)
|
||||
);
|
||||
|
||||
create index model_requests_pending_continuation_recovery_idx
|
||||
on model_requests (
|
||||
client_api_key_ref,
|
||||
continuation_affinity_hash,
|
||||
completed_at desc,
|
||||
id desc
|
||||
)
|
||||
where error_kind = 'continuation_recovery_required'
|
||||
and recovered_at is null
|
||||
and continuation_affinity_hash is not null;
|
||||
|
||||
create index model_requests_recovery_request_idx
|
||||
on model_requests (recovery_request_id)
|
||||
where recovery_request_id is not null;
|
||||
@@ -1,61 +0,0 @@
|
||||
create index model_requests_pending_ws_transport_recovery_idx
|
||||
on model_requests (
|
||||
client_api_key_ref,
|
||||
continuation_affinity_hash,
|
||||
completed_at desc,
|
||||
id desc
|
||||
)
|
||||
where provider_kind = 'openai'
|
||||
and outcome = 'failed'
|
||||
and error_kind = 'upstream_unavailable'
|
||||
and upstream_transport = 'websocket'
|
||||
and upstream_send_state = 'ambiguous'
|
||||
and downstream_committed_at is null
|
||||
and recovered_at is null
|
||||
and continuation_affinity_hash is not null;
|
||||
|
||||
with session_transport_recoveries as (
|
||||
select
|
||||
prior.id as failed_request_id,
|
||||
recovered.id as recovery_request_id,
|
||||
recovered.started_at as recovery_started_at,
|
||||
recovered.completed_at as recovered_at
|
||||
from model_requests prior
|
||||
cross join lateral (
|
||||
select current.id, current.started_at, current.completed_at
|
||||
from model_requests current
|
||||
where current.client_api_key_ref = prior.client_api_key_ref
|
||||
and current.continuation_affinity_hash = prior.continuation_affinity_hash
|
||||
and current.provider_kind = prior.provider_kind
|
||||
and current.outcome = 'succeeded'
|
||||
and current.upstream_transport = 'http_sse'
|
||||
and current.completed_at is not null
|
||||
and current.started_at >= prior.completed_at
|
||||
and current.started_at <= prior.completed_at + interval '30 seconds'
|
||||
order by current.started_at, current.id
|
||||
limit 1
|
||||
) recovered
|
||||
where prior.provider_kind = 'openai'
|
||||
and prior.outcome = 'failed'
|
||||
and prior.error_kind = 'upstream_unavailable'
|
||||
and prior.upstream_transport = 'websocket'
|
||||
and prior.upstream_send_state = 'ambiguous'
|
||||
and prior.downstream_committed_at is null
|
||||
and prior.recovered_at is null
|
||||
and prior.continuation_affinity_hash is not null
|
||||
)
|
||||
update model_requests prior
|
||||
set recovery_attempt_count = prior.recovery_attempt_count + 1,
|
||||
recovery_request_id = recovery.recovery_request_id,
|
||||
recovered_at = recovery.recovered_at,
|
||||
recovery_retry_delay_ms = greatest(
|
||||
0,
|
||||
floor(extract(epoch from (recovery.recovery_started_at - prior.completed_at)) * 1000)
|
||||
)::bigint,
|
||||
recovery_total_latency_ms = greatest(
|
||||
0,
|
||||
floor(extract(epoch from (recovery.recovered_at - prior.completed_at)) * 1000)
|
||||
)::bigint
|
||||
from session_transport_recoveries recovery
|
||||
where prior.id = recovery.failed_request_id
|
||||
and prior.recovered_at is null;
|
||||
@@ -1,5 +0,0 @@
|
||||
-- Bounded request evidence is retained with its owning model request.
|
||||
-- NULL identifies historical requests for which no trace was captured.
|
||||
alter table model_requests add column diagnostic_trace_json jsonb;
|
||||
alter table model_requests add constraint model_requests_diagnostic_trace_object
|
||||
check (diagnostic_trace_json is null or jsonb_typeof(diagnostic_trace_json) = 'object');
|
||||
@@ -1,41 +0,0 @@
|
||||
-- Existing value checks also need explicit presence checks: CHECK accepts NULL.
|
||||
alter table model_requests
|
||||
add constraint model_requests_fact_completeness_ck check (
|
||||
(provider_account_id is null or provider_account_ref is not null)
|
||||
and (cost_source = 'unavailable' or cost_currency is not null)
|
||||
and num_nonnulls(capacity_used_slots, capacity_total_slots) in (0, 2)
|
||||
and num_nonnulls(
|
||||
upstream_connection_id, upstream_connection_exit_reason,
|
||||
upstream_connection_age_ms, upstream_connection_idle_ms
|
||||
) in (0, 4)
|
||||
and (
|
||||
recovered_at is null
|
||||
or (
|
||||
completed_at is not null
|
||||
and recovery_retry_delay_ms is not null
|
||||
and recovery_total_latency_ms is not null
|
||||
)
|
||||
)
|
||||
);
|
||||
|
||||
alter table ops_events
|
||||
add constraint ops_events_account_snapshot_present_ck check (
|
||||
provider_account_id is null or provider_account_ref is not null
|
||||
);
|
||||
|
||||
alter table backup_records
|
||||
add constraint backup_records_completion_order_ck check (
|
||||
completed_at is null or completed_at >= started_at
|
||||
);
|
||||
|
||||
update admin_audit_events
|
||||
set actor_ref = 'admin:' || actor_admin_user_id
|
||||
where actor_admin_user_id is not null
|
||||
and actor_kind = 'admin_session'
|
||||
and actor_ref is distinct from 'admin:' || actor_admin_user_id;
|
||||
|
||||
alter table admin_audit_events
|
||||
add constraint admin_audit_events_actor_identity_ck check (
|
||||
actor_admin_user_id is null
|
||||
or (actor_kind = 'admin_session' and actor_ref = 'admin:' || actor_admin_user_id)
|
||||
);
|
||||
@@ -1,28 +0,0 @@
|
||||
-- Live references support FK maintenance; historical queries use *_ref indexes.
|
||||
drop index model_requests_account_idx;
|
||||
create index model_requests_account_idx on model_requests (provider_account_id)
|
||||
where provider_account_id is not null;
|
||||
|
||||
drop index model_requests_client_idx;
|
||||
create index model_requests_client_idx on model_requests (client_api_key_id)
|
||||
where client_api_key_id is not null;
|
||||
|
||||
drop index ops_events_account_idx;
|
||||
create index ops_events_account_idx on ops_events (provider_account_id)
|
||||
where provider_account_id is not null;
|
||||
|
||||
drop index admin_audit_events_actor_idx;
|
||||
create index admin_audit_events_actor_idx on admin_audit_events (actor_admin_user_id)
|
||||
where actor_admin_user_id is not null;
|
||||
|
||||
-- Key usage is updated continuously; a small management list can sort on demand.
|
||||
drop index client_api_keys_last_used_idx;
|
||||
|
||||
-- These leading keys no longer match the current runtime queries.
|
||||
drop index provider_accounts_quota_access_idx;
|
||||
drop index provider_accounts_refresh_due_idx;
|
||||
drop index account_groups_list_idx;
|
||||
drop index ops_events_component_idx;
|
||||
drop index ops_events_failure_idx;
|
||||
drop index admin_audit_events_actor_ref_idx;
|
||||
drop index admin_audit_events_entity_idx;
|
||||
@@ -1,8 +0,0 @@
|
||||
-- Each row records one event; aggregation is performed by the read queries.
|
||||
alter table ops_events drop constraint ops_events_values_ck;
|
||||
alter table ops_events drop column occurrence_count;
|
||||
alter table ops_events add constraint ops_events_values_ck check (
|
||||
(status_code is null or status_code between 100 and 599)
|
||||
and (retry_after_ms is null or retry_after_ms >= 0)
|
||||
and (latency_ms is null or latency_ms >= 0)
|
||||
);
|
||||
@@ -3,8 +3,8 @@
|
||||
sqlx 在服务启动、监听请求之前按编号执行迁移,并把文件 checksum 记入 `_sqlx_migrations`。
|
||||
已应用迁移的字节与记录不一致时,服务会拒绝启动。
|
||||
|
||||
`0001_initial.sql` 是初始基线,后续编号继续增加字段和索引;
|
||||
当前完整清单以本目录 SQL 文件和 `.frozen-sha256` 为准,不要只执行 `0001`。
|
||||
`0001_initial.sql` 定义数据库初始化基线,后续 schema 变更按编号新增迁移。
|
||||
完整清单以本目录 SQL 文件和 `.frozen-sha256` 为准。
|
||||
|
||||
## 冻结规则
|
||||
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user