From fc16c48beb974b4e0305d78b9812bbdc6ccc14c6 Mon Sep 17 00:00:00 2001 From: zyycn Date: Thu, 11 Jun 2026 22:06:17 +0800 Subject: [PATCH] feat: add runtime log rotation Co-authored-by: Codex --- README.md | 3 +- docs/implementation-status.md | 6 +- .../plans/2026-06-11-codex-proxy-rs.md | 39 ++-- src/app.rs | 2 + src/logs/rotation.rs | 197 +++++++++++++++++- src/main.rs | 7 + tests/log_rotation_test.rs | 34 +++ ...gration_test.rs => storage_schema_test.rs} | 2 +- 8 files changed, 266 insertions(+), 24 deletions(-) create mode 100644 tests/log_rotation_test.rs rename tests/{storage_migration_test.rs => storage_schema_test.rs} (89%) diff --git a/README.md b/README.md index f787b3164..6af27fb18 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@ Rust rewrite of Codex Proxy focused only on ChatGPT/Codex accounts and the Codex - `/v1/models` - ChatGPT/Codex OAuth and refresh-token based account pool - Codex Desktop-style TLS, headers, Cookies, and fingerprint updates -- SQLite storage with sqlx migrations +- SQLite storage with sqlx-managed schema - Structured logs with pagination and file rotation ## Excluded @@ -19,6 +19,7 @@ Rust rewrite of Codex Proxy focused only on ChatGPT/Codex accounts and the Codex - Ollama bridge - Per-account proxy assignment - Electron app +- Legacy compatibility layers and old-project data migration scripts ## Auth Model diff --git a/docs/implementation-status.md b/docs/implementation-status.md index fcc841193..8f5b88a7b 100644 --- a/docs/implementation-status.md +++ b/docs/implementation-status.md @@ -7,13 +7,13 @@ Update this file before each feature commit. | Scaffold | Completed | `fc47967` | `cargo fmt --check`; `cargo test`; `cargo clippy --all-targets --all-features --locked -- -D warnings` | Rust crate, lint policy, pinned TLS-sensitive dependencies, `.gitignore`, and minimal entry points are in place. | | API contract docs | Completed | `fc47967` | `cargo fmt --check`; `cargo test`; `cargo clippy --all-targets --all-features --locked -- -D warnings` | `/v1/*` OpenAI-compatible body, `/admin/*` frontend envelope, body codes, camelCase, and request ID policy are documented. | | Configuration | Completed | `9ed8579` | `cargo test default_config_keeps_only_codex_backend`; full `fmt/test/clippy` before commit | Defines Codex-only config, default YAML, and `Arc` state shell. | -| SQLite storage | Completed | `ffec8b5` | `cargo test migrations_create_accounts_and_event_tables`; full `fmt/test/clippy` before commit | Creates SQLite WAL connector, migrations for accounts/API keys/sessions/cookies/fingerprints/events, and event/account indexes. | +| SQLite storage | Completed | `ffec8b5` | `cargo test sqlite_schema_creates_accounts_and_event_tables`; full `fmt/test/clippy` before commit | Creates SQLite WAL connector, sqlx-managed schema for accounts/API keys/sessions/cookies/fingerprints/events, and event/account indexes. | | Admin auth and client API keys | Completed | `24d5799` | `cargo test client_api_key_has_proxy_prefix_and_verifies_against_hash`; `cargo test admin_password_hash_is_not_a_client_api_key`; full `fmt/test/clippy` before commit | Admin passwords use Argon2id, client API keys use `cpr_` prefix plus HMAC-SHA256 with server-side pepper. | | Secret encryption | Completed | `105aa58` | `cargo test secret_box_encrypts_and_decrypts_without_storing_plaintext`; full `fmt/test/clippy` before commit | AES-256-GCM `SecretBox` stores upstream secrets as `v1::` without plaintext. | -| Logging and pagination | Completed | `eb0341e` | `cargo test event_logs_are_cursor_paginated`; full `fmt/test/clippy` before commit | Adds cursor pagination, `Page` camelCase serialization, event log repository, and rotation config shell. | +| Logging and pagination | Completed | `eb0341e` + runtime logging scope commit | `cargo test event_logs_are_cursor_paginated`; `cargo test rotating_writer_splits_files_when_size_limit_is_reached`; full `fmt/test/clippy` before commit | Adds cursor pagination, `Page` camelCase serialization, event log repository, runtime tracing initialization, HTTP trace layer, and date/size-based file rotation. | | TLS headers and fingerprint | Completed | `1bdd08c` | `cargo test codex_headers_include_desktop_identity_and_turn_state`; `cargo tree \| rg 'reqwest\|rustls'`; full `fmt/test/clippy` before commit | Adds Codex Desktop fingerprint model, exact identity/request headers, and pinned reqwest/rustls client builder. | | Cookie persistence | Completed | `4710926` | `cargo test cookie_jar_captures_and_replays_account_scoped_cookies`; full `fmt/test/clippy` before commit | Adds account-scoped in-memory Cookie jar and repository shell; database persistence follows storage integration. | -| Account pool and refresh | Completed | `0f97c5c` | `cargo test account_pool_skips_expired_disabled_banned_and_quota_exhausted_accounts`; full `fmt/test/clippy` before commit | Adds account statuses, acquisition filter, lifecycle events, token pair, OAuth config, and refresh policy shell. | +| Account pool and refresh | Completed | `0f97c5c` | `cargo test account_pool_skips_expired_disabled_banned_and_quota_exhausted_accounts`; full `fmt/test/clippy` before commit | Adds account statuses, acquisition filter, lifecycle events, token pair, OAuth config, and native refresh policy shell with no legacy compatibility layer. | | Translation | Completed | `eff182e` | `cargo test chat_completion_translates_to_codex_response_request`; full `fmt/test/clippy` before commit | Adds OpenAI Chat request types, Codex Responses request type, translation, and OpenAI-compatible error body helper. | | HTTP routes | Completed | `ab0ad8f` | `cargo test v1_requires_client_api_key_not_admin_cookie`; full `fmt/test/clippy` before commit | Adds Axum router, health route, `/v1` unauthorized OpenAI error body, global request-id response header, and binary listener. | | Admin/frontend response contract | Completed | `1c34ea7` | `cargo test admin_envelope_serializes_lower_camel_case_request_id`; `cargo test v1_response_has_request_id_header_without_admin_body_field`; full `fmt/test/clippy` before commit | Adds reusable admin envelopes with body codes, lower camelCase JSON, paginated list shape, requestId bodies, and keeps `/v1/*` OpenAI-compatible. | diff --git a/docs/superpowers/plans/2026-06-11-codex-proxy-rs.md b/docs/superpowers/plans/2026-06-11-codex-proxy-rs.md index 68c07de2b..e0c061e15 100644 --- a/docs/superpowers/plans/2026-06-11-codex-proxy-rs.md +++ b/docs/superpowers/plans/2026-06-11-codex-proxy-rs.md @@ -19,7 +19,7 @@ Keep: - OpenAI-compatible inbound API: `/v1/chat/completions`, `/v1/responses`, `/v1/models`. - Codex Responses passthrough for clients that already speak Responses format. - ChatGPT/Codex OAuth login, device login, CLI token import, manual access-token import. -- Account pool, account rotation, quota state, token refresh behavior compatible with the current TypeScript implementation. +- Account pool, account rotation, quota state, and token refresh behavior reimplemented natively with the same required semantics, not through a TypeScript compatibility layer. - Rust native TLS path using pinned `reqwest` and `rustls`. - Codex Desktop fingerprint headers and fingerprint auto-update. - Account-scoped Cloudflare Cookie capture and replay. @@ -32,6 +32,7 @@ Remove: - Ollama bridge and Ollama settings. - Per-account proxy assignment, proxy health checks, proxy UI, proxy routing. - Electron desktop packaging and large dashboard UI. +- Legacy compatibility layers, old-project API adapters, and old-project data migration scripts. Security split: @@ -52,7 +53,8 @@ These adjustments come from the `rust-best-practices` review and should be treat - Use the `config` crate with `config/default.yaml`, optional `config/local.yaml`, and `CPRS_*` environment overrides. Keep `.env` support via `dotenvy` for local development only. - Prefer borrowed parameters in public APIs (`&str`, `&[T]`, references to request structs) unless ownership transfer is part of the domain model. - Keep tests descriptive and targeted. Unit tests live beside modules for private behavior; integration tests under `tests/` cover route and storage behavior. Use `insta` only for structured payload snapshots that are worth reviewing. -- Add sparse Chinese comments only where the reason is not obvious from names or types. Good targets are protocol/security/business invariants: TLS fingerprint pinning, Codex Desktop header parity, account-scoped Cookie replay, admin-session versus client-API-key boundaries, and token refresh compatibility with the TypeScript implementation. Do not comment every line, do not restate obvious code, and do not leave untracked TODO comments. +- Add sparse Chinese comments only where the reason is not obvious from names or types. Good targets are protocol/security/business invariants: TLS fingerprint pinning, Codex Desktop header parity, account-scoped Cookie replay, admin-session versus client-API-key boundaries, and token refresh invariants. Do not comment every line, do not restate obvious code, and do not leave untracked TODO comments. +- This project has no old deployment or data history to preserve. Do not add compatibility shims, legacy adapters, dual-mode behavior for removed features, or scripts that migrate data from the TypeScript project. SQL schema version files are allowed only for this Rust service's own database creation and future schema evolution. - Keep Rust identifiers idiomatic: type names use `PascalCase`, functions/variables/fields use `snake_case`, constants use `SCREAMING_SNAKE_CASE`. JSON fields exposed to the frontend use lower camelCase via `#[serde(rename_all = "camelCase")]`; do not expose PascalCase or snake_case JSON fields in admin/frontend APIs. ## Dependency Version Policy @@ -919,25 +921,25 @@ git commit -m "feat: add codex-only configuration" --- -### Task 3: SQLite Storage with sqlx Migrations +### Task 3: SQLite Storage with sqlx-Managed Schema **Files:** -- Create: `/home/zyy/桌面/Codes/codex-proxy-rs/migrations/0001_initial.sql` -- Create: `/home/zyy/桌面/Codes/codex-proxy-rs/migrations/0002_events_indexes.sql` +- Create schema version file: `/home/zyy/桌面/Codes/codex-proxy-rs/migrations/0001_initial.sql` +- Create schema version file: `/home/zyy/桌面/Codes/codex-proxy-rs/migrations/0002_events_indexes.sql` - Create: `/home/zyy/桌面/Codes/codex-proxy-rs/src/storage/mod.rs` - Create: `/home/zyy/桌面/Codes/codex-proxy-rs/src/storage/db.rs` - Modify: `/home/zyy/桌面/Codes/codex-proxy-rs/src/lib.rs` -- Test: `/home/zyy/桌面/Codes/codex-proxy-rs/tests/storage_migration_test.rs` +- Test: `/home/zyy/桌面/Codes/codex-proxy-rs/tests/storage_schema_test.rs` -- [x] **Step 1: Write failing migration test** +- [x] **Step 1: Write failing schema test** -Create `/home/zyy/桌面/Codes/codex-proxy-rs/tests/storage_migration_test.rs`: +Create `/home/zyy/桌面/Codes/codex-proxy-rs/tests/storage_schema_test.rs`: ```rust use codex_proxy_rs::storage::db::connect_sqlite; #[tokio::test] -async fn migrations_create_accounts_and_event_tables() { +async fn sqlite_schema_creates_accounts_and_event_tables() { let dir = tempfile::tempdir().unwrap(); let db = dir.path().join("test.sqlite"); let url = format!("sqlite://{}", db.display()); @@ -956,12 +958,12 @@ async fn migrations_create_accounts_and_event_tables() { Run: ```bash -cargo test migrations_create_accounts_and_event_tables +cargo test sqlite_schema_creates_accounts_and_event_tables ``` Expected: compile failure because `storage::db` does not exist. -- [x] **Step 3: Create migrations** +- [x] **Step 3: Create sqlx schema version files** Create `/home/zyy/桌面/Codes/codex-proxy-rs/migrations/0001_initial.sql`: @@ -1103,12 +1105,12 @@ pub async fn connect_sqlite(database_url: &str) -> Result` runtime state boundary. - Open item scan: no open-ended implementation gaps are intentionally left in task steps. - Type consistency: the early scaffold uses `AppConfig`, `AppState`, `AccountStatus`, `EventLog`, `Fingerprint`, and `CodexResponsesRequest` consistently across later tasks. diff --git a/src/app.rs b/src/app.rs index e29143c8c..cbe9daeaa 100644 --- a/src/app.rs +++ b/src/app.rs @@ -3,6 +3,7 @@ use axum::{ routing::{get, post}, Router, }; +use tower_http::trace::TraceLayer; use crate::{ http::{health::health, middleware::attach_request_id, v1::responses}, @@ -15,5 +16,6 @@ pub fn build_router(state: AppState) -> Router { .route("/v1/responses", post(responses)) .route("/v1/chat/completions", post(responses)) .with_state(state) + .layer(TraceLayer::new_for_http()) .layer(from_fn(attach_request_id)) } diff --git a/src/logs/rotation.rs b/src/logs/rotation.rs index 86c971ef7..af09eb839 100644 --- a/src/logs/rotation.rs +++ b/src/logs/rotation.rs @@ -1,4 +1,16 @@ -use std::path::{Path, PathBuf}; +use std::{ + fs::{self, File, OpenOptions}, + io::{self, Write}, + path::{Path, PathBuf}, + sync::{Arc, Mutex, MutexGuard}, + time::{Duration, SystemTime}, +}; + +use chrono::Utc; +use thiserror::Error; +use tracing_subscriber::{fmt::MakeWriter, EnvFilter}; + +const LOG_PREFIX: &str = "codex-proxy-rs"; #[derive(Debug, Clone)] pub struct RotationConfig { @@ -16,3 +28,186 @@ impl RotationConfig { } } } + +#[derive(Debug, Error)] +pub enum LogError { + #[error("log io error: {0}")] + Io(#[from] io::Error), + #[error("global tracing subscriber is already initialized")] + SubscriberAlreadyInitialized, +} + +#[derive(Clone)] +pub struct RotatingLogWriter { + inner: Arc>, +} + +impl RotatingLogWriter { + pub fn new(config: RotationConfig) -> Result { + fs::create_dir_all(&config.directory)?; + let current_date = current_date(); + let active_path = active_path(&config.directory, ¤t_date); + if active_path.exists() + && fs::metadata(&active_path)?.len() >= config.max_file_bytes + && config.max_file_bytes > 0 + { + rotate_existing(&active_path, ¤t_date)?; + } + cleanup_retention(&config.directory, config.retention_days)?; + let file = open_append(&active_path)?; + let current_size = file.metadata()?.len(); + Ok(Self { + inner: Arc::new(Mutex::new(RotatingLogInner { + config, + current_date, + current_size, + file, + })), + }) + } + + pub fn active_path(&self) -> PathBuf { + let inner = self.lock_inner(); + active_path(&inner.config.directory, &inner.current_date) + } + + fn lock_inner(&self) -> MutexGuard<'_, RotatingLogInner> { + match self.inner.lock() { + Ok(guard) => guard, + Err(poisoned) => poisoned.into_inner(), + } + } +} + +impl<'a> MakeWriter<'a> for RotatingLogWriter { + type Writer = RotatingLogFile; + + fn make_writer(&'a self) -> Self::Writer { + RotatingLogFile { + inner: Arc::clone(&self.inner), + } + } +} + +pub struct RotatingLogFile { + inner: Arc>, +} + +impl Write for RotatingLogFile { + fn write(&mut self, buf: &[u8]) -> io::Result { + let mut inner = match self.inner.lock() { + Ok(guard) => guard, + Err(poisoned) => poisoned.into_inner(), + }; + inner.roll_if_needed(buf.len() as u64)?; + let written = inner.file.write(buf)?; + inner.current_size += written as u64; + Ok(written) + } + + fn flush(&mut self) -> io::Result<()> { + let mut inner = match self.inner.lock() { + Ok(guard) => guard, + Err(poisoned) => poisoned.into_inner(), + }; + inner.file.flush() + } +} + +struct RotatingLogInner { + config: RotationConfig, + current_date: String, + current_size: u64, + file: File, +} + +impl RotatingLogInner { + fn roll_if_needed(&mut self, incoming_bytes: u64) -> io::Result<()> { + let today = current_date(); + let date_changed = today != self.current_date; + let size_exceeded = self.config.max_file_bytes > 0 + && self.current_size > 0 + && self.current_size.saturating_add(incoming_bytes) > self.config.max_file_bytes; + if !date_changed && !size_exceeded { + return Ok(()); + } + + self.file.flush()?; + let old_active_path = active_path(&self.config.directory, &self.current_date); + if old_active_path.exists() { + rotate_existing(&old_active_path, &self.current_date)?; + } + cleanup_retention(&self.config.directory, self.config.retention_days)?; + self.current_date = today; + let new_active_path = active_path(&self.config.directory, &self.current_date); + self.file = open_append(&new_active_path)?; + self.current_size = self.file.metadata()?.len(); + Ok(()) + } +} + +pub fn init_tracing(config: RotationConfig) -> Result { + let writer = RotatingLogWriter::new(config)?; + let filter = EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new("info")); + // 中文注释：日志必须写入可轮转文件，不能只依赖 stdout，否则线上问题无法回溯。 + tracing_subscriber::fmt() + .json() + .with_env_filter(filter) + .with_writer(writer.clone()) + .try_init() + .map_err(|_| LogError::SubscriberAlreadyInitialized)?; + Ok(writer) +} + +fn current_date() -> String { + Utc::now().format("%Y-%m-%d").to_string() +} + +fn active_path(directory: &Path, date: &str) -> PathBuf { + directory.join(format!("{LOG_PREFIX}.{date}.log")) +} + +fn archive_path(active_path: &Path, date: &str) -> PathBuf { + let stamp = Utc::now().format("%H%M%S%.3f"); + let suffix = uuid::Uuid::new_v4(); + active_path.with_file_name(format!("{LOG_PREFIX}.{date}.{stamp}.{suffix}.log")) +} + +fn rotate_existing(active_path: &Path, date: &str) -> io::Result<()> { + if fs::metadata(active_path) + .map(|metadata| metadata.len()) + .unwrap_or(0) + == 0 + { + return Ok(()); + } + fs::rename(active_path, archive_path(active_path, date)) +} + +fn open_append(path: &Path) -> io::Result { + OpenOptions::new().create(true).append(true).open(path) +} + +fn cleanup_retention(directory: &Path, retention_days: u64) -> io::Result<()> { + if retention_days == 0 { + return Ok(()); + } + let cutoff = SystemTime::now() + .checked_sub(Duration::from_secs(retention_days.saturating_mul(86_400))) + .unwrap_or(SystemTime::UNIX_EPOCH); + for entry in fs::read_dir(directory)? { + let entry = entry?; + let path = entry.path(); + let Some(file_name) = path.file_name().and_then(|value| value.to_str()) else { + continue; + }; + if !file_name.starts_with(LOG_PREFIX) || !file_name.ends_with(".log") { + continue; + } + let metadata = entry.metadata()?; + if metadata.modified().is_ok_and(|modified| modified < cutoff) { + let _ = fs::remove_file(path); + } + } + Ok(()) +} diff --git a/src/main.rs b/src/main.rs index c5300d780..6038e52f0 100644 --- a/src/main.rs +++ b/src/main.rs @@ -4,6 +4,7 @@ use codex_proxy_rs::{ AdminConfig, ApiConfig, AppConfig, AuthConfig, DatabaseConfig, LoggingConfig, SecurityConfig, ServerConfig, TlsConfig, }, + logs::rotation::{init_tracing, RotationConfig}, state::AppState, }; @@ -42,10 +43,16 @@ async fn main() -> anyhow::Result<()> { }, }; + let _log_writer = init_tracing(RotationConfig::new( + &config.logging.directory, + config.logging.max_file_bytes, + config.logging.retention_days, + ))?; let host = config.server.host.clone(); let port = config.server.port; let app = build_router(AppState::new(config)); let listener = tokio::net::TcpListener::bind((host.as_str(), port)).await?; + tracing::info!(host, port, "codex-proxy-rs listening"); axum::serve(listener, app).await?; Ok(()) } diff --git a/tests/log_rotation_test.rs b/tests/log_rotation_test.rs new file mode 100644 index 000000000..6b23238c4 --- /dev/null +++ b/tests/log_rotation_test.rs @@ -0,0 +1,34 @@ +use std::{fs, io::Write}; + +use tracing_subscriber::fmt::MakeWriter; + +use codex_proxy_rs::logs::rotation::{RotatingLogWriter, RotationConfig}; + +#[test] +fn rotating_writer_splits_files_when_size_limit_is_reached() { + let dir = tempfile::tempdir().unwrap(); + let writer = RotatingLogWriter::new(RotationConfig::new(dir.path(), 16, 14)).unwrap(); + + { + let mut file = writer.make_writer(); + file.write_all(b"first log line\n").unwrap(); + file.write_all(b"second log line\n").unwrap(); + file.flush().unwrap(); + } + + let log_files = fs::read_dir(dir.path()) + .unwrap() + .filter_map(Result::ok) + .filter(|entry| { + entry + .file_name() + .to_str() + .is_some_and(|name| name.starts_with("codex-proxy-rs") && name.ends_with(".log")) + }) + .count(); + + assert!( + log_files >= 2, + "expected rotated log files, found {log_files}" + ); +} diff --git a/tests/storage_migration_test.rs b/tests/storage_schema_test.rs similarity index 89% rename from tests/storage_migration_test.rs rename to tests/storage_schema_test.rs index af2241f0d..bf37e4e47 100644 --- a/tests/storage_migration_test.rs +++ b/tests/storage_schema_test.rs @@ -1,7 +1,7 @@ use codex_proxy_rs::storage::db::connect_sqlite; #[tokio::test] -async fn migrations_create_accounts_and_event_tables() { +async fn sqlite_schema_creates_accounts_and_event_tables() { let dir = tempfile::tempdir().unwrap(); let db = dir.path().join("test.sqlite"); let url = format!("sqlite://{}", db.display());