codex-proxy-rs/backend/tests/api/admin/system/mod.rs

use std::{
    io::Write,
    path::Path,
    sync::Arc,
    time::{Duration, SystemTime},
};

use axum::{
    body::{to_bytes, Body},
    http::{header::CONTENT_TYPE, Request, StatusCode},
};
use codex_proxy_rs::{bootstrap::services::Services, bootstrap::state::AppState};
use flate2::{write::GzEncoder, Compression};
use serde_json::{json, Value};
use sha2::{Digest, Sha256};
use tar::{Builder, EntryType, Header};
use tokio::sync::Mutex;
use tower::util::ServiceExt;
use wiremock::{
    matchers::{method, path},
    Mock, MockServer, ResponseTemplate,
};

use crate::support::{
    admin::seed_admin_session,
    config::test_config,
    fingerprint::{runtime_fingerprint, test_fingerprint},
    http::response_json,
};

static SYSTEM_ENV_LOCK: Mutex<()> = Mutex::const_new(());
const VERSION_OLD: &str = "0.1.0";
const VERSION_NEW: &str = "0.2.0";

#[tokio::test]
async fn version_should_return_backend_build_metadata() {
    let _guard = SYSTEM_ENV_LOCK.lock().await;
    let repository = "zyycn/codex-proxy-rs-version-route";
    let github = MockServer::start().await;
    mount_latest_release(&github, repository, "0.2.0").await;
    set_system_update_env(repository, &github.uri());
    std::env::set_var("CPR_GIT_SHA", "version-test-sha");
    std::env::set_var("CPR_BUILD_TIME", "2026-07-01T00:00:00Z");
    let (app, _dir) = admin_system_test_app("system-version").await;

    let response = app
        .oneshot(
            Request::builder()
                .method("GET")
                .uri("/api/admin/system/version")
                .header("cookie", "cpr_admin_session=session_1")
                .header("x-request-id", "req_system_version")
                .body(Body::empty())
                .unwrap(),
        )
        .await
        .unwrap();
    let body = response_json(response).await;

    assert!(
        body["data"]["version"] == VERSION_OLD
            && body["data"]["gitSha"] == "version-test-sha"
            && body["data"]["buildTime"] == "2026-07-01T00:00:00Z"
            && body["data"]["deploymentMode"] == "docker"
            && body["data"]["deploymentModeLabel"] == "Docker"
            && body["data"]["updateChannel"] == "stable"
            && body["data"]["latestVersion"] == VERSION_NEW
            && body["data"]["hasUpdate"] == true
            && body["data"]["updateCached"] == false
            && body["data"]["updateWarning"].is_null()
    );
}

#[tokio::test]
async fn update_detail_should_use_cached_release_when_not_refreshed() {
    let _guard = SYSTEM_ENV_LOCK.lock().await;
    let repository = "zyycn/codex-proxy-rs-cache-route";
    let success_server = MockServer::start().await;
    mount_latest_release(&success_server, repository, "0.2.0").await;
    set_system_update_env(repository, &success_server.uri());
    let (app, _dir) = admin_system_test_app("system-cache").await;
    let initial = get_update_detail(&app, true, "req_system_cache_initial").await;

    let failing_server = MockServer::start().await;
    set_system_update_env(repository, &failing_server.uri());
    let cached = get_update_detail(&app, false, "req_system_cache_cached").await;

    assert!(
        initial["hasUpdate"] == true
            && cached["cached"] == true
            && cached["latestVersion"] == "0.2.0"
    );
}

#[tokio::test]
async fn update_detail_should_fallback_to_cache_when_refresh_fetch_fails() {
    let _guard = SYSTEM_ENV_LOCK.lock().await;
    let repository = "zyycn/codex-proxy-rs-cache-forced-route";
    let success_server = MockServer::start().await;
    mount_latest_release(&success_server, repository, "0.3.0").await;
    set_system_update_env(repository, &success_server.uri());
    let (app, _dir) = admin_system_test_app("system-cache-forced").await;
    let initial = get_update_detail(&app, true, "req_system_cache_forced_initial").await;

    let failing_server = MockServer::start().await;
    Mock::given(method("GET"))
        .and(path(format!("/repos/{repository}/releases/latest")))
        .respond_with(ResponseTemplate::new(500))
        .mount(&failing_server)
        .await;
    set_system_update_env(repository, &failing_server.uri());
    let fallback = get_update_detail(&app, true, "req_system_cache_forced_fallback").await;

    assert!(
        initial["hasUpdate"] == true
            && fallback["cached"] == true
            && fallback["latestVersion"] == "0.3.0"
            && fallback["warning"].is_null()
    );
}

#[tokio::test]
async fn update_detail_should_report_source_build_as_unsupported() {
    let _guard = SYSTEM_ENV_LOCK.lock().await;
    let repository = "zyycn/codex-proxy-rs-source-route";
    let github = MockServer::start().await;
    mount_latest_release(&github, repository, "0.2.0").await;
    set_system_update_env(repository, &github.uri());
    std::env::set_var("CPR_BUILD_TYPE", "source");
    let (app, _dir) = admin_system_test_app("system-source-build").await;

    let info = get_update_detail(&app, true, "req_system_source_build").await;

    assert!(
        info["currentVersion"] == VERSION_OLD
            && info["latestVersion"] == VERSION_NEW
            && info["hasUpdate"] == true
            && info["buildType"] == "source"
            && info["buildTypeLabel"] == "源码构建"
            && info["updateSupported"] == false
            && info["unsupportedReason"] == "一键更新需要正式构建包"
    );
}

#[tokio::test]
async fn update_detail_should_reject_untrusted_github_api_base() {
    let _guard = SYSTEM_ENV_LOCK.lock().await;
    set_system_update_env(
        "zyycn/codex-proxy-rs-untrusted-api-base-route",
        "https://mirror.example",
    );
    let (app, _dir) = admin_system_test_app("system-untrusted-api-base").await;

    let info = get_update_detail(&app, true, "req_system_untrusted_api_base").await;

    assert!(
        info["hasUpdate"] == false
            && info["updateSupported"] == false
            && info["unsupportedReason"] == "GitHub API base must be https://api.github.com/repos"
    );
}

#[tokio::test]
async fn restart_should_request_process_restart_inside_docker() {
    let _guard = SYSTEM_ENV_LOCK.lock().await;
    set_system_update_env("zyycn/codex-proxy-rs-restart-route", "http://127.0.0.1:9");
    std::env::set_var("CPR_ENABLE_SELF_RESTART", "true");
    let (app, _dir) = admin_system_test_app("system-restart").await;

    let response = app
        .oneshot(
            Request::builder()
                .method("POST")
                .uri("/api/admin/system/restart")
                .header("cookie", "cpr_admin_session=session_1")
                .header("x-request-id", "req_system_restart")
                .body(Body::empty())
                .unwrap(),
        )
        .await
        .unwrap();
    let body = response_json(response).await;
    let shutdown = tokio::time::timeout(
        Duration::from_secs(2),
        codex_proxy_rs::bootstrap::shutdown::shutdown_signal(),
    )
    .await;

    assert!(
        body["data"]["operationId"]
            .as_str()
            .is_some_and(|id| id.starts_with("sysop-restart-"))
            && body["data"]["message"] == "已安排进程内重启"
            && shutdown.is_ok()
    );
}

#[tokio::test]
async fn restart_should_spawn_replacement_before_shutdown_outside_docker() {
    let _guard = SYSTEM_ENV_LOCK.lock().await;
    let replacement = Path::new("/bin/true");
    if !replacement.exists() {
        return;
    }
    set_system_update_env(
        "zyycn/codex-proxy-rs-binary-restart-route",
        "http://127.0.0.1:9",
    );
    std::env::set_var("CPR_DEPLOYMENT_MODE", "binary");
    std::env::set_var("CPR_ENABLE_SELF_RESTART", "true");
    std::env::set_var("CPR_UPDATE_EXE_PATH", replacement);
    let (app, _dir) = admin_system_test_app("system-binary-restart").await;

    let response = app
        .oneshot(
            Request::builder()
                .method("POST")
                .uri("/api/admin/system/restart")
                .header("cookie", "cpr_admin_session=session_1")
                .header("x-request-id", "req_system_binary_restart")
                .body(Body::empty())
                .unwrap(),
        )
        .await
        .unwrap();
    let body = response_json(response).await;
    let shutdown = tokio::time::timeout(
        Duration::from_secs(2),
        codex_proxy_rs::bootstrap::shutdown::shutdown_signal(),
    )
    .await;

    assert!(body["data"]["message"] == "已安排自重启" && shutdown.is_ok());
}

#[tokio::test]
async fn restart_should_not_shutdown_when_replacement_spawn_fails() {
    let _guard = SYSTEM_ENV_LOCK.lock().await;
    set_system_update_env(
        "zyycn/codex-proxy-rs-restart-spawn-failure-route",
        "http://127.0.0.1:9",
    );
    std::env::set_var("CPR_DEPLOYMENT_MODE", "binary");
    std::env::set_var("CPR_ENABLE_SELF_RESTART", "true");
    std::env::set_var(
        "CPR_UPDATE_EXE_PATH",
        "/tmp/codex-proxy-rs-missing-restart-binary",
    );
    let (app, _dir) = admin_system_test_app("system-restart-spawn-failure").await;

    let response = app
        .oneshot(
            Request::builder()
                .method("POST")
                .uri("/api/admin/system/restart")
                .header("cookie", "cpr_admin_session=session_1")
                .header("x-request-id", "req_system_restart_spawn_failure")
                .body(Body::empty())
                .unwrap(),
        )
        .await
        .unwrap();
    let status = response.status();
    let body = response_json(response).await;
    let shutdown = tokio::time::timeout(
        Duration::from_millis(100),
        codex_proxy_rs::bootstrap::shutdown::shutdown_signal(),
    )
    .await;
    let message = body["message"].as_str().unwrap_or_default();

    assert!(
        status == StatusCode::INTERNAL_SERVER_ERROR
            && message.starts_with("Failed to schedule replacement process:")
            && shutdown.is_err()
    );
}

#[tokio::test]
async fn update_should_replace_local_release_files_with_latest_asset() {
    let _guard = SYSTEM_ENV_LOCK.lock().await;
    let repository = "zyycn/codex-proxy-rs-update-route";
    let github = MockServer::start().await;
    let deploy = tempfile::tempdir().unwrap();
    let exe_path = deploy.path().join("codex-proxy-rs");
    let web_dist = deploy.path().join("web/dist");
    std::fs::create_dir_all(&web_dist).unwrap();
    std::fs::write(&exe_path, "old-binary").unwrap();
    std::fs::write(web_dist.join("index.html"), "old-web").unwrap();
    mount_latest_release_with_archive(&github, repository, "0.4.0", "new-binary", "new-web").await;
    set_system_update_env(repository, &github.uri());
    set_system_update_paths(deploy.path(), &exe_path, &web_dist);
    let (app, _dir) = admin_system_test_app("system-update").await;

    let response = app
        .clone()
        .oneshot(
            Request::builder()
                .method("POST")
                .uri("/api/admin/system/update")
                .header("cookie", "cpr_admin_session=session_1")
                .header("x-request-id", "req_system_update_latest")
                .header(CONTENT_TYPE, "application/json")
                .body(confirmed_update_body("0.4.0"))
                .unwrap(),
        )
        .await
        .unwrap();
    let body = response_json(response).await;
    let status = wait_for_operation_status(&app, "succeeded").await;

    assert!(
        body["data"]["operationId"]
            .as_str()
            .is_some_and(|id| id.starts_with("sysop-update-"))
            && body["data"]["targetVersion"] == "0.4.0"
            && body["data"]["needRestart"] == true
            && status["operation"]["status"] == "succeeded"
            && status["currentVersion"] == "0.4.0"
            && std::fs::read_to_string(&exe_path).unwrap() == "new-binary"
            && std::fs::read_to_string(web_dist.join("index.html")).unwrap() == "new-web"
            && std::fs::read_to_string(deploy.path().join("codex-proxy-rs.backup")).unwrap()
                == "old-binary"
            && std::fs::read_to_string(deploy.path().join("web/dist.backup/index.html")).unwrap()
                == "old-web"
    );
}

#[tokio::test]
async fn update_should_reject_when_confirmed_target_differs_from_remote_latest() {
    let _guard = SYSTEM_ENV_LOCK.lock().await;
    let repository = "zyycn/codex-proxy-rs-update-target-changed-route";
    let initial_github = MockServer::start().await;
    let latest_github = MockServer::start().await;
    let deploy = tempfile::tempdir().unwrap();
    let exe_path = deploy.path().join("codex-proxy-rs");
    let web_dist = deploy.path().join("web/dist");
    std::fs::create_dir_all(&web_dist).unwrap();
    std::fs::write(&exe_path, "old-binary").unwrap();
    std::fs::write(web_dist.join("index.html"), "old-web").unwrap();
    mount_latest_release(&initial_github, repository, "0.4.0").await;
    mount_latest_release(&latest_github, repository, "0.5.0").await;
    set_system_update_env(repository, &initial_github.uri());
    set_system_update_paths(deploy.path(), &exe_path, &web_dist);
    let (app, _dir) = admin_system_test_app("system-update-target-changed").await;
    let initial = get_update_detail(&app, true, "req_system_update_target_initial").await;
    set_system_update_env(repository, &latest_github.uri());

    let response = app
        .clone()
        .oneshot(
            Request::builder()
                .method("POST")
                .uri("/api/admin/system/update")
                .header("cookie", "cpr_admin_session=session_1")
                .header("x-request-id", "req_system_update_target_changed")
                .header(CONTENT_TYPE, "application/json")
                .body(confirmed_update_body("0.4.0"))
                .unwrap(),
        )
        .await
        .unwrap();
    let status = response.status();
    let body = response_json(response).await;

    assert!(
        initial["latestVersion"] == "0.4.0"
            && status == StatusCode::CONFLICT
            && body["message"] == "远端最新版本已变更为 v0.5.0，请重新检查并确认"
            && std::fs::read_to_string(&exe_path).unwrap() == "old-binary"
            && std::fs::read_to_string(web_dist.join("index.html")).unwrap() == "old-web"
    );
}

#[tokio::test]
async fn update_should_reject_untrusted_github_api_base() {
    let _guard = SYSTEM_ENV_LOCK.lock().await;
    let deploy = tempfile::tempdir().unwrap();
    let exe_path = deploy.path().join("codex-proxy-rs");
    let web_dist = deploy.path().join("web/dist");
    std::fs::create_dir_all(&web_dist).unwrap();
    std::fs::write(&exe_path, "old-binary").unwrap();
    std::fs::write(web_dist.join("index.html"), "old-web").unwrap();
    set_system_update_env(
        "zyycn/codex-proxy-rs-update-untrusted-api-base-route",
        "https://mirror.example",
    );
    set_system_update_paths(deploy.path(), &exe_path, &web_dist);
    let (app, _dir) = admin_system_test_app("system-update-untrusted-api-base").await;

    let response = app
        .oneshot(
            Request::builder()
                .method("POST")
                .uri("/api/admin/system/update")
                .header("cookie", "cpr_admin_session=session_1")
                .header("x-request-id", "req_system_update_untrusted_api_base")
                .body(Body::empty())
                .unwrap(),
        )
        .await
        .unwrap();
    let status = response.status();
    let body = response_json(response).await;

    assert!(
        status == StatusCode::CONFLICT
            && body["message"] == "GitHub API base must be https://api.github.com/repos"
            && std::fs::read_to_string(&exe_path).unwrap() == "old-binary"
            && std::fs::read_to_string(web_dist.join("index.html")).unwrap() == "old-web"
    );
}

#[tokio::test]
async fn update_should_fail_when_release_checksum_is_missing() {
    let _guard = SYSTEM_ENV_LOCK.lock().await;
    let repository = "zyycn/codex-proxy-rs-missing-checksum-route";
    let github = MockServer::start().await;
    let deploy = tempfile::tempdir().unwrap();
    let exe_path = deploy.path().join("codex-proxy-rs");
    let web_dist = deploy.path().join("web/dist");
    std::fs::create_dir_all(&web_dist).unwrap();
    std::fs::write(&exe_path, "old-binary").unwrap();
    std::fs::write(web_dist.join("index.html"), "old-web").unwrap();
    mount_latest_release_with_archive_without_checksum(
        &github,
        repository,
        "0.4.0",
        "new-binary",
        "new-web",
    )
    .await;
    set_system_update_env(repository, &github.uri());
    set_system_update_paths(deploy.path(), &exe_path, &web_dist);
    let (app, _dir) = admin_system_test_app("system-update-missing-checksum").await;

    assert_update_failure_preserves_files(
        &app,
        "req_system_update_missing_checksum",
        StatusCode::BAD_GATEWAY,
        "Release checksums.txt is required",
        &exe_path,
        &web_dist,
    )
    .await;
}

#[tokio::test]
async fn update_should_fail_when_release_checksum_mismatches() {
    let _guard = SYSTEM_ENV_LOCK.lock().await;
    let repository = "zyycn/codex-proxy-rs-bad-checksum-route";
    let github = MockServer::start().await;
    let deploy = tempfile::tempdir().unwrap();
    let exe_path = deploy.path().join("codex-proxy-rs");
    let web_dist = deploy.path().join("web/dist");
    std::fs::create_dir_all(&web_dist).unwrap();
    std::fs::write(&exe_path, "old-binary").unwrap();
    std::fs::write(web_dist.join("index.html"), "old-web").unwrap();
    mount_latest_release_with_archive_bad_checksum(
        &github,
        repository,
        "0.4.0",
        "new-binary",
        "new-web",
    )
    .await;
    set_system_update_env(repository, &github.uri());
    set_system_update_paths(deploy.path(), &exe_path, &web_dist);
    let (app, _dir) = admin_system_test_app("system-update-bad-checksum").await;

    assert_update_failure_preserves_files(
        &app,
        "req_system_update_bad_checksum",
        StatusCode::BAD_GATEWAY,
        "Checksum mismatch",
        &exe_path,
        &web_dist,
    )
    .await;
}

#[tokio::test]
async fn update_should_reject_release_archive_from_untrusted_host() {
    let _guard = SYSTEM_ENV_LOCK.lock().await;
    let repository = "zyycn/codex-proxy-rs-untrusted-download-route";
    let github = MockServer::start().await;
    let deploy = tempfile::tempdir().unwrap();
    let exe_path = deploy.path().join("codex-proxy-rs");
    let web_dist = deploy.path().join("web/dist");
    std::fs::create_dir_all(&web_dist).unwrap();
    std::fs::write(&exe_path, "old-binary").unwrap();
    std::fs::write(web_dist.join("index.html"), "old-web").unwrap();
    mount_latest_release_with_archive_urls(
        &github,
        repository,
        "0.4.0",
        "https://evil.example/download/codex-proxy-rs.tar.gz",
        "https://github.com/zyycn/codex-proxy-rs/releases/download/v0.4.0/checksums.txt",
    )
    .await;
    set_system_update_env(repository, &github.uri());
    set_system_update_paths(deploy.path(), &exe_path, &web_dist);
    let (app, _dir) = admin_system_test_app("system-update-untrusted-download").await;

    assert_update_failure_preserves_files(
        &app,
        "req_system_update_untrusted_download",
        StatusCode::BAD_REQUEST,
        "Download host is not allowed: evil.example",
        &exe_path,
        &web_dist,
    )
    .await;
}

#[tokio::test]
async fn update_should_reject_insecure_release_archive_even_when_env_flag_is_set() {
    let _guard = SYSTEM_ENV_LOCK.lock().await;
    let repository = "zyycn/codex-proxy-rs-insecure-download-route";
    let github = MockServer::start().await;
    let deploy = tempfile::tempdir().unwrap();
    let exe_path = deploy.path().join("codex-proxy-rs");
    let web_dist = deploy.path().join("web/dist");
    std::fs::create_dir_all(&web_dist).unwrap();
    std::fs::write(&exe_path, "old-binary").unwrap();
    std::fs::write(web_dist.join("index.html"), "old-web").unwrap();
    mount_latest_release_with_archive_urls(
        &github,
        repository,
        "0.4.0",
        "http://github.com/zyycn/codex-proxy-rs/releases/download/v0.4.0/codex-proxy-rs.tar.gz",
        "https://github.com/zyycn/codex-proxy-rs/releases/download/v0.4.0/checksums.txt",
    )
    .await;
    set_system_update_env(repository, &github.uri());
    std::env::set_var("CPR_UPDATE_ALLOW_INSECURE_DOWNLOADS", "true");
    set_system_update_paths(deploy.path(), &exe_path, &web_dist);
    let (app, _dir) = admin_system_test_app("system-update-insecure-download").await;

    assert_update_failure_preserves_files(
        &app,
        "req_system_update_insecure_download",
        StatusCode::BAD_REQUEST,
        "Only HTTPS release downloads are allowed",
        &exe_path,
        &web_dist,
    )
    .await;
}

#[tokio::test]
async fn update_should_reject_release_archive_with_unsafe_path() {
    let _guard = SYSTEM_ENV_LOCK.lock().await;
    let repository = "zyycn/codex-proxy-rs-unsafe-archive-route";
    let github = MockServer::start().await;
    let deploy = tempfile::tempdir().unwrap();
    let exe_path = deploy.path().join("codex-proxy-rs");
    let web_dist = deploy.path().join("web/dist");
    std::fs::create_dir_all(&web_dist).unwrap();
    std::fs::write(&exe_path, "old-binary").unwrap();
    std::fs::write(web_dist.join("index.html"), "old-web").unwrap();
    mount_latest_release_with_unsafe_archive(&github, repository, "0.4.0", "new-binary", "new-web")
        .await;
    set_system_update_env(repository, &github.uri());
    set_system_update_paths(deploy.path(), &exe_path, &web_dist);
    let (app, _dir) = admin_system_test_app("system-update-unsafe-archive").await;

    assert_update_failure_preserves_files(
        &app,
        "req_system_update_unsafe_archive",
        StatusCode::BAD_REQUEST,
        "Unsafe archive path",
        &exe_path,
        &web_dist,
    )
    .await;
}

#[tokio::test]
async fn update_should_restore_web_assets_when_binary_backup_fails() {
    let _guard = SYSTEM_ENV_LOCK.lock().await;
    let repository = "zyycn/codex-proxy-rs-binary-backup-failure-route";
    let github = MockServer::start().await;
    let deploy = tempfile::tempdir().unwrap();
    let exe_path = deploy.path().join("codex-proxy-rs");
    let web_dist = deploy.path().join("web/dist");
    std::fs::create_dir_all(&web_dist).unwrap();
    std::fs::write(&exe_path, "old-binary").unwrap();
    std::fs::write(web_dist.join("index.html"), "old-web").unwrap();
    std::fs::create_dir(deploy.path().join("codex-proxy-rs.backup")).unwrap();
    mount_latest_release_with_archive(&github, repository, "0.4.0", "new-binary", "new-web").await;
    set_system_update_env(repository, &github.uri());
    set_system_update_paths(deploy.path(), &exe_path, &web_dist);
    let (app, _dir) = admin_system_test_app("system-update-binary-backup-failure").await;

    let response = app
        .clone()
        .oneshot(
            Request::builder()
                .method("POST")
                .uri("/api/admin/system/update")
                .header("cookie", "cpr_admin_session=session_1")
                .header("x-request-id", "req_system_update_binary_backup_failure")
                .header(CONTENT_TYPE, "application/json")
                .body(confirmed_update_body("0.4.0"))
                .unwrap(),
        )
        .await
        .unwrap();
    let status_code = response.status();
    let body = response_json(response).await;
    let status = wait_for_operation_status(&app, "failed").await;
    let message = body["message"].as_str().unwrap_or_default();
    let state_error = status["operation"]["error"].as_str().unwrap_or_default();

    assert!(
        status_code == StatusCode::INTERNAL_SERVER_ERROR
            && message.starts_with("Failed to remove old binary backup:")
            && state_error.starts_with("Failed to remove old binary backup:")
            && std::fs::read_to_string(&exe_path).unwrap() == "old-binary"
            && std::fs::read_to_string(web_dist.join("index.html")).unwrap() == "old-web"
            && !deploy.path().join("web/dist.backup").exists(),
        "status={status_code}, message={message}, state_error={state_error}, exe={}, web={}, web_backup_exists={}",
        std::fs::read_to_string(&exe_path).unwrap_or_default(),
        std::fs::read_to_string(web_dist.join("index.html")).unwrap_or_default(),
        deploy.path().join("web/dist.backup").exists()
    );
}

#[tokio::test]
async fn update_should_remove_stale_file_lock_and_continue() {
    let _guard = SYSTEM_ENV_LOCK.lock().await;
    let repository = "zyycn/codex-proxy-rs-stale-lock-route";
    let github = MockServer::start().await;
    let deploy = tempfile::tempdir().unwrap();
    let exe_path = deploy.path().join("codex-proxy-rs");
    let web_dist = deploy.path().join("web/dist");
    let lock_path = deploy.path().join(".runtime/update.lock");
    std::fs::create_dir_all(&web_dist).unwrap();
    std::fs::create_dir_all(lock_path.parent().unwrap()).unwrap();
    std::fs::write(&exe_path, "old-binary").unwrap();
    std::fs::write(web_dist.join("index.html"), "old-web").unwrap();
    std::fs::write(&lock_path, "pid=1\ncreated_at=stale\n").unwrap();
    std::fs::File::options()
        .write(true)
        .open(&lock_path)
        .unwrap()
        .set_modified(SystemTime::now() - Duration::from_secs(31 * 60))
        .unwrap();
    mount_latest_release_with_archive(&github, repository, "0.4.0", "new-binary", "new-web").await;
    set_system_update_env(repository, &github.uri());
    set_system_update_paths(deploy.path(), &exe_path, &web_dist);
    let (app, _dir) = admin_system_test_app("system-update-stale-lock").await;

    let response = app
        .clone()
        .oneshot(
            Request::builder()
                .method("POST")
                .uri("/api/admin/system/update")
                .header("cookie", "cpr_admin_session=session_1")
                .header("x-request-id", "req_system_update_stale_lock")
                .header(CONTENT_TYPE, "application/json")
                .body(confirmed_update_body("0.4.0"))
                .unwrap(),
        )
        .await
        .unwrap();
    let body = response_json(response).await;
    let status = wait_for_operation_status(&app, "succeeded").await;

    assert!(
        body["data"]["targetVersion"] == "0.4.0"
            && status["operation"]["status"] == "succeeded"
            && std::fs::read_to_string(&exe_path).unwrap() == "new-binary"
            && std::fs::read_to_string(web_dist.join("index.html")).unwrap() == "new-web"
            && !lock_path.exists()
    );
}

#[tokio::test]
async fn update_should_replace_web_assets_across_filesystems() {
    let _guard = SYSTEM_ENV_LOCK.lock().await;
    let Ok(web_root) = tempfile::tempdir_in("/dev/shm") else {
        return;
    };
    let repository = "zyycn/codex-proxy-rs-cross-device-update-route";
    let github = MockServer::start().await;
    let deploy = tempfile::tempdir().unwrap();
    let exe_path = deploy.path().join("codex-proxy-rs");
    let web_dist = web_root.path().join("web/dist");
    std::fs::create_dir_all(&web_dist).unwrap();
    std::fs::write(&exe_path, "old-binary").unwrap();
    std::fs::write(web_dist.join("index.html"), "old-web").unwrap();
    mount_latest_release_with_archive(&github, repository, "0.4.0", "new-binary", "new-web").await;
    set_system_update_env(repository, &github.uri());
    set_system_update_paths(deploy.path(), &exe_path, &web_dist);
    let (app, _dir) = admin_system_test_app("system-cross-device-update").await;

    let response = app
        .clone()
        .oneshot(
            Request::builder()
                .method("POST")
                .uri("/api/admin/system/update")
                .header("cookie", "cpr_admin_session=session_1")
                .header("x-request-id", "req_system_cross_device_update")
                .header(CONTENT_TYPE, "application/json")
                .body(confirmed_update_body("0.4.0"))
                .unwrap(),
        )
        .await
        .unwrap();
    let body = response_json(response).await;
    let status = wait_for_operation_status(&app, "succeeded").await;

    assert!(
        body["data"]["targetVersion"] == "0.4.0"
            && status["operation"]["status"] == "succeeded"
            && std::fs::read_to_string(&exe_path).unwrap() == "new-binary"
            && std::fs::read_to_string(web_dist.join("index.html")).unwrap() == "new-web"
            && std::fs::read_to_string(web_root.path().join("web/dist.backup/index.html")).unwrap()
                == "old-web"
    );
}

#[tokio::test]
async fn update_status_should_read_local_update_state() {
    let _guard = SYSTEM_ENV_LOCK.lock().await;
    let deploy = tempfile::tempdir().unwrap();
    set_system_update_env("zyycn/codex-proxy-rs-status-route", "http://127.0.0.1:9");
    set_system_update_paths(
        deploy.path(),
        &deploy.path().join("codex-proxy-rs"),
        &deploy.path().join("web/dist"),
    );
    std::fs::create_dir_all(deploy.path().join(".runtime")).unwrap();
    std::fs::write(
        deploy.path().join(".runtime/update-state.json"),
        json!({
            "previousVersion": VERSION_OLD,
            "currentVersion": VERSION_NEW,
            "operation": {
                "operationId": "system-update-status",
                "kind": "update",
                "status": "running",
                "targetVersion": VERSION_NEW,
                "message": "operation running",
                "error": null,
                "startedAt": "2026-07-01T00:00:00Z",
                "finishedAt": null
            }
        })
        .to_string(),
    )
    .unwrap();
    let (app, _dir) = admin_system_test_app("system-status").await;

    let response = app
        .oneshot(
            Request::builder()
                .method("GET")
                .uri("/api/admin/system/update-status")
                .header("cookie", "cpr_admin_session=session_1")
                .header("x-request-id", "req_system_update_status")
                .body(Body::empty())
                .unwrap(),
        )
        .await
        .unwrap();
    let body = response_json(response).await;

    assert!(
        body["data"]["operation"]["operationId"] == "system-update-status"
            && body["data"]["operation"]["status"] == "running"
            && body["data"]["currentVersion"] == VERSION_NEW
    );
}

#[tokio::test]
async fn update_events_should_open_authenticated_sse_stream() {
    let _guard = SYSTEM_ENV_LOCK.lock().await;
    let deploy = tempfile::tempdir().unwrap();
    set_system_update_env("zyycn/codex-proxy-rs-events-route", "http://127.0.0.1:9");
    set_system_update_paths(
        deploy.path(),
        &deploy.path().join("codex-proxy-rs"),
        &deploy.path().join("web/dist"),
    );
    let (app, _dir) = admin_system_test_app("system-events").await;

    let response = app
        .oneshot(
            Request::builder()
                .method("GET")
                .uri("/api/admin/system/update-events")
                .header("cookie", "cpr_admin_session=session_1")
                .header("x-request-id", "req_system_update_events")
                .body(Body::empty())
                .unwrap(),
        )
        .await
        .unwrap();
    let content_type = response
        .headers()
        .get(CONTENT_TYPE)
        .and_then(|value| value.to_str().ok())
        .unwrap_or_default();

    assert!(response.status() == StatusCode::OK && content_type.starts_with("text/event-stream"));
}

#[tokio::test]
async fn update_events_should_close_on_shutdown() {
    let _guard = SYSTEM_ENV_LOCK.lock().await;
    let deploy = tempfile::tempdir().unwrap();
    set_system_update_env(
        "zyycn/codex-proxy-rs-events-shutdown-route",
        "http://127.0.0.1:9",
    );
    set_system_update_paths(
        deploy.path(),
        &deploy.path().join("codex-proxy-rs"),
        &deploy.path().join("web/dist"),
    );
    let (app, _dir) = admin_system_test_app("system-events-shutdown").await;

    let response = app
        .oneshot(
            Request::builder()
                .method("GET")
                .uri("/api/admin/system/update-events")
                .header("cookie", "cpr_admin_session=session_1")
                .header("x-request-id", "req_system_update_events_shutdown")
                .body(Body::empty())
                .unwrap(),
        )
        .await
        .unwrap();
    codex_proxy_rs::bootstrap::shutdown::request_shutdown();

    let body = tokio::time::timeout(Duration::from_secs(2), to_bytes(response.into_body(), 1024))
        .await
        .expect("update event stream should end after shutdown")
        .unwrap();

    assert!(body.is_empty());
}

#[tokio::test]
async fn update_events_should_close_after_terminal_update_log() {
    let _guard = SYSTEM_ENV_LOCK.lock().await;
    let repository = "zyycn/codex-proxy-rs-events-terminal-route";
    let github = MockServer::start().await;
    let deploy = tempfile::tempdir().unwrap();
    let exe_path = deploy.path().join("codex-proxy-rs");
    let web_dist = deploy.path().join("web/dist");
    std::fs::create_dir_all(&web_dist).unwrap();
    std::fs::write(&exe_path, "old-binary").unwrap();
    std::fs::write(web_dist.join("index.html"), "old-web").unwrap();
    mount_latest_release_with_archive(&github, repository, "0.4.0", "new-binary", "new-web").await;
    set_system_update_env(repository, &github.uri());
    set_system_update_paths(deploy.path(), &exe_path, &web_dist);
    let (app, _dir) = admin_system_test_app("system-events-terminal").await;

    let sse_response = app
        .clone()
        .oneshot(
            Request::builder()
                .method("GET")
                .uri("/api/admin/system/update-events")
                .header("cookie", "cpr_admin_session=session_1")
                .header("x-request-id", "req_system_update_events_terminal")
                .body(Body::empty())
                .unwrap(),
        )
        .await
        .unwrap();
    let update_app = app.clone();
    let update_task = tokio::spawn(async move {
        update_app
            .oneshot(
                Request::builder()
                    .method("POST")
                    .uri("/api/admin/system/update")
                    .header("cookie", "cpr_admin_session=session_1")
                    .header("x-request-id", "req_system_update_events_terminal_update")
                    .header(CONTENT_TYPE, "application/json")
                    .body(confirmed_update_body("0.4.0"))
                    .unwrap(),
            )
            .await
            .unwrap()
    });

    let body = tokio::time::timeout(
        Duration::from_secs(2),
        to_bytes(sse_response.into_body(), 128 * 1024),
    )
    .await
    .expect("update event stream should end after terminal update log")
    .unwrap();
    let update_response = update_task.await.unwrap();
    let text = String::from_utf8(body.to_vec()).unwrap();

    assert!(
        update_response.status() == StatusCode::OK
            && text.contains("\"step\":\"done\"")
            && text.contains("\"terminal\":true")
            && text.contains("更新文件已替换，等待服务重启生效")
    );
}

async fn get_update_detail(app: &axum::Router, refresh: bool, request_id: &str) -> Value {
    let response = app
        .clone()
        .oneshot(
            Request::builder()
                .method("GET")
                .uri(format!("/api/admin/system/update-detail?refresh={refresh}"))
                .header("cookie", "cpr_admin_session=session_1")
                .header("x-request-id", request_id)
                .body(Body::empty())
                .unwrap(),
        )
        .await
        .unwrap();
    assert_eq!(response.status(), StatusCode::OK);
    response_json(response).await["data"].clone()
}

fn confirmed_update_body(version: &str) -> Body {
    Body::from(json!({ "targetVersion": version }).to_string())
}

async fn assert_update_failure_preserves_files(
    app: &axum::Router,
    request_id: &str,
    expected_status: StatusCode,
    expected_message: &str,
    exe_path: &Path,
    web_dist: &Path,
) {
    let response = app
        .clone()
        .oneshot(
            Request::builder()
                .method("POST")
                .uri("/api/admin/system/update")
                .header("cookie", "cpr_admin_session=session_1")
                .header("x-request-id", request_id)
                .header(CONTENT_TYPE, "application/json")
                .body(confirmed_update_body("0.4.0"))
                .unwrap(),
        )
        .await
        .unwrap();
    let status_code = response.status();
    let body = response_json(response).await;
    let status = wait_for_operation_status(app, "failed").await;

    assert!(
        status_code == expected_status
            && body["message"] == expected_message
            && status["operation"]["error"] == expected_message
            && std::fs::read_to_string(exe_path).unwrap() == "old-binary"
            && std::fs::read_to_string(web_dist.join("index.html")).unwrap() == "old-web"
    );
}

async fn mount_latest_release(server: &MockServer, repository: &str, version: &str) {
    Mock::given(method("GET"))
        .and(path(format!("/repos/{repository}/releases/latest")))
        .respond_with(ResponseTemplate::new(200).set_body_json(json!({
            "tag_name": format!("v{version}"),
            "name": format!("v{version}"),
            "body": "## Changes\n- Update package",
            "html_url": format!("https://github.com/{repository}/releases/tag/v{version}"),
            "prerelease": false,
            "assets": []
        })))
        .mount(server)
        .await;
}

async fn mount_latest_release_with_archive(
    server: &MockServer,
    repository: &str,
    version: &str,
    binary: &str,
    index: &str,
) {
    let archive_name = format!(
        "codex-proxy-rs_{version}_{}_{}.tar.gz",
        std::env::consts::OS,
        release_arch()
    );
    let archive = release_archive(binary, index);
    let checksum = format!(
        "{}  {archive_name}\n",
        hex::encode(Sha256::digest(&archive))
    );
    mount_latest_release_archive(
        server,
        repository,
        version,
        archive_name,
        archive,
        Some(checksum),
    )
    .await;
}

async fn mount_latest_release_with_archive_without_checksum(
    server: &MockServer,
    repository: &str,
    version: &str,
    binary: &str,
    index: &str,
) {
    let archive_name = format!(
        "codex-proxy-rs_{version}_{}_{}.tar.gz",
        std::env::consts::OS,
        release_arch()
    );
    let archive = release_archive(binary, index);
    mount_latest_release_archive(server, repository, version, archive_name, archive, None).await;
}

async fn mount_latest_release_with_archive_bad_checksum(
    server: &MockServer,
    repository: &str,
    version: &str,
    binary: &str,
    index: &str,
) {
    let archive_name = format!(
        "codex-proxy-rs_{version}_{}_{}.tar.gz",
        std::env::consts::OS,
        release_arch()
    );
    let archive = release_archive(binary, index);
    let checksum = format!(
        "{}  {archive_name}\n",
        "0".repeat(Sha256::output_size() * 2)
    );
    mount_latest_release_archive(
        server,
        repository,
        version,
        archive_name,
        archive,
        Some(checksum),
    )
    .await;
}

async fn mount_latest_release_with_archive_urls(
    server: &MockServer,
    repository: &str,
    version: &str,
    archive_url: &str,
    checksum_url: &str,
) {
    let archive_name = format!(
        "codex-proxy-rs_{version}_{}_{}.tar.gz",
        std::env::consts::OS,
        release_arch()
    );
    Mock::given(method("GET"))
        .and(path(format!("/repos/{repository}/releases/latest")))
        .respond_with(ResponseTemplate::new(200).set_body_json(json!({
            "tag_name": format!("v{version}"),
            "name": format!("v{version}"),
            "body": "## Changes\n- Update package",
            "html_url": format!("https://github.com/{repository}/releases/tag/v{version}"),
            "prerelease": false,
            "assets": [
                {
                    "name": archive_name,
                    "browser_download_url": archive_url,
                    "size": 128
                },
                {
                    "name": "checksums.txt",
                    "browser_download_url": checksum_url,
                    "size": 128
                }
            ]
        })))
        .mount(server)
        .await;
}

async fn mount_latest_release_with_unsafe_archive(
    server: &MockServer,
    repository: &str,
    version: &str,
    binary: &str,
    index: &str,
) {
    let archive_name = format!(
        "codex-proxy-rs_{version}_{}_{}.tar.gz",
        std::env::consts::OS,
        release_arch()
    );
    let archive = release_archive_with_extra_file(binary, index, "../escape", b"unsafe");
    let checksum = format!(
        "{}  {archive_name}\n",
        hex::encode(Sha256::digest(&archive))
    );
    mount_latest_release_archive(
        server,
        repository,
        version,
        archive_name,
        archive,
        Some(checksum),
    )
    .await;
}

async fn mount_latest_release_archive(
    server: &MockServer,
    repository: &str,
    version: &str,
    archive_name: String,
    archive: Vec<u8>,
    checksum: Option<String>,
) {
    let mut assets = vec![json!({
        "name": archive_name,
        "browser_download_url": format!("{}/download/{archive_name}", server.uri()),
        "size": archive.len()
    })];
    if let Some(checksum) = checksum.as_ref() {
        assets.push(json!({
            "name": "checksums.txt",
            "browser_download_url": format!("{}/download/checksums.txt", server.uri()),
            "size": checksum.len()
        }));
    }

    Mock::given(method("GET"))
        .and(path(format!("/repos/{repository}/releases/latest")))
        .respond_with(ResponseTemplate::new(200).set_body_json(json!({
            "tag_name": format!("v{version}"),
            "name": format!("v{version}"),
            "body": "## Changes\n- Update package",
            "html_url": format!("https://github.com/{repository}/releases/tag/v{version}"),
            "prerelease": false,
            "assets": assets
        })))
        .mount(server)
        .await;
    Mock::given(method("GET"))
        .and(path(format!("/download/{archive_name}")))
        .respond_with(ResponseTemplate::new(200).set_body_bytes(archive))
        .mount(server)
        .await;
    if let Some(checksum) = checksum {
        Mock::given(method("GET"))
            .and(path("/download/checksums.txt"))
            .respond_with(ResponseTemplate::new(200).set_body_string(checksum))
            .mount(server)
            .await;
    }
}

fn release_arch() -> &'static str {
    match std::env::consts::ARCH {
        "x86_64" => "amd64",
        "aarch64" => "arm64",
        arch => arch,
    }
}

fn release_archive(binary: &str, index: &str) -> Vec<u8> {
    release_archive_with_extra_file(binary, index, "", &[])
}

fn release_archive_with_extra_file(
    binary: &str,
    index: &str,
    extra_path: &str,
    extra_bytes: &[u8],
) -> Vec<u8> {
    let mut data = Vec::new();
    {
        let encoder = GzEncoder::new(&mut data, Compression::default());
        let mut builder = Builder::new(encoder);
        append_tar_file(&mut builder, "codex-proxy-rs", binary.as_bytes());
        append_tar_file(&mut builder, "web/dist/index.html", index.as_bytes());
        if !extra_path.is_empty() {
            append_raw_tar_file(&mut builder, extra_path, extra_bytes);
        }
        builder.finish().unwrap();
    }
    data
}

fn append_tar_file<W: Write>(builder: &mut Builder<W>, path: &str, bytes: &[u8]) {
    let mut header = Header::new_gnu();
    header.set_size(bytes.len() as u64);
    header.set_cksum();
    builder.append_data(&mut header, path, bytes).unwrap();
}

fn append_raw_tar_file<W: Write>(builder: &mut Builder<W>, path: &str, bytes: &[u8]) {
    let mut header = Header::new_old();
    header.set_size(bytes.len() as u64);
    header.set_mode(0o644);
    header.set_entry_type(EntryType::Regular);
    let name = path.as_bytes();
    header.as_old_mut().name[..name.len()].copy_from_slice(name);
    header.set_cksum();
    builder.append(&header, bytes).unwrap();
}

fn set_system_update_env(repository: &str, github_api_base: &str) {
    std::env::set_var("CPR_VERSION", "0.1.0");
    std::env::set_var("CPR_DEPLOYMENT_MODE", "docker");
    std::env::set_var("CPR_BUILD_TYPE", "release");
    std::env::set_var("CPR_UPDATE_CHANNEL", "stable");
    std::env::set_var("CPR_UPDATE_REPOSITORY", repository);
    std::env::set_var("CPR_GITHUB_API_BASE", format!("{github_api_base}/repos"));
    std::env::remove_var("CPR_UPDATE_ALLOW_INSECURE_DOWNLOADS");
    std::env::remove_var("CPR_UPDATE_EXE_PATH");
    std::env::remove_var("CPR_WEB_DIST_DIR");
    std::env::remove_var("CPR_UPDATE_STATE_FILE");
    std::env::remove_var("CPR_UPDATE_LOCK_FILE");
    std::env::remove_var("CPR_UPDATE_TEMP_DIR");
    std::env::remove_var("CPR_ENABLE_SELF_RESTART");
}

fn set_system_update_paths(root: &Path, exe_path: &Path, web_dist: &Path) {
    std::env::set_var("CPR_UPDATE_EXE_PATH", exe_path);
    std::env::set_var("CPR_WEB_DIST_DIR", web_dist);
    std::env::set_var(
        "CPR_UPDATE_STATE_FILE",
        root.join(".runtime/update-state.json"),
    );
    std::env::set_var("CPR_UPDATE_LOCK_FILE", root.join(".runtime/update.lock"));
    std::env::set_var("CPR_UPDATE_TEMP_DIR", root.join(".runtime/update-tmp"));
}

async fn wait_for_operation_status(app: &axum::Router, expected: &str) -> Value {
    let response = app
        .clone()
        .oneshot(
            Request::builder()
                .method("GET")
                .uri("/api/admin/system/update-status")
                .header("cookie", "cpr_admin_session=session_1")
                .header("x-request-id", "req_system_update_status_poll")
                .body(Body::empty())
                .unwrap(),
        )
        .await
        .unwrap();
    let data = response_json(response).await["data"].clone();
    assert_eq!(data["operation"]["status"], expected);
    data
}

async fn admin_system_test_app(db_name: &str) -> (axum::Router, tempfile::TempDir) {
    let (pool, dir) = crate::support::storage::init_test_db(db_name).await;
    let redis = crate::support::storage::create_test_redis(db_name).await;
    seed_admin_session(&pool, &redis, "session_1").await;
    let config = test_config(crate::support::storage::test_database_url());
    let stores = crate::support::storage::background_task_stores(pool, redis);
    let services = Arc::new(Services::new(
        &config,
        stores,
        runtime_fingerprint(test_fingerprint()),
    ));
    let state = AppState {
        services: (*services).clone(),
    };
    let app = codex_proxy_rs::api::router::router().with_state(state);
    (app, dir)
}