From dfedb5069d3b2f6c20a2141c962a0d44e105b2e4 Mon Sep 17 00:00:00 2001 From: zyycn Date: Sat, 1 Aug 2026 22:12:23 +0800 Subject: [PATCH] feat(backup): add S3-compatible PostgreSQL backups with expiry TTL Implement end-to-end backup to S3-compatible object storage (Cloudflare R2): admin settings UI, scheduled and manual jobs, records, retention and expiry, download via presigned URL, and delete convergence. - gateway-admin: backup model, schedule/retention policy, daemon worker, storage identity lock and audit - gateway-store: pg_dump staging, S3 client with probe, backup repository - gateway-api: backup settings/records/create/download/delete routes - migrations: fold expiry TTL into 0003_s3_backup.sql (single migration) - frontend: S3 storage, schedule and records cards under settings/backup Co-Authored-By: Claude Fable 5 --- backend/Cargo.lock | 801 ++++++++++++++-- backend/apps/gateway/src/bootstrap.rs | 3 +- backend/crates/gateway-admin/Cargo.toml | 5 + .../crates/gateway-admin/src/backup/mod.rs | 4 + .../crates/gateway-admin/src/backup/policy.rs | 161 ++++ .../crates/gateway-admin/src/backup/task.rs | 602 ++++++++++++ backend/crates/gateway-admin/src/lib.rs | 67 +- .../crates/gateway-admin/src/model/backup.rs | 566 ++++++++++++ backend/crates/gateway-admin/src/model/mod.rs | 11 + .../crates/gateway-admin/src/ports/backup.rs | 442 +++++++++ backend/crates/gateway-admin/src/ports/mod.rs | 1 + .../crates/gateway-admin/src/ports/store.rs | 9 + .../gateway-admin/src/use_case/backup.rs | 437 +++++++++ .../crates/gateway-admin/src/use_case/mod.rs | 1 + backend/crates/gateway-admin/tests/backup.rs | 271 ++++++ .../gateway-admin/tests/backup_runtime.rs | 800 ++++++++++++++++ backend/crates/gateway-admin/tests/main.rs | 2 + .../gateway-admin/tests/use_case/backup.rs | 104 +++ .../gateway-admin/tests/use_case/mod.rs | 10 + backend/crates/gateway-api/Cargo.toml | 1 + .../crates/gateway-api/src/admin/backups.rs | 459 +++++++++ backend/crates/gateway-api/src/admin/mod.rs | 2 + backend/crates/gateway-api/tests/admin/mod.rs | 1 + .../crates/gateway-api/tests/architecture.rs | 1 + backend/crates/gateway-core/src/task/mod.rs | 5 +- .../crates/gateway-host/tests/workers/mod.rs | 3 +- backend/crates/gateway-store/Cargo.toml | 11 +- .../crates/gateway-store/src/backup/mod.rs | 5 + .../gateway-store/src/backup/pg_dump.rs | 205 +++++ backend/crates/gateway-store/src/backup/s3.rs | 544 +++++++++++ .../gateway-store/src/backup/staging.rs | 104 +++ backend/crates/gateway-store/src/lib.rs | 19 + .../gateway-store/src/postgres/backup.rs | 870 ++++++++++++++++++ .../crates/gateway-store/src/postgres/mod.rs | 2 + .../gateway-store/tests/postgres/backup.rs | 308 +++++++ .../gateway-store/tests/postgres/mod.rs | 8 +- backend/migrations/.frozen-sha256 | 1 + backend/migrations/0003_s3_backup.sql | 189 ++++ deploy/Dockerfile | 20 +- deploy/README.md | 32 + frontend/src/api/index.ts | 3 + frontend/src/api/modules/backups.ts | 157 ++++ frontend/src/components/base/BaseCheckbox.vue | 8 + frontend/src/router/routes.ts | 2 +- .../settings/components/ModelAliasesCard.vue | 9 +- .../components/backup/BackupRecordsCard.vue | 218 +++++ .../components/backup/BackupScheduleCard.vue | 97 ++ .../components/backup/BackupStatusBadge.vue | 84 ++ .../components/backup/BackupStorageCard.vue | 183 ++++ .../components/backup/R2GuideModal.vue | 75 ++ .../backup/SettingsBackupSection.vue | 131 +++ .../settings/composables/useAdminApiKey.ts | 7 +- .../settings/composables/useBackupRecords.ts | 192 ++++ .../settings/composables/useBackupSettings.ts | 172 ++++ .../settings/composables/useSettingsForm.ts | 7 +- frontend/src/views/settings/index.vue | 56 +- 56 files changed, 8402 insertions(+), 86 deletions(-) create mode 100644 backend/crates/gateway-admin/src/backup/mod.rs create mode 100644 backend/crates/gateway-admin/src/backup/policy.rs create mode 100644 backend/crates/gateway-admin/src/backup/task.rs create mode 100644 backend/crates/gateway-admin/src/model/backup.rs create mode 100644 backend/crates/gateway-admin/src/ports/backup.rs create mode 100644 backend/crates/gateway-admin/src/use_case/backup.rs create mode 100644 backend/crates/gateway-admin/tests/backup.rs create mode 100644 backend/crates/gateway-admin/tests/backup_runtime.rs create mode 100644 backend/crates/gateway-admin/tests/use_case/backup.rs create mode 100644 backend/crates/gateway-api/src/admin/backups.rs create mode 100644 backend/crates/gateway-store/src/backup/mod.rs create mode 100644 backend/crates/gateway-store/src/backup/pg_dump.rs create mode 100644 backend/crates/gateway-store/src/backup/s3.rs create mode 100644 backend/crates/gateway-store/src/backup/staging.rs create mode 100644 backend/crates/gateway-store/src/postgres/backup.rs create mode 100644 backend/crates/gateway-store/tests/postgres/backup.rs create mode 100644 backend/migrations/0003_s3_backup.sql create mode 100644 frontend/src/api/modules/backups.ts create mode 100644 frontend/src/views/settings/components/backup/BackupRecordsCard.vue create mode 100644 frontend/src/views/settings/components/backup/BackupScheduleCard.vue create mode 100644 frontend/src/views/settings/components/backup/BackupStatusBadge.vue create mode 100644 frontend/src/views/settings/components/backup/BackupStorageCard.vue create mode 100644 frontend/src/views/settings/components/backup/R2GuideModal.vue create mode 100644 frontend/src/views/settings/components/backup/SettingsBackupSection.vue create mode 100644 frontend/src/views/settings/composables/useBackupRecords.ts create mode 100644 frontend/src/views/settings/composables/useBackupSettings.ts diff --git a/backend/Cargo.lock b/backend/Cargo.lock index 6e705cf56..a776ffc20 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -177,6 +177,18 @@ version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" +[[package]] +name = "aws-credential-types" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e93964ffdaf57857f544be3666a5f57570bb699e934700f11b49708f61bb556e" +dependencies = [ + "aws-smithy-async", + "aws-smithy-runtime-api", + "aws-smithy-types", + "zeroize", +] + [[package]] name = "aws-lc-rs" version = "1.17.1" @@ -201,6 +213,332 @@ dependencies = [ "pkg-config", ] +[[package]] +name = "aws-runtime" +version = "1.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9007227e10b5fed2f3e0a2beff489211e2b5604c400b7a9d5d81ca9d64c24bb" +dependencies = [ + "aws-credential-types", + "aws-sigv4", + "aws-smithy-async", + "aws-smithy-eventstream", + "aws-smithy-http", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-types", + "bytes", + "bytes-utils", + "fastrand", + "http 0.2.12", + "http 1.4.2", + "http-body 0.4.6", + "http-body 1.0.1", + "percent-encoding", + "pin-project-lite", + "tracing", + "uuid", +] + +[[package]] +name = "aws-sdk-s3" +version = "1.140.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e9660cf991e512fbe6094f1041ff3d3282bc5aeeeb6184e8de437ec2de024a10" +dependencies = [ + "arc-swap", + "aws-credential-types", + "aws-runtime", + "aws-sigv4", + "aws-smithy-async", + "aws-smithy-checksums", + "aws-smithy-eventstream", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-observability", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-schema", + "aws-smithy-types", + "aws-smithy-xml", + "aws-types", + "bytes", + "fastrand", + "hex", + "hmac 0.13.0", + "http 0.2.12", + "http 1.4.2", + "http-body 1.0.1", + "lru", + "percent-encoding", + "regex-lite", + "sha2 0.11.0", + "tracing", + "url", +] + +[[package]] +name = "aws-sigv4" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "723c2234ad7511ceef63eab016b7ba6ff7c55590fefb96fa8467af014a07309f" +dependencies = [ + "aws-credential-types", + "aws-smithy-eventstream", + "aws-smithy-http", + "aws-smithy-runtime-api", + "aws-smithy-types", + "bytes", + "crypto-bigint", + "form_urlencoded", + "hex", + "hmac 0.13.0", + "http 0.2.12", + "http 1.4.2", + "p256", + "percent-encoding", + "sha2 0.11.0", + "subtle", + "time", + "tracing", + "zeroize", +] + +[[package]] +name = "aws-smithy-async" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f02e407fb3b54891734224b9ffac8a71fdd35f542500fa1af95754a6b2beb316" +dependencies = [ + "futures-util", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "aws-smithy-checksums" +version = "0.65.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b67ecd999972b58e67cab052f5129906c08c25883bd0788ceefc55ef97d61307" +dependencies = [ + "aws-smithy-http", + "aws-smithy-types", + "bytes", + "crc-fast", + "hex", + "http 1.4.2", + "http-body 1.0.1", + "http-body-util", + "md-5", + "pin-project-lite", + "sha1 0.11.0", + "sha2 0.11.0", + "tracing", +] + +[[package]] +name = "aws-smithy-eventstream" +version = "0.61.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a9381123ab62d20c13082b151f30f962a3b112b727345394536dfa39a482944" +dependencies = [ + "aws-smithy-types", + "bytes", + "crc32fast", +] + +[[package]] +name = "aws-smithy-http" +version = "0.64.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37843d9add67c3aff5856f409c6dc315d3cdff60f9c0cb5b670dab1e9920306d" +dependencies = [ + "aws-smithy-eventstream", + "aws-smithy-runtime-api", + "aws-smithy-types", + "bytes", + "bytes-utils", + "futures-core", + "futures-util", + "http 1.4.2", + "http-body 1.0.1", + "http-body-util", + "percent-encoding", + "pin-project-lite", + "pin-utils", + "tracing", +] + +[[package]] +name = "aws-smithy-http-client" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "635d23afda0a6ab48d666c4d447c4873e8d1e83518a2be2093122397e50b838e" +dependencies = [ + "aws-smithy-async", + "aws-smithy-runtime-api", + "aws-smithy-types", + "h2 0.3.27", + "h2 0.4.15", + "http 0.2.12", + "http 1.4.2", + "http-body 0.4.6", + "hyper 0.14.32", + "hyper 1.10.1", + "hyper-rustls 0.24.2", + "hyper-rustls 0.27.9", + "hyper-util", + "pin-project-lite", + "rustls 0.21.12", + "rustls 0.23.42", + "rustls-native-certs", + "rustls-pki-types", + "tokio", + "tokio-rustls 0.26.4", + "tower", + "tracing", +] + +[[package]] +name = "aws-smithy-json" +version = "0.63.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3dc65a121adb4b33729919fcfa14fa36fb33c1555a8f06bb0e2188dbfdc1d9ef" +dependencies = [ + "aws-smithy-runtime-api", + "aws-smithy-schema", + "aws-smithy-types", +] + +[[package]] +name = "aws-smithy-observability" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e86338c869539a581bf161247762a6e87f92c5c075060057b5ed6d06632ed0c" +dependencies = [ + "aws-smithy-runtime-api", +] + +[[package]] +name = "aws-smithy-runtime" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07505b34e8f4b3591a4fa69e9792b52289b95488dbbc68c3c0075b7bedb245e1" +dependencies = [ + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-http-client", + "aws-smithy-observability", + "aws-smithy-runtime-api", + "aws-smithy-schema", + "aws-smithy-types", + "bytes", + "fastrand", + "http 0.2.12", + "http 1.4.2", + "http-body 0.4.6", + "http-body 1.0.1", + "http-body-util", + "pin-project-lite", + "pin-utils", + "tokio", + "tracing", +] + +[[package]] +name = "aws-smithy-runtime-api" +version = "1.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b98f2e1fd67ec06618f9c291e5e495a468e60519e44c9c1979cd0521f3affdb" +dependencies = [ + "aws-smithy-async", + "aws-smithy-runtime-api-macros", + "aws-smithy-types", + "bytes", + "http 0.2.12", + "http 1.4.2", + "pin-project-lite", + "tokio", + "tracing", + "zeroize", +] + +[[package]] +name = "aws-smithy-runtime-api-macros" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "221eaa237ddf1ca79b60d1372aad77e47f9c0ea5b3ce5099da8c61d027dc77b3" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "aws-smithy-schema" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d56e0a4e53127a632224e43633b0fe045fa9e1e3cfc68b9830f1115e103f910" +dependencies = [ + "aws-smithy-runtime-api", + "aws-smithy-types", + "http 1.4.2", +] + +[[package]] +name = "aws-smithy-types" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6dc683efb34b9e755675b37fedbe0103141e5b6df7bdc9eb6967756a8c167d8" +dependencies = [ + "base64-simd", + "bytes", + "bytes-utils", + "futures-core", + "http 0.2.12", + "http 1.4.2", + "http-body 0.4.6", + "http-body 1.0.1", + "http-body-util", + "itoa", + "num-integer", + "pin-project-lite", + "pin-utils", + "ryu", + "serde", + "time", + "tokio", + "tokio-util", +] + +[[package]] +name = "aws-smithy-xml" +version = "0.62.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce84f71c72fee2cbbadde6e7d082f5fb466e3a84733855295fa7aafd1b31b7d8" +dependencies = [ + "aws-smithy-runtime-api", + "aws-smithy-schema", + "aws-smithy-types", + "xmlparser", +] + +[[package]] +name = "aws-types" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eec1cd5469f328c782dc3e33d4153cf118a54e33cbb3356d60d16f89883e1f94" +dependencies = [ + "aws-credential-types", + "aws-smithy-async", + "aws-smithy-runtime-api", + "aws-smithy-schema", + "aws-smithy-types", + "rustc_version", + "tracing", +] + [[package]] name = "axum" version = "0.8.9" @@ -212,10 +550,10 @@ dependencies = [ "bytes", "form_urlencoded", "futures-util", - "http", - "http-body", + "http 1.4.2", + "http-body 1.0.1", "http-body-util", - "hyper", + "hyper 1.10.1", "hyper-util", "itoa", "matchit", @@ -244,8 +582,8 @@ checksum = "08c78f31d7b1291f7ee735c1c6780ccde7785daae9a9206026862dab7d8792d1" dependencies = [ "bytes", "futures-core", - "http", - "http-body", + "http 1.4.2", + "http-body 1.0.1", "http-body-util", "mime", "pin-project-lite", @@ -281,6 +619,16 @@ version = "0.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b25655df2c3cdd83c5e5b293b88acd880332b2ddadd7c30ac43144fdc0033da9" +[[package]] +name = "base64-simd" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "339abbe78e73178762e23bea9dfd08e697eb3f3301cd4be981c0f78ba5859195" +dependencies = [ + "outref", + "vsimd", +] + [[package]] name = "base64ct" version = "1.8.3" @@ -362,6 +710,16 @@ version = "1.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" +[[package]] +name = "bytes-utils" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dafe3a8757b027e2be6e4e5601ed563c55989fcf1546e933c66c8eb3a058d35" +dependencies = [ + "bytes", + "either", +] + [[package]] name = "cc" version = "1.2.67" @@ -411,6 +769,16 @@ dependencies = [ "windows-link", ] +[[package]] +name = "chrono-tz" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6139a8597ed92cf816dfb33f5dd6cf0bb93a6adc938f11039f371bc5bcd26c3" +dependencies = [ + "chrono", + "phf 0.12.1", +] + [[package]] name = "cipher" version = "0.4.4" @@ -507,7 +875,7 @@ checksum = "b85f248a4de22d204ceabc6299d89d2c70fbd7f09fea53c06c852369652d8139" dependencies = [ "pathdiff", "serde_core", - "winnow", + "winnow 1.0.3", "yaml-rust2", ] @@ -601,6 +969,16 @@ version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "217698eaf96b4a3f0bc4f3662aaa55bdf913cd54d7204591faa790070c6d0853" +[[package]] +name = "crc-fast" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e75b2483e97a5a7da73ac68a05b629f9c53cff58d8ed1c77866079e18b00dba5" +dependencies = [ + "digest 0.10.7", + "spin 0.10.1", +] + [[package]] name = "crc32fast" version = "1.5.0" @@ -610,6 +988,18 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "cron" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5dcd6f69605c2956916ce24e8af637b754964c9a83f4662d3a2361654cdba09" +dependencies = [ + "chrono", + "once_cell", + "phf 0.11.3", + "winnow 0.7.15", +] + [[package]] name = "crossbeam-channel" version = "0.5.16" @@ -1041,7 +1431,7 @@ checksum = "5e139bc46ca777eb5efaf62df0ab8cc5fd400866427e56c68b22e414e53bd3be" dependencies = [ "futures-core", "futures-sink", - "spin", + "spin 0.9.9", ] [[package]] @@ -1065,6 +1455,16 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "fs2" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9564fc758e15025b46aa6643b1b77d047d1a56a1aea6e01002ac0c7026876213" +dependencies = [ + "libc", + "winapi", +] + [[package]] name = "fs_extra" version = "1.3.0" @@ -1184,6 +1584,8 @@ dependencies = [ "async-trait", "base64 0.23.0", "chrono", + "chrono-tz", + "cron", "futures", "gateway-core", "hex", @@ -1191,7 +1593,9 @@ dependencies = [ "secrecy", "serde", "serde_json", + "sha2 0.11.0", "subtle", + "tempfile", "thiserror", "tokio", "tracing", @@ -1211,6 +1615,7 @@ dependencies = [ "gateway-admin", "gateway-core", "gateway-protocol", + "secrecy", "serde", "serde_json", "thiserror", @@ -1283,16 +1688,23 @@ name = "gateway-store" version = "0.1.0" dependencies = [ "async-trait", + "aws-credential-types", + "aws-sdk-s3", + "aws-smithy-runtime-api", + "aws-smithy-types", "chrono", + "fs2", "futures", "gateway-admin", "gateway-core", "hex", "redis", + "secrecy", "serde", "serde_json", "sha2 0.11.0", "sqlx", + "tempfile", "thiserror", "tokio", "tracing", @@ -1361,6 +1773,25 @@ dependencies = [ "subtle", ] +[[package]] +name = "h2" +version = "0.3.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0beca50380b1fc32983fc1cb4587bfa4bb9e78fc259aad4a0032d2080309222d" +dependencies = [ + "bytes", + "fnv", + "futures-core", + "futures-sink", + "futures-util", + "http 0.2.12", + "indexmap", + "slab", + "tokio", + "tokio-util", + "tracing", +] + [[package]] name = "h2" version = "0.4.15" @@ -1372,7 +1803,7 @@ dependencies = [ "fnv", "futures-core", "futures-sink", - "http", + "http 1.4.2", "indexmap", "slab", "tokio", @@ -1415,7 +1846,7 @@ dependencies = [ "base64 0.22.1", "bytes", "headers-core", - "http", + "http 1.4.2", "httpdate", "mime", "sha1 0.10.7", @@ -1427,7 +1858,7 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "54b4a22553d4242c49fddb9ba998a99962b5cc6f22cb5a3482bec22522403ce4" dependencies = [ - "http", + "http 1.4.2", ] [[package]] @@ -1475,6 +1906,17 @@ dependencies = [ "digest 0.11.3", ] +[[package]] +name = "http" +version = "0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "601cbb57e577e2f5ef5be8e7b83f0f63994f25aa94d673e54a92d5c516d101f1" +dependencies = [ + "bytes", + "fnv", + "itoa", +] + [[package]] name = "http" version = "1.4.2" @@ -1485,6 +1927,17 @@ dependencies = [ "itoa", ] +[[package]] +name = "http-body" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ceab25649e9960c0311ea418d17bee82c0dcec1bd053b5f9a66e265a693bed2" +dependencies = [ + "bytes", + "http 0.2.12", + "pin-project-lite", +] + [[package]] name = "http-body" version = "1.0.1" @@ -1492,7 +1945,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" dependencies = [ "bytes", - "http", + "http 1.4.2", ] [[package]] @@ -1503,8 +1956,8 @@ checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a" dependencies = [ "bytes", "futures-core", - "http", - "http-body", + "http 1.4.2", + "http-body 1.0.1", "pin-project-lite", ] @@ -1535,6 +1988,30 @@ dependencies = [ "typenum", ] +[[package]] +name = "hyper" +version = "0.14.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41dfc780fdec9373c01bae43289ea34c972e40ee3c9f6b3c8801a35f35586ce7" +dependencies = [ + "bytes", + "futures-channel", + "futures-core", + "futures-util", + "h2 0.3.27", + "http 0.2.12", + "http-body 0.4.6", + "httparse", + "httpdate", + "itoa", + "pin-project-lite", + "socket2 0.5.10", + "tokio", + "tower-service", + "tracing", + "want", +] + [[package]] name = "hyper" version = "1.10.1" @@ -1545,9 +2022,9 @@ dependencies = [ "bytes", "futures-channel", "futures-core", - "h2", - "http", - "http-body", + "h2 0.4.15", + "http 1.4.2", + "http-body 1.0.1", "httparse", "httpdate", "itoa", @@ -1557,19 +2034,34 @@ dependencies = [ "want", ] +[[package]] +name = "hyper-rustls" +version = "0.24.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec3efd23720e2049821a693cbc7e65ea87c72f1c58ff2f9522ff332b1491e590" +dependencies = [ + "futures-util", + "http 0.2.12", + "hyper 0.14.32", + "log", + "rustls 0.21.12", + "tokio", + "tokio-rustls 0.24.1", +] + [[package]] name = "hyper-rustls" version = "0.27.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "33ca68d021ef39cf6463ab54c1d0f5daf03377b70561305bb89a8f83aab66e0f" dependencies = [ - "http", - "hyper", + "http 1.4.2", + "hyper 1.10.1", "hyper-util", - "rustls", + "rustls 0.23.42", "rustls-native-certs", "tokio", - "tokio-rustls", + "tokio-rustls 0.26.4", "tower-service", ] @@ -1583,14 +2075,14 @@ dependencies = [ "bytes", "futures-channel", "futures-util", - "http", - "http-body", - "hyper", + "http 1.4.2", + "http-body 1.0.1", + "hyper 1.10.1", "ipnet", "libc", "percent-encoding", "pin-project-lite", - "socket2", + "socket2 0.6.4", "tokio", "tower-service", "tracing", @@ -1848,6 +2340,15 @@ version = "0.4.33" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad" +[[package]] +name = "lru" +version = "0.16.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f66e8d5d03f609abc3a39e6f08e4164ebf1447a732906d39eb9b99b7919ef39" +dependencies = [ + "hashbrown 0.16.1", +] + [[package]] name = "lru-slab" version = "0.1.2" @@ -1993,6 +2494,12 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" +[[package]] +name = "outref" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a80800c0488c3a21695ea981a54918fbb37abf04f4d0720c453632255e2ff0e" + [[package]] name = "p256" version = "0.13.2" @@ -2076,12 +2583,78 @@ version = "2.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" +[[package]] +name = "phf" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd6780a80ae0c52cc120a26a1a42c1ae51b247a253e4e06113d23d2c2edd078" +dependencies = [ + "phf_macros", + "phf_shared 0.11.3", +] + +[[package]] +name = "phf" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "913273894cec178f401a31ec4b656318d95473527be05c0752cc41cdc32be8b7" +dependencies = [ + "phf_shared 0.12.1", +] + +[[package]] +name = "phf_generator" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c80231409c20246a13fddb31776fb942c38553c51e871f8cbd687a4cfb5843d" +dependencies = [ + "phf_shared 0.11.3", + "rand 0.8.7", +] + +[[package]] +name = "phf_macros" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f84ac04429c13a7ff43785d75ad27569f2951ce0ffd30a3321230db2fc727216" +dependencies = [ + "phf_generator", + "phf_shared 0.11.3", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "phf_shared" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67eabc2ef2a60eb7faa00097bd1ffdb5bd28e62bf39990626a582201b7a754e5" +dependencies = [ + "siphasher", +] + +[[package]] +name = "phf_shared" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06005508882fb681fd97892ecff4b7fd0fee13ef1aa569f8695dae7ab9099981" +dependencies = [ + "siphasher", +] + [[package]] name = "pin-project-lite" version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" +[[package]] +name = "pin-utils" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" + [[package]] name = "pkcs8" version = "0.10.2" @@ -2182,7 +2755,7 @@ dependencies = [ "jsonwebtoken", "reqwest", "roxmltree", - "rustls", + "rustls 0.23.42", "rustls-native-certs", "rustls-pki-types", "secrecy", @@ -2261,8 +2834,8 @@ dependencies = [ "quinn-proto", "quinn-udp", "rustc-hash", - "rustls", - "socket2", + "rustls 0.23.42", + "socket2 0.6.4", "thiserror", "tokio", "tracing", @@ -2282,7 +2855,7 @@ dependencies = [ "rand_pcg", "ring", "rustc-hash", - "rustls", + "rustls 0.23.42", "rustls-pki-types", "slab", "thiserror", @@ -2300,7 +2873,7 @@ dependencies = [ "cfg_aliases", "libc", "once_cell", - "socket2", + "socket2 0.6.4", "tracing", "windows-sys 0.61.2", ] @@ -2326,6 +2899,15 @@ version = "6.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" +[[package]] +name = "rand" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22f6172bdec972074665ed81ed53b71da00bfc44b65a753cfde883ec4c702a1a" +dependencies = [ + "rand_core 0.6.4", +] + [[package]] name = "rand" version = "0.9.4" @@ -2410,7 +2992,7 @@ dependencies = [ "pin-project-lite", "ryu", "sha1_smol", - "socket2", + "socket2 0.6.4", "tokio", "tokio-util", "url", @@ -2449,6 +3031,12 @@ dependencies = [ "regex-syntax", ] +[[package]] +name = "regex-lite" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cab834c73d247e67f4fae452806d17d3c7501756d98c8808d7c9c7aa7d18f973" + [[package]] name = "regex-syntax" version = "0.8.11" @@ -2467,19 +3055,19 @@ dependencies = [ "cookie_store", "futures-core", "futures-util", - "h2", - "http", - "http-body", + "h2 0.4.15", + "http 1.4.2", + "http-body 1.0.1", "http-body-util", - "hyper", - "hyper-rustls", + "hyper 1.10.1", + "hyper-rustls 0.27.9", "hyper-util", "js-sys", "log", "percent-encoding", "pin-project-lite", "quinn", - "rustls", + "rustls 0.23.42", "rustls-native-certs", "rustls-pki-types", "serde", @@ -2487,7 +3075,7 @@ dependencies = [ "serde_urlencoded", "sync_wrapper", "tokio", - "tokio-rustls", + "tokio-rustls 0.26.4", "tokio-util", "tower", "tower-http 0.6.11", @@ -2560,16 +3148,29 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "rustls" +version = "0.21.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f56a14d1f48b391359b22f731fd4bd7e43c97f3c50eee276f3aa09c94784d3e" +dependencies = [ + "log", + "ring", + "rustls-webpki 0.101.7", + "sct", +] + [[package]] name = "rustls" version = "0.23.42" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3c54fcab019b409d04215d3a17cb438fd7fbf192ee61461f20f4fe18704bc138" dependencies = [ + "aws-lc-rs", "once_cell", "ring", "rustls-pki-types", - "rustls-webpki", + "rustls-webpki 0.103.13", "subtle", "zeroize", ] @@ -2596,12 +3197,23 @@ dependencies = [ "zeroize", ] +[[package]] +name = "rustls-webpki" +version = "0.101.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b6275d1ee7a1cd780b64aca7726599a1dbc893b1e64144529e55c3c2f745765" +dependencies = [ + "ring", + "untrusted 0.9.0", +] + [[package]] name = "rustls-webpki" version = "0.103.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "61c429a8649f110dddef65e2a5ad240f747e85f7758a6bccc7e5777bd33f756e" dependencies = [ + "aws-lc-rs", "ring", "rustls-pki-types", "untrusted 0.9.0", @@ -2643,6 +3255,16 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" +[[package]] +name = "sct" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da046153aa2352493d6cb7da4b6e5c0c057d8a1d0a9aa8560baffdd945acd414" +dependencies = [ + "ring", + "untrusted 0.9.0", +] + [[package]] name = "sec1" version = "0.7.3" @@ -2872,6 +3494,12 @@ dependencies = [ "time", ] +[[package]] +name = "siphasher" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ee5873ec9cce0195efcb7a4e9507a04cd49aec9c83d0389df45b1ef7ba2e649" + [[package]] name = "slab" version = "0.4.12" @@ -2887,6 +3515,16 @@ dependencies = [ "serde", ] +[[package]] +name = "socket2" +version = "0.5.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e22376abed350d73dd1cd119b57ffccad95b4e585a7cda43e286245ce23c0678" +dependencies = [ + "libc", + "windows-sys 0.52.0", +] + [[package]] name = "socket2" version = "0.6.4" @@ -2906,6 +3544,12 @@ dependencies = [ "lock_api", ] +[[package]] +name = "spin" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "023a211cb3138dbc438680b32560ad89f699977624c9f8dbb95a47d5b4c07dd3" + [[package]] name = "spki" version = "0.7.3" @@ -2963,7 +3607,7 @@ dependencies = [ "log", "memchr", "percent-encoding", - "rustls", + "rustls 0.23.42", "serde", "serde_json", "sha2 0.10.9", @@ -3293,7 +3937,7 @@ dependencies = [ "mio", "pin-project-lite", "signal-hook-registry", - "socket2", + "socket2 0.6.4", "tokio-macros", "windows-sys 0.61.2", ] @@ -3309,13 +3953,23 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "tokio-rustls" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c28327cf380ac148141087fbfb9de9d7bd4e84ab5d2c28fbc911d753de8a7081" +dependencies = [ + "rustls 0.21.12", + "tokio", +] + [[package]] name = "tokio-rustls" version = "0.26.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1729aa945f29d91ba541258c8df89027d5792d85a8841fb65e8bf0f4ede4ef61" dependencies = [ - "rustls", + "rustls 0.23.42", "tokio", ] @@ -3337,11 +3991,11 @@ source = "git+https://github.com/openai-oss-forks/tokio-tungstenite?rev=0e5b2d73 dependencies = [ "futures-util", "log", - "rustls", + "rustls 0.23.42", "rustls-native-certs", "rustls-pki-types", "tokio", - "tokio-rustls", + "tokio-rustls 0.26.4", "tungstenite 0.27.0", ] @@ -3398,8 +4052,8 @@ dependencies = [ "bytes", "futures-core", "futures-util", - "http", - "http-body", + "http 1.4.2", + "http-body 1.0.1", "http-body-util", "pin-project-lite", "tokio", @@ -3420,8 +4074,8 @@ dependencies = [ "bytes", "futures-core", "futures-util", - "http", - "http-body", + "http 1.4.2", + "http-body 1.0.1", "http-body-util", "http-range-header", "httpdate", @@ -3552,11 +4206,11 @@ dependencies = [ "data-encoding", "flate2", "headers", - "http", + "http 1.4.2", "httparse", "log", "rand 0.9.4", - "rustls", + "rustls 0.23.42", "rustls-pki-types", "sha1 0.10.7", "thiserror", @@ -3571,7 +4225,7 @@ checksum = "6c01152af293afb9c7c2a57e4b559c5620b421f6d133261c60dd2d0cdb38e6b8" dependencies = [ "bytes", "data-encoding", - "http", + "http 1.4.2", "httparse", "log", "rand 0.9.4", @@ -3693,6 +4347,12 @@ version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" +[[package]] +name = "vsimd" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c3082ca00d5a5ef149bb8b555a72ae84c9c59f7250f013ac822ac2e49b19c64" + [[package]] name = "want" version = "0.3.1" @@ -3820,6 +4480,28 @@ version = "2.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "998767ef88740d1f5b0682a9c53c24431453923962269c2db68ee43788c5a40d" +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + [[package]] name = "windows-core" version = "0.62.2" @@ -3961,6 +4643,15 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" +[[package]] +name = "winnow" +version = "0.7.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df79d97927682d2fd8adb29682d1140b343be4ac0f08fd68b7765d9c059d3945" +dependencies = [ + "memchr", +] + [[package]] name = "winnow" version = "1.0.3" @@ -3980,9 +4671,9 @@ dependencies = [ "base64 0.22.1", "deadpool", "futures", - "http", + "http 1.4.2", "http-body-util", - "hyper", + "hyper 1.10.1", "hyper-util", "log", "once_cell", @@ -4015,6 +4706,12 @@ dependencies = [ "rustix", ] +[[package]] +name = "xmlparser" +version = "0.13.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "66fee0b777b0f5ac1c69bb06d361268faafa61cd4682ae064a171c16c433e9e4" + [[package]] name = "xxhash-rust" version = "0.8.16" diff --git a/backend/apps/gateway/src/bootstrap.rs b/backend/apps/gateway/src/bootstrap.rs index 85a8428ff..5e1aa79b4 100644 --- a/backend/apps/gateway/src/bootstrap.rs +++ b/backend/apps/gateway/src/bootstrap.rs @@ -72,7 +72,7 @@ pub async fn run() -> Result<(), BootstrapError> { let providers = ProviderRegistry::new([openai.core_provider(), xai.core_provider()])?; let mut core = gateway_core::initialize(store.core_ports(), providers).await?; host.report_startup_ready("Core"); - let admin = gateway_admin::initialize( + let mut admin = gateway_admin::initialize( admin, store.admin_ports(), vec![openai.admin_provider(), xai.admin_provider()], @@ -99,6 +99,7 @@ pub async fn run() -> Result<(), BootstrapError> { plan.extend(core.take_worker_contributions()); plan.extend(openai.take_worker_contributions()); plan.extend(xai.take_worker_contributions()); + plan.extend(admin.take_worker_contributions()); host.start_workers(plan, store.worker_leader_lease())?; host.report_startup_ready("Workers"); host.serve(api.router()).await?; diff --git a/backend/crates/gateway-admin/Cargo.toml b/backend/crates/gateway-admin/Cargo.toml index f4c08e8b8..c58b46804 100644 --- a/backend/crates/gateway-admin/Cargo.toml +++ b/backend/crates/gateway-admin/Cargo.toml @@ -26,9 +26,14 @@ subtle = "2.6.1" thiserror = "2.0.18" tracing = "0.1.44" uuid = { version = "1.23.3", features = ["v7"] } +cron = "0.17" +chrono-tz = "0.10" +tokio = { version = "1.52", features = ["rt", "time", "macros"] } [dev-dependencies] serde_json = { version = "1.0.150", features = ["arbitrary_precision", "preserve_order"] } +sha2 = "0.11" +tempfile = "3" tokio = { version = "1.52.3", default-features = false, features = ["macros", "rt"] } [lints] diff --git a/backend/crates/gateway-admin/src/backup/mod.rs b/backend/crates/gateway-admin/src/backup/mod.rs new file mode 100644 index 000000000..98eb0ee1e --- /dev/null +++ b/backend/crates/gateway-admin/src/backup/mod.rs @@ -0,0 +1,4 @@ +//! 备份纯策略与 Worker 任务。 + +pub mod policy; +pub mod task; diff --git a/backend/crates/gateway-admin/src/backup/policy.rs b/backend/crates/gateway-admin/src/backup/policy.rs new file mode 100644 index 000000000..ac71063b8 --- /dev/null +++ b/backend/crates/gateway-admin/src/backup/policy.rs @@ -0,0 +1,161 @@ +//! 备份纯策略：Cron 推进与保留候选。 +//! +//! 本模块不持有任何端口或对象，是 `BackupService` 与 `BackupTask` 共用的纯函数集。 + +use std::str::FromStr; + +use chrono::{DateTime, Duration, TimeZone as _, Utc}; +use chrono_tz::Tz; +use cron::Schedule; + +use crate::model::backup::{BackupError, BackupRecord, BackupStatus, code}; + +/// 向历史回溯的最大范围（分钟）。覆盖 Feb 29 + 特定星期等罕见组合的最长间隔。 +const MAX_BACKDATE_MINUTES: i64 = 40 * 366 * 24 * 60; + +/// 已解析并校验的备份计划。 +#[derive(Debug, Clone)] +pub struct BackupSchedule { + schedule: Schedule, + timezone: Tz, +} + +impl BackupSchedule { + /// 解析并校验 5 段 Cron 与 IANA 时区。 + /// + /// # Errors + /// + /// Cron 非法或时区不是 IANA 名称时返回 [`BackupError`]。 + pub fn parse(cron_expression: &str, schedule_timezone: &str) -> Result { + let expression = format!("0 {cron_expression}"); + let schedule = Schedule::from_str(&expression).map_err(|_| { + BackupError::new(code::INVALID_CRON, "Cron 表达式必须是 5 段格式".to_owned()) + })?; + let timezone = schedule_timezone.parse::().map_err(|_| { + BackupError::new(code::INVALID_TIMEZONE, "时区必须是 IANA 名称".to_owned()) + })?; + Ok(Self { schedule, timezone }) + } + + /// 返回最接近 `at`（含 `at` 所在分钟）的最近一次触发时间。 + /// + /// 若过去 `MAX_BACKDATE_MINUTES` 内没有任何触发（极罕见的稀疏计划或超长停机）， + /// 返回 `None`，调用方不应为历史点补任务。 + #[must_use] + pub fn last_firing_at_or_before(&self, at: DateTime) -> Option> { + let high_min = at.timestamp().div_euclid(60); + let low_min = high_min.saturating_sub(MAX_BACKDATE_MINUTES); + + // 检查窗口内是否存在触发；不存在则直接返回 None。 + let low_instant = instant(low_min, self.timezone)?; + if self + .schedule + .after(&low_instant) + .next() + .is_none_or(|firing| firing.timestamp() > high_min * 60 + 59) + { + return None; + } + + // 二分查找最小的 u（分钟）使 after(u) > high；prev = after(u - 1 分钟)。 + let mut lo = low_min; + let mut hi = high_min; + while lo < hi { + let mid = lo + (hi - lo) / 2; + if self + .firing_after(mid) + .is_some_and(|firing| firing > high_min) + { + hi = mid; + } else { + lo = mid + 1; + } + } + self.firing_after(lo - 1) + .filter(|&firing| firing <= high_min) + .and_then(|firing| Utc.timestamp_opt(firing * 60, 0).single()) + } + + /// 返回第一个严格晚于 `at` 的触发时间。 + #[must_use] + pub fn next_after(&self, at: DateTime) -> Option> { + let local = at.with_timezone(&self.timezone); + self.schedule + .after(&local) + .next() + .map(|firing| firing.with_timezone(&Utc)) + } + + /// `after(u 分钟)` 返回的第一个触发时间（分钟索引），用于二分搜索。 + fn firing_after(&self, minute_index: i64) -> Option { + let instant = instant(minute_index, self.timezone)?; + self.schedule + .after(&instant) + .next() + .map(|firing| firing.timestamp().div_euclid(60)) + } +} + +/// 构造指定分钟索引（UTC epoch 分钟）在目标时区下的 `DateTime`。 +fn instant(minute_index: i64, timezone: Tz) -> Option> { + timezone.timestamp_opt(minute_index * 60, 0).single() +} + +/// 一条记录被纳入删除流程的原因。 +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum RetentionReason { + /// 超过 `retention_days`。 + Days, + /// 超过 `retention_count`。 + Count, +} + +/// 保留决策。 +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct RetentionDecision { + pub record_id: String, + pub reason: RetentionReason, +} + +/// 从按完成时间倒序的 completed 计划备份中决定纳入删除流程的记录。 +/// +/// 规则（`docs/s3-backup-design-audit.md` §4.4）： +/// - `retentionDays = 0` 时不按天数清理；`retentionCount = 0` 时不按份数清理。 +/// - 自动计划备份满足任一启用阈值即可进入删除。 +/// - 无论阈值如何，至少保留最近一个成功的计划备份（倒序下标 0）。 +/// - 手动备份不受自动阈值影响；无 TTL，仅由管理员手动删除。 +#[must_use] +pub fn decide_retention( + retention_days: u32, + retention_count: u32, + now: DateTime, + scheduled_completed_desc: &[BackupRecord], +) -> Vec { + let mut decisions = Vec::new(); + for (index, record) in scheduled_completed_desc.iter().enumerate() { + if record.status != BackupStatus::Completed { + continue; + } + // 最近一个成功的计划备份永远保留。 + if index == 0 { + continue; + } + let expired_by_days = retention_days > 0 + && record.completed_at.is_some_and(|completed_at| { + completed_at <= now - Duration::days(i64::from(retention_days)) + }); + let exceeded_count = + retention_count > 0 && index >= usize::try_from(retention_count).unwrap_or(usize::MAX); + if expired_by_days || exceeded_count { + decisions.push(RetentionDecision { + record_id: record.id.clone(), + reason: if expired_by_days { + RetentionReason::Days + } else { + RetentionReason::Count + }, + }); + } + } + decisions +} diff --git a/backend/crates/gateway-admin/src/backup/task.rs b/backend/crates/gateway-admin/src/backup/task.rs new file mode 100644 index 000000000..bd2992026 --- /dev/null +++ b/backend/crates/gateway-admin/src/backup/task.rs @@ -0,0 +1,602 @@ +//! Backup Worker 贡献：单个可取消 `DaemonTask` 承担调度、执行、删除收敛与保留清理。 +//! +//! 当前部署边界是单副本，因此不需要 Redis lease、fencing token 或 heartbeat。长时间 +//! `pg_dump` 与上传由本 Daemon 自身持有；Host 只负责 panic 后重启、健康与关闭。 + +use std::sync::Arc; +use std::time::Duration; + +use chrono::{DateTime, Utc}; +use futures::future::BoxFuture; +use gateway_core::{ + engine::CancellationToken, + task::{DaemonTask, WorkerTaskError}, +}; +use tracing::{info, warn}; + +use crate::{ + model::backup::{ + BackupError, BackupObjectMetadata, BackupRecord, BackupStatus, BackupStorageConfig, + BackupTriggerKind, build_backup_seed, code, + }, + ports::backup::{ + BackupObjectStorePort, BackupRepository, DatabaseDumpPort, DumpRequest, + StatusTransitionUpdate, UploadObjectRequest, + }, +}; + +impl From for WorkerTaskError { + fn from(error: BackupError) -> Self { + WorkerTaskError::safe(error.message()) + } +} + +use super::policy::{BackupSchedule, decide_retention}; + +/// 循环间隔：管理员修改 Cron 后该时长内生效；无工作时可取消等待。 +const BACKUP_LOOP_INTERVAL: Duration = Duration::from_secs(30); +/// 每个循环最多执行的保留删除数量。 +const RETENTION_BATCH_SIZE: usize = 10; +/// 保留扫描一次读取的 completed 计划备份上限。 +const RETENTION_SCAN_LIMIT: u32 = 1000; +/// 每个循环最多完成的待删除记录数量。 +const PENDING_DELETION_BATCH: u32 = 20; + +/// 备份 Daemon 任务。 +pub struct BackupTask { + repository: Arc, + dump: Arc, + object_store: Arc, +} + +impl BackupTask { + /// 组合仓储、导出器与对象存储适配器。 + #[must_use] + pub fn new( + repository: Arc, + dump: Arc, + object_store: Arc, + ) -> Self { + Self { + repository, + dump, + object_store, + } + } +} + +impl DaemonTask for BackupTask { + fn run(&self, cancellation: CancellationToken) -> BoxFuture<'_, Result<(), WorkerTaskError>> { + Box::pin(async move { self.run_daemon(cancellation).await }) + } +} + +impl BackupTask { + async fn run_daemon(&self, cancellation: CancellationToken) -> Result<(), WorkerTaskError> { + loop { + if cancellation.is_cancelled() { + return Ok(()); + } + self.run_cycle(&cancellation).await?; + + tokio::select! { + _ = cancellation.cancelled() => return Ok(()), + _ = tokio::time::sleep(BACKUP_LOOP_INTERVAL) => {} + } + } + } + + /// 执行一个周期：推进计划、恢复中间态、删除收敛、领取执行一个任务、保留清理。 + /// + /// 对外暴露以便集成测试逐周期驱动；daemon 循环内部复用。 + /// + /// # Errors + /// + /// 仓储或基础设施不可用、任务执行失败时返回 [`WorkerTaskError`]。 + pub async fn run_cycle(&self, cancellation: &CancellationToken) -> Result<(), WorkerTaskError> { + if cancellation.is_cancelled() { + return Ok(()); + } + self.advance_schedule(Utc::now()).await?; + self.recover_intermediate(cancellation).await?; + self.finish_pending_deletions().await?; + self.process_next_task(cancellation).await?; + self.run_retention_batch(Utc::now()).await?; + Ok(()) + } + + /// 推进到期计划并幂等插入 scheduled 任务。 + async fn advance_schedule(&self, now: DateTime) -> Result<(), WorkerTaskError> { + let settings = self.repository.load_settings().await.map_err(repo_error)?; + if !settings.schedule_enabled { + return Ok(()); + } + let cron = settings.cron_expression.as_deref().unwrap_or_default(); + let timezone = settings.schedule_timezone.as_deref().unwrap_or_default(); + if cron.is_empty() || timezone.is_empty() { + return Ok(()); + } + let schedule = BackupSchedule::parse(cron, timezone) + .map_err(|_| WorkerTaskError::safe("backup schedule is invalid"))?; + + let due = match settings.next_run_at { + Some(next_run_at) if next_run_at <= now => true, + Some(_) => false, + None => { + // 刚启用计划但尚无游标：直接初始化。 + if let Some(next_run_at) = schedule.next_after(now) { + let _ = self + .repository + .advance_schedule_cursor(next_run_at, cron, timezone) + .await + .map_err(repo_error)?; + } + return Ok(()); + } + }; + if !due { + return Ok(()); + } + + let scheduled_at = match schedule.last_firing_at_or_before(now) { + Some(scheduled_at) => scheduled_at, + None => { + // 无法从计划推导最近到期时间：推进游标避免反复触发。 + if let Some(next_run_at) = schedule.next_after(now) { + let _ = self + .repository + .advance_schedule_cursor(next_run_at, cron, timezone) + .await + .map_err(repo_error)?; + } + return Ok(()); + } + }; + let prefix = settings.prefix.as_deref().unwrap_or_default(); + let expires_at = if settings.retention_days > 0 { + Some(now + chrono::Duration::days(i64::from(settings.retention_days))) + } else { + None + }; + let seed = build_backup_seed( + BackupTriggerKind::Scheduled, + Some(scheduled_at), + prefix, + now, + expires_at, + ) + .map_err(infra_error)?; + let inserted = self + .repository + .insert_scheduled_record(seed) + .await + .map_err(repo_error)?; + if inserted { + info!(scheduled_at = %scheduled_at, "计划备份任务已创建"); + } else { + warn!(scheduled_at = %scheduled_at, "计划时间点冲突，跳过并推进游标"); + } + if let Some(next_run_at) = schedule.next_after(now) { + let _ = self + .repository + .advance_schedule_cursor(next_run_at, cron, timezone) + .await + .map_err(repo_error)?; + } + Ok(()) + } + + /// 恢复中间状态（dumping/uploading/deleting）。 + async fn recover_intermediate( + &self, + cancellation: &CancellationToken, + ) -> Result<(), WorkerTaskError> { + let records = self + .repository + .list_intermediate_records() + .await + .map_err(repo_error)?; + for record in records { + match record.status { + BackupStatus::Dumping => self.recover_dumping(&record, cancellation).await?, + BackupStatus::Uploading => self.recover_uploading(&record, cancellation).await?, + _ => {} + } + } + Ok(()) + } + + /// dumping：有完整暂存则继续上传，否则清理暂存并标记失败。 + async fn recover_dumping( + &self, + record: &BackupRecord, + cancellation: &CancellationToken, + ) -> Result<(), WorkerTaskError> { + match self.dump.inspect_staging(&record.id).await? { + Some(artifact) => { + let now = Utc::now(); + let update = StatusTransitionUpdate { + size_bytes: Some(artifact.size_bytes), + sha256: Some(artifact.sha256.clone()), + ..Default::default() + }; + self.repository + .transition_status( + &record.id, + BackupStatus::Dumping, + BackupStatus::Uploading, + update, + now, + ) + .await + .map_err(repo_error)?; + self.upload_and_verify( + record, + &artifact.path, + artifact.size_bytes, + &artifact.sha256, + now, + cancellation, + ) + .await?; + } + None => { + let _ = self.dump.cleanup_staging(&record.id).await; + self.fail_task(record, code::PG_DUMP_FAILED, "导出进程中断，暂存归档不完整") + .await?; + } + } + Ok(()) + } + + /// uploading：远端匹配则补记完成，有完整暂存则重试上传，否则失败。 + async fn recover_uploading( + &self, + record: &BackupRecord, + cancellation: &CancellationToken, + ) -> Result<(), WorkerTaskError> { + let settings = self.repository.load_settings().await.map_err(repo_error)?; + let Some(config) = BackupStorageConfig::from_settings(&settings) else { + return Ok(()); + }; + if let Some(remote) = self + .object_store + .head_object(&config, &record.object_key) + .await? + && record.size_bytes == Some(remote.size_bytes) + && record.sha256.as_deref() == Some(remote.sha256.as_str()) + { + let now = Utc::now(); + self.repository + .transition_status( + &record.id, + BackupStatus::Uploading, + BackupStatus::Completed, + StatusTransitionUpdate::default(), + now, + ) + .await + .map_err(repo_error)?; + return Ok(()); + } + match self.dump.inspect_staging(&record.id).await? { + Some(artifact) => { + self.upload_and_verify( + record, + &artifact.path, + artifact.size_bytes, + &artifact.sha256, + Utc::now(), + cancellation, + ) + .await + } + None => { + self.fail_task( + record, + code::S3_UPLOAD_FAILED, + "远端对象缺失且本地无完整暂存", + ) + .await + } + } + } + + /// 完成待删除记录（deleting → DeleteObject → 硬删除）。 + async fn finish_pending_deletions(&self) -> Result<(), WorkerTaskError> { + let settings = self.repository.load_settings().await.map_err(repo_error)?; + let Some(config) = BackupStorageConfig::from_settings(&settings) else { + return Ok(()); + }; + let records = self + .repository + .list_pending_deletions(PENDING_DELETION_BATCH) + .await + .map_err(repo_error)?; + for record in records { + if self + .object_store + .delete_object(&config, &record.object_key) + .await + .is_ok() + { + self.repository + .delete_record(&record.id) + .await + .map_err(repo_error)?; + } + } + Ok(()) + } + + /// 领取一个 queued 任务并执行导出、上传与远端校验。 + async fn process_next_task( + &self, + cancellation: &CancellationToken, + ) -> Result<(), WorkerTaskError> { + let now = Utc::now(); + let Some(record) = self + .repository + .claim_next_queued(now) + .await + .map_err(repo_error)? + else { + return Ok(()); + }; + info!( + backup_id = %record.id, + attempt = record.attempt_count, + "领取备份任务" + ); + if let Err(error) = self.execute_task(&record, cancellation).await { + warn!(backup_id = %record.id, error = %error, "备份任务执行失败"); + } + Ok(()) + } + + /// 执行单个任务；内部处理全部状态迁移。 + async fn execute_task( + &self, + record: &BackupRecord, + cancellation: &CancellationToken, + ) -> Result<(), WorkerTaskError> { + let settings = self.repository.load_settings().await.map_err(repo_error)?; + if BackupStorageConfig::from_settings(&settings).is_none() { + return self + .fail_task(record, code::INVALID_CONFIG, "备份存储配置不完整") + .await; + } + if !settings.storage_verified() { + return self + .fail_task(record, code::INVALID_CONFIG, "备份存储配置尚未通过连接测试") + .await; + } + + let now = Utc::now(); + let artifact = match self + .dump + .dump(DumpRequest { + backup_id: record.id.clone(), + cancellation: cancellation.clone(), + }) + .await + { + Ok(artifact) => artifact, + Err(error) => { + let (error_code, message) = classify_dump_error(error); + return self.fail_task(record, error_code, message).await; + } + }; + + // dumping → uploading，持久化 size/sha256。 + let update = StatusTransitionUpdate { + size_bytes: Some(artifact.size_bytes), + sha256: Some(artifact.sha256.clone()), + ..Default::default() + }; + self.repository + .transition_status( + &record.id, + BackupStatus::Dumping, + BackupStatus::Uploading, + update, + now, + ) + .await + .map_err(repo_error)?; + + self.upload_and_verify( + record, + &artifact.path, + artifact.size_bytes, + &artifact.sha256, + now, + cancellation, + ) + .await?; + let _ = self.dump.cleanup_staging(&record.id).await; + Ok(()) + } + + /// 上传 + 远端校验，内部完成终态迁移。 + async fn upload_and_verify( + &self, + record: &BackupRecord, + source: &std::path::Path, + size_bytes: u64, + sha256: &str, + now: DateTime, + cancellation: &CancellationToken, + ) -> Result<(), WorkerTaskError> { + let settings = self.repository.load_settings().await.map_err(repo_error)?; + let Some(config) = BackupStorageConfig::from_settings(&settings) else { + return Ok(()); + }; + let store = self.object_store.clone(); + let metadata = + BackupObjectMetadata::new(record.id.clone(), sha256.to_owned(), now, size_bytes); + if let Err(error) = store + .upload_file( + &config, + UploadObjectRequest { + object_key: record.object_key.clone(), + source: source.to_path_buf(), + metadata, + cancellation: cancellation.clone(), + }, + ) + .await + { + let _ = store.delete_object(&config, &record.object_key).await; + self.fail_task(record, error.code(), "上传到对象存储失败") + .await?; + return Ok(()); + } + + match store.head_object(&config, &record.object_key).await { + Ok(Some(remote)) if remote.size_bytes == size_bytes && remote.sha256 == sha256 => { + if self + .repository + .transition_status( + &record.id, + BackupStatus::Uploading, + BackupStatus::Completed, + StatusTransitionUpdate::default(), + Utc::now(), + ) + .await + .map_err(repo_error)? + .is_some() + { + info!( + backup_id = %record.id, + size_bytes, + "备份完成" + ); + } + } + Ok(_) => { + let _ = store.delete_object(&config, &record.object_key).await; + self.fail_task( + record, + code::REMOTE_VERIFICATION_FAILED, + "远端对象大小或校验值不一致", + ) + .await?; + } + Err(error) => { + let _ = store.delete_object(&config, &record.object_key).await; + self.fail_task(record, error.code(), "远端对象校验失败") + .await?; + } + } + Ok(()) + } + + /// 把活跃任务写入失败终态并清理暂存。 + async fn fail_task( + &self, + record: &BackupRecord, + error_code: &'static str, + message: impl Into, + ) -> Result<(), WorkerTaskError> { + let now = Utc::now(); + let update = StatusTransitionUpdate { + error_code: Some(error_code.to_string()), + error_message: Some(message.into()), + completed_at: Some(now), + ..Default::default() + }; + let _ = self + .repository + .transition_status(&record.id, record.status, BackupStatus::Failed, update, now) + .await + .map_err(repo_error)?; + let _ = self.dump.cleanup_staging(&record.id).await; + Ok(()) + } + + /// 执行一小批到期保留清理：先处理 `expires_at` 已到期的记录，再按天数/份数清理计划备份。 + async fn run_retention_batch(&self, now: DateTime) -> Result<(), WorkerTaskError> { + let settings = self.repository.load_settings().await.map_err(repo_error)?; + + // 1. expires_at 已到期的记录（手动或计划）无条件进入删除。 + let expired = self + .repository + .list_expired_records(RETENTION_BATCH_SIZE as u32) + .await + .map_err(repo_error)?; + for record in expired { + let Some(record) = self + .repository + .transition_to_deleting(&record.id, now) + .await + .map_err(repo_error)? + else { + continue; + }; + self.finalize_deletion(&record).await?; + } + + // 2. 计划备份按 retentionDays / retentionCount 清理。 + let records = self + .repository + .list_scheduled_completed_desc(RETENTION_SCAN_LIMIT) + .await + .map_err(repo_error)?; + let decisions = decide_retention( + settings.retention_days, + settings.retention_count, + now, + &records, + ); + for decision in decisions.into_iter().take(RETENTION_BATCH_SIZE) { + let Some(record) = self + .repository + .transition_to_deleting(&decision.record_id, now) + .await + .map_err(repo_error)? + else { + continue; + }; + self.finalize_deletion(&record).await?; + } + Ok(()) + } + + /// 对已进入 deleting 的记录执行对象删除并硬删除记录行。 + async fn finalize_deletion(&self, record: &BackupRecord) -> Result<(), WorkerTaskError> { + let settings = self.repository.load_settings().await.map_err(repo_error)?; + let Some(config) = BackupStorageConfig::from_settings(&settings) else { + return Ok(()); + }; + if self + .object_store + .delete_object(&config, &record.object_key) + .await + .is_ok() + { + self.repository + .delete_record(&record.id) + .await + .map_err(repo_error)?; + } + Ok(()) + } +} + +fn classify_dump_error(error: BackupError) -> (&'static str, &'static str) { + match error.code() { + code::CANCELLED => (code::CANCELLED, "备份已取消"), + code::STAGING_SPACE_EXHAUSTED => (code::STAGING_SPACE_EXHAUSTED, "暂存磁盘空间不足"), + _ => (code::PG_DUMP_FAILED, "数据库导出失败"), + } +} + +fn repo_error(error: crate::ports::store::AdminStoreError) -> WorkerTaskError { + WorkerTaskError::safe(format!("backup store failed: {error}")) +} + +fn infra_error(error: BackupError) -> WorkerTaskError { + WorkerTaskError::safe(error.message()) +} diff --git a/backend/crates/gateway-admin/src/lib.rs b/backend/crates/gateway-admin/src/lib.rs index c3fe565a1..2ff747863 100644 --- a/backend/crates/gateway-admin/src/lib.rs +++ b/backend/crates/gateway-admin/src/lib.rs @@ -2,23 +2,28 @@ //! //! 本 crate 不包含 HTTP wire、数据库实现或具体 Provider 实现。 -use std::{fmt, path::Path, sync::Arc}; +use std::{fmt, path::Path, sync::Arc, time::Duration}; use gateway_core::{ engine::probe::AccountProbe, routing::{ProviderKind, snapshot::SnapshotControl}, + task::{ + DaemonRestartPolicy, WorkerContribution, WorkerId, WorkerKind, WorkerRegistration, + WorkerRunnable, + }, }; use secrecy::{ExposeSecret as _, SecretString}; use serde::Deserialize; +pub mod backup; pub mod model; pub mod ports; mod use_case; pub use use_case::{ - accounts::AccountsService, auth::AuthService, client_keys::ClientKeyService, - observability::ObservabilityService, openai::OpenAiService, settings::SettingsService, - system::SystemService, xai::XaiService, + accounts::AccountsService, auth::AuthService, backup::BackupService, + client_keys::ClientKeyService, observability::ObservabilityService, openai::OpenAiService, + settings::SettingsService, system::SystemService, xai::XaiService, }; use model::{AdminError, AdminErrorKind}; @@ -28,7 +33,7 @@ use ports::{ system::SystemOperations, }; use use_case::{ - accounts::DefaultAccountsService, auth::DefaultAuthService, + accounts::DefaultAccountsService, auth::DefaultAuthService, backup::DefaultBackupService, client_keys::DefaultClientKeyService, observability::DefaultObservabilityService, openai::DefaultOpenAiService, settings::DefaultSettingsService, system::DefaultSystemService, xai::DefaultXaiService, @@ -48,6 +53,8 @@ const WEAK_INITIAL_PASSWORDS: &[&str] = &[ "codex-proxy-rs", ]; +const BACKUP_WORKER_OWNER: &str = "backup"; + /// 只用于首次幂等创建默认管理员的启动密码。 #[derive(Clone, Deserialize)] #[serde(transparent)] @@ -146,6 +153,7 @@ pub struct AdminServices { system: Arc, openai: Arc, xai: Arc, + backups: Arc, } impl AdminServices { @@ -188,11 +196,17 @@ impl AdminServices { pub fn xai(&self) -> &dyn XaiService { self.xai.as_ref() } + + #[must_use] + pub fn backups(&self) -> &dyn BackupService { + self.backups.as_ref() + } } /// Admin 初始化完成后的封闭能力包。 pub struct AdminBundle { services: AdminServices, + worker_contributions: Vec, } impl AdminBundle { @@ -200,6 +214,11 @@ impl AdminBundle { pub fn services(&self) -> AdminServices { self.services.clone() } + + /// 取出 Backup Worker 贡献；只能调用一次，与其它 Bundle 的贡献一并交给 Host。 + pub fn take_worker_contributions(&mut self) -> Vec { + std::mem::take(&mut self.worker_contributions) + } } /// 校验配置、建立动态 Provider 注册表并完成默认管理员幂等初始化。 @@ -241,6 +260,18 @@ pub async fn initialize( snapshot.clone(), probe.clone(), )); + let backup_ports = store.backup(); + let backups = Arc::new(DefaultBackupService::new( + backup_ports.repository(), + backup_ports.object_store(), + store.auth(), + snapshot.clone(), + )); + let backup_task = backup::task::BackupTask::new( + backup_ports.repository(), + backup_ports.dump(), + backup_ports.object_store(), + ); let services = AdminServices { auth, accounts, @@ -268,8 +299,32 @@ pub async fn initialize( store.accounts(), snapshot.clone(), )), + backups, }; - Ok(AdminBundle { services }) + let worker_contributions = backup_worker_contribution(backup_task)?; + Ok(AdminBundle { + services, + worker_contributions, + }) +} + +/// Backup Worker 注册：单个可取消 Daemon，owner 固定为 `backup`。 +fn backup_worker_contribution( + task: backup::task::BackupTask, +) -> Result, AdminError> { + let id = WorkerId::try_new(WorkerKind::Backup, BACKUP_WORKER_OWNER) + .map_err(|_| AdminError::internal("Backup worker id is invalid"))?; + let restart = DaemonRestartPolicy::try_new(Duration::from_secs(1), Duration::from_secs(60)) + .map_err(|_| AdminError::internal("Backup worker restart policy is invalid"))?; + let registration = WorkerRegistration::try_new( + id, + WorkerRunnable::Daemon { + restart, + task: Box::new(task), + }, + ) + .map_err(|_| AdminError::internal("Backup worker registration is invalid"))?; + Ok(vec![WorkerContribution::Registration(registration)]) } fn provider_kind(value: &'static str) -> Result { diff --git a/backend/crates/gateway-admin/src/model/backup.rs b/backend/crates/gateway-admin/src/model/backup.rs new file mode 100644 index 000000000..2dc39bcf8 --- /dev/null +++ b/backend/crates/gateway-admin/src/model/backup.rs @@ -0,0 +1,566 @@ +//! S3 备份领域模型、状态机与稳定错误。 +//! +//! 领域类型只携带事实，不承载基础设施细节。Secret 使用 [`secrecy::SecretString`]， +//! 全流程不实现 `Serialize`，避免被错误地写入 wire 或日志。管理员身份等审计事实 +//! 只存在于 `admin_audit_events`，本模型不重复保存。 + +use std::fmt; + +use chrono::{DateTime, Utc}; +use secrecy::SecretString; +use uuid::Uuid; + +use super::PageSize; + +/// 备份触发来源。 +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum BackupTriggerKind { + Manual, + Scheduled, +} + +impl BackupTriggerKind { + /// 稳定持久化值。 + #[must_use] + pub const fn as_str(self) -> &'static str { + match self { + Self::Manual => "manual", + Self::Scheduled => "scheduled", + } + } + + /// 解析持久化值。 + #[must_use] + pub fn parse(value: &str) -> Option { + match value { + "manual" => Some(Self::Manual), + "scheduled" => Some(Self::Scheduled), + _ => None, + } + } +} + +impl fmt::Display for BackupTriggerKind { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str(self.as_str()) + } +} + +/// 备份任务状态机（六状态）。 +/// +/// 迁移 `backup_records_lifecycle_ck` 与字段空值语义共同定义合法持久化行； +/// [`Self::allows_transition_to`] 只约束领域内的显式状态迁移，两层都必须满足。 +/// 删除成功后记录行被硬删除，因此不存在 `deleted` 终态。 +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub enum BackupStatus { + Queued, + Dumping, + Uploading, + Completed, + Failed, + Deleting, +} + +impl BackupStatus { + /// 稳定持久化值。 + #[must_use] + pub const fn as_str(self) -> &'static str { + match self { + Self::Queued => "queued", + Self::Dumping => "dumping", + Self::Uploading => "uploading", + Self::Completed => "completed", + Self::Failed => "failed", + Self::Deleting => "deleting", + } + } + + /// 解析持久化值。 + #[must_use] + pub fn parse(value: &str) -> Option { + match value { + "queued" => Some(Self::Queued), + "dumping" => Some(Self::Dumping), + "uploading" => Some(Self::Uploading), + "completed" => Some(Self::Completed), + "failed" => Some(Self::Failed), + "deleting" => Some(Self::Deleting), + _ => None, + } + } + + /// 该状态是否消耗全局活跃名额。 + #[must_use] + pub const fn is_active(self) -> bool { + matches!(self, Self::Queued | Self::Dumping | Self::Uploading) + } + + /// 该状态是否允许进入删除流程。 + #[must_use] + pub const fn can_be_deleted(self) -> bool { + matches!(self, Self::Completed | Self::Failed) + } + + /// 领域内允许的显式迁移。 + #[must_use] + pub fn allows_transition_to(self, target: BackupStatus) -> bool { + matches!( + (self, target), + (Self::Queued, Self::Dumping) + | (Self::Queued, Self::Failed) + | (Self::Dumping, Self::Uploading) + | (Self::Dumping, Self::Failed) + | (Self::Uploading, Self::Completed) + | (Self::Uploading, Self::Failed) + | (Self::Completed, Self::Deleting) + | (Self::Failed, Self::Deleting) + ) + } +} + +impl fmt::Display for BackupStatus { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str(self.as_str()) + } +} + +/// S3 存储配置更新命令；`secret_access_key` 为 `None` 表示保留旧值。 +#[derive(Clone)] +pub struct UpdateBackupStorageCommand { + pub endpoint: String, + pub region: String, + pub bucket: String, + pub access_key_id: String, + pub secret_access_key: Option, + pub prefix: String, + pub force_path_style: bool, +} + +impl fmt::Debug for UpdateBackupStorageCommand { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter + .debug_struct("UpdateBackupStorageCommand") + .field("endpoint", &self.endpoint) + .field("region", &self.region) + .field("bucket", &self.bucket) + .field("access_key_id", &self.access_key_id) + .field("secret_access_key", &self.secret_access_key.is_some()) + .field("prefix", &self.prefix) + .field("force_path_style", &self.force_path_style) + .finish() + } +} + +/// 调度配置更新命令。 +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct UpdateBackupScheduleCommand { + pub schedule_enabled: bool, + pub cron_expression: String, + pub schedule_timezone: String, + pub retention_days: u32, + pub retention_count: u32, +} + +/// 完整保存的备份设置（含 Secret）。 +/// +/// 只从仓储读取；不实现 `Serialize`，防止 Secret 进入 wire 或日志。 +#[derive(Clone)] +pub struct BackupSettings { + pub storage_revision: u64, + pub endpoint: Option, + pub region: Option, + pub bucket: Option, + pub access_key_id: Option, + pub secret_access_key: Option, + pub prefix: Option, + pub force_path_style: bool, + pub schedule_enabled: bool, + pub cron_expression: Option, + pub schedule_timezone: Option, + pub retention_days: u32, + pub retention_count: u32, + pub next_run_at: Option>, + pub last_verified_at: Option>, + pub updated_at: DateTime, +} + +impl fmt::Debug for BackupSettings { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter + .debug_struct("BackupSettings") + .field("storage_revision", &self.storage_revision) + .field("endpoint", &self.endpoint) + .field("region", &self.region) + .field("bucket", &self.bucket) + .field("access_key_id", &self.access_key_id) + .field("secret_access_key", &self.secret_access_key.is_some()) + .field("prefix", &self.prefix) + .field("force_path_style", &self.force_path_style) + .field("schedule_enabled", &self.schedule_enabled) + .field("cron_expression", &self.cron_expression) + .field("schedule_timezone", &self.schedule_timezone) + .field("retention_days", &self.retention_days) + .field("retention_count", &self.retention_count) + .field("next_run_at", &self.next_run_at) + .field("last_verified_at", &self.last_verified_at) + .field("updated_at", &self.updated_at) + .finish() + } +} + +impl BackupSettings { + /// S3 存储配置是否完整（bucket/region/密钥/前缀齐全）。 + #[must_use] + pub fn storage_configured(&self) -> bool { + self.endpoint.is_some() + && self.region.is_some() + && self.bucket.is_some() + && self.access_key_id.is_some() + && self.secret_access_key.is_some() + && self.prefix.is_some() + } + + /// 探测是否已对当前 revision 成功；`last_verified_at` 非空即表示当前配置已通过探针。 + #[must_use] + pub fn storage_verified(&self) -> bool { + self.last_verified_at.is_some() + } +} + +/// 已经校验并可用于对象存储操作的存储配置快照。 +#[derive(Clone)] +pub struct BackupStorageConfig { + pub storage_revision: u64, + pub endpoint: String, + pub region: String, + pub bucket: String, + pub access_key_id: String, + pub secret_access_key: SecretString, + pub prefix: String, + pub force_path_style: bool, +} + +impl fmt::Debug for BackupStorageConfig { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter + .debug_struct("BackupStorageConfig") + .field("storage_revision", &self.storage_revision) + .field("endpoint", &self.endpoint) + .field("region", &self.region) + .field("bucket", &self.bucket) + .field("access_key_id", &self.access_key_id) + .field("secret_access_key", &"[REDACTED]") + .field("prefix", &self.prefix) + .field("force_path_style", &self.force_path_style) + .finish() + } +} + +impl BackupStorageConfig { + /// 从完整设置推导对象存储配置；设置不完整时返回 `None`。 + #[must_use] + pub fn from_settings(settings: &BackupSettings) -> Option { + Some(Self { + storage_revision: settings.storage_revision, + endpoint: settings.endpoint.clone()?, + region: settings.region.clone()?, + bucket: settings.bucket.clone()?, + access_key_id: settings.access_key_id.clone()?, + secret_access_key: settings.secret_access_key.clone()?, + prefix: settings.prefix.clone()?, + force_path_style: settings.force_path_style, + }) + } +} + +/// 一条备份记录。 +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct BackupRecord { + pub id: String, + pub trigger_kind: BackupTriggerKind, + pub status: BackupStatus, + pub scheduled_at: Option>, + pub object_key: String, + pub size_bytes: Option, + pub sha256: Option, + pub attempt_count: u32, + pub error_code: Option, + pub error_message: Option, + pub started_at: Option>, + pub completed_at: Option>, + /// 手动过期时间；创建时确定，到期自动清理。 + pub expires_at: Option>, + pub created_at: DateTime, + pub updated_at: DateTime, +} + +/// 备份记录分页查询。 +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct BackupRecordListQuery { + pub page: u32, + pub page_size: PageSize, + pub status: Option, + pub trigger: Option, +} + +/// 备份记录分页结果。 +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct BackupRecordPage { + pub items: Vec, + pub total: u64, + pub page: u32, + pub page_size: PageSize, +} + +impl BackupRecordPage { + /// 计算总页数；空集合视为一页。 + #[must_use] + pub fn total_pages(&self) -> u32 { + let size = u64::from(self.page_size.get()); + if self.total == 0 { + 1 + } else { + self.total.div_ceil(size).min(u64::from(u32::MAX)) as u32 + } + } +} + +/// 创建备份任务的持久化数据；id 与对象 key 由领域生成。 +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct BackupRecordSeed { + pub id: String, + pub trigger_kind: BackupTriggerKind, + pub scheduled_at: Option>, + pub object_key: String, + pub expires_at: Option>, +} + +/// 连接测试结果；只携带稳定阶段与脱敏消息。 +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ConnectionTestResult { + pub ok: bool, + pub stage: &'static str, + pub code: Option<&'static str>, + pub message: String, +} + +/// 对象存储探测阶段。 +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ConnectionTestStage { + PutObject, + HeadObject, + GetObject, + DeleteObject, +} + +impl ConnectionTestStage { + /// 稳定 wire 值。 + #[must_use] + pub const fn as_str(self) -> &'static str { + match self { + Self::PutObject => "putObject", + Self::HeadObject => "headObject", + Self::GetObject => "getObject", + Self::DeleteObject => "deleteObject", + } + } +} + +/// 上传到对象存储并期望远端校验的归档 metadata。 +/// +/// `size_bytes` 在上传时不写入 metadata，但 `HeadObject` 会返回远端大小， +/// 用于与本地归档校验一致。 +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct BackupObjectMetadata { + pub backup_id: String, + pub sha256: String, + pub created_at: DateTime, + pub size_bytes: u64, +} + +impl BackupObjectMetadata { + /// 构造上传 metadata。 + #[must_use] + pub const fn new( + backup_id: String, + sha256: String, + created_at: DateTime, + size_bytes: u64, + ) -> Self { + Self { + backup_id, + sha256, + created_at, + size_bytes, + } + } +} + +/// 短时下载地址结果。 +#[derive(Clone, PartialEq, Eq)] +pub struct DownloadUrlResult { + pub url: String, + pub file_name: String, + pub expires_in: std::time::Duration, +} + +impl fmt::Debug for DownloadUrlResult { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter + .debug_struct("DownloadUrlResult") + .field("url", &"[REDACTED]") + .field("file_name", &self.file_name) + .field("expires_in", &self.expires_in) + .finish() + } +} + +/// 备份基础设施错误：只携带稳定错误码与脱敏消息。 +/// +/// 该类型不构成第二套 kind 枚举；`gateway-admin` 边界统一映射为既有 +/// `AdminErrorKind`。稳定 `error_code` 同时用于任务记录、日志与 UI。 +#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] +#[error("{message}")] +pub struct BackupError { + code: &'static str, + message: String, +} + +impl BackupError { + /// 构造基础设施错误。 + #[must_use] + pub const fn new(code: &'static str, message: String) -> Self { + Self { code, message } + } + + /// 稳定错误码，例如 `backup.s3_auth_failed`。 + #[must_use] + pub const fn code(&self) -> &'static str { + self.code + } + + /// 脱敏后的可操作消息。 + #[must_use] + pub fn message(&self) -> &str { + &self.message + } +} + +/// 构造稳定错误码常量。 +pub mod code { + /// 配置或参数非法。 + pub const INVALID_CONFIG: &str = "backup.invalid_config"; + /// Cron 表达式非法。 + pub const INVALID_CRON: &str = "backup.invalid_cron"; + /// IANA 时区非法。 + pub const INVALID_TIMEZONE: &str = "backup.invalid_timezone"; + /// 记录不存在。 + pub const RECORD_NOT_FOUND: &str = "backup.record_not_found"; + /// 已存在活跃任务，拒绝创建。 + pub const ACTIVE_TASK_CONFLICT: &str = "backup.active_task_conflict"; + /// 记录状态与期望不符，无法迁移。 + pub const STATE_CONFLICT: &str = "backup.state_conflict"; + /// 已有记录时禁止切换存储目标。 + pub const STORAGE_IDENTITY_LOCKED: &str = "backup.storage_identity_locked"; + /// `pg_dump` 非零退出或失败。 + pub const PG_DUMP_FAILED: &str = "backup.pg_dump_failed"; + /// 暂存磁盘空间不足。 + pub const STAGING_SPACE_EXHAUSTED: &str = "backup.staging_space_exhausted"; + /// S3 认证失败。 + pub const S3_AUTH_FAILED: &str = "backup.s3_auth_failed"; + /// S3 权限拒绝。 + pub const S3_PERMISSION_DENIED: &str = "backup.s3_permission_denied"; + /// S3 上传失败。 + pub const S3_UPLOAD_FAILED: &str = "backup.s3_upload_failed"; + /// 远端对象校验失败。 + pub const REMOTE_VERIFICATION_FAILED: &str = "backup.remote_verification_failed"; + /// 任务被取消。 + pub const CANCELLED: &str = "backup.cancelled"; + /// 仓储或对象存储暂不可用。 + pub const STORE_UNAVAILABLE: &str = "backup.store_unavailable"; + /// 上游 S3 服务返回无效或失败响应。 + pub const UPSTREAM_FAILURE: &str = "backup.upstream_failure"; +} + +/// 构建对象 key：`{prefix}/YYYY/MM/DD/codex-proxy-rs_{UTC seconds}_{backup id}.dump`。 +/// +/// 规范化规则：prefix 不得为空、不得以前导 `/` 开头、不得包含 `..`、反斜杠或控制字符； +/// 输出统一使用 `/` 分隔且不含空段。 +pub fn build_object_key( + prefix: &str, + backup_id: &str, + at: DateTime, +) -> Result { + let prefix = normalize_prefix(prefix)?; + Ok(format!( + "{prefix}/{}/{:02}/{:02}/codex-proxy-rs_{}_{}.dump", + at.format("%Y"), + at.format("%m"), + at.format("%d"), + at.format("%Y%m%d_%H%M%S"), + short_backup_id(backup_id), + )) +} + +/// 下载文件名：`codex-proxy-rs_{短 id}.dump`，与对象 key 的文件名一致。 +#[must_use] +pub fn build_download_file_name(backup_id: &str) -> String { + format!("codex-proxy-rs_{}.dump", short_backup_id(backup_id)) +} + +/// 截取备份 id 的可读短标识：去掉 `backup_` 前缀，最多保留 8 位 hex。 +#[must_use] +fn short_backup_id(backup_id: &str) -> String { + let hex = backup_id.strip_prefix("backup_").unwrap_or(backup_id); + hex.get(..8).unwrap_or(hex).to_owned() +} + +/// 规范化并校验对象 prefix。 +/// +/// 返回去掉尾部 `/` 的规范 prefix；空串、前导 `/`、`..` 段、反斜杠和控制字符均拒绝。 +pub fn normalize_prefix(prefix: &str) -> Result { + if prefix.is_empty() || prefix.starts_with('/') { + return Err(BackupError::new( + code::INVALID_CONFIG, + "存储对象前缀必须非空且不能以前导 / 开头".to_owned(), + )); + } + if prefix.bytes().any(|byte| byte.is_ascii_control()) || prefix.contains('\\') { + return Err(BackupError::new( + code::INVALID_CONFIG, + "存储对象前缀包含非法字符".to_owned(), + )); + } + if prefix.split('/').any(|segment| segment == "..") { + return Err(BackupError::new( + code::INVALID_CONFIG, + "存储对象前缀不能包含 .. 路径段".to_owned(), + )); + } + Ok(prefix.trim_end_matches('/').to_owned()) +} + +/// 生成新的备份记录 id。 +#[must_use] +pub fn new_backup_id() -> String { + format!("backup_{}", Uuid::now_v7().simple()) +} + +/// 从触发类型与计划时间构造可持久化 seed；id 与对象 key 一次生成保持一致。 +pub fn build_backup_seed( + trigger_kind: BackupTriggerKind, + scheduled_at: Option>, + prefix: &str, + at: DateTime, + expires_at: Option>, +) -> Result { + let id = new_backup_id(); + let object_key = build_object_key(prefix, &id, at)?; + Ok(BackupRecordSeed { + id, + trigger_kind, + scheduled_at, + object_key, + expires_at, + }) +} diff --git a/backend/crates/gateway-admin/src/model/mod.rs b/backend/crates/gateway-admin/src/model/mod.rs index cd96d914d..9f4fe1cb3 100644 --- a/backend/crates/gateway-admin/src/model/mod.rs +++ b/backend/crates/gateway-admin/src/model/mod.rs @@ -4,6 +4,7 @@ use std::num::{NonZeroU16, NonZeroU64}; pub mod accounts; pub mod auth; +pub mod backup; pub mod client_keys; pub mod observability; pub mod provider_credentials; @@ -50,6 +51,11 @@ impl AdminError { Self::new(AdminErrorKind::Invalid, message) } + #[must_use] + pub fn bad_request(message: impl Into) -> Self { + Self::new(AdminErrorKind::Invalid, message) + } + #[must_use] pub fn not_found(message: impl Into) -> Self { Self::new(AdminErrorKind::NotFound, message) @@ -60,6 +66,11 @@ impl AdminError { Self::new(AdminErrorKind::Conflict, message) } + #[must_use] + pub fn bad_gateway(message: impl Into) -> Self { + Self::new(AdminErrorKind::BadGateway, message) + } + #[must_use] pub fn unavailable(message: impl Into) -> Self { Self::new(AdminErrorKind::Unavailable, message) diff --git a/backend/crates/gateway-admin/src/ports/backup.rs b/backend/crates/gateway-admin/src/ports/backup.rs new file mode 100644 index 000000000..939c2ea3c --- /dev/null +++ b/backend/crates/gateway-admin/src/ports/backup.rs @@ -0,0 +1,442 @@ +//! 备份控制面所需的持久化与基础设施能力。 +//! +//! 端口方法只接受/返回领域类型与既有错误体系：`BackupRepository` 复用 +//! `AdminStoreError`；`DatabaseDumpPort` 与 `BackupObjectStorePort` 返回只携带稳定 +//! 错误码的 [`BackupError`]。`gateway-store` 实现全部端口；`gateway-admin` 拥有 +//! 状态机、Cron、保留策略与 BackupTask 算法。 + +use std::path::PathBuf; +use std::sync::Arc; +use std::time::Duration; + +use async_trait::async_trait; +use chrono::{DateTime, Utc}; + +use crate::model::Revision; +use crate::model::{ + MutationContext, + backup::{ + BackupError, BackupObjectMetadata, BackupRecord, BackupRecordListQuery, BackupRecordPage, + BackupRecordSeed, BackupSettings, BackupStatus, BackupStorageConfig, ConnectionTestResult, + UpdateBackupScheduleCommand, UpdateBackupStorageCommand, + }, +}; +use crate::ports::store::AdminStoreResult; + +/// Worker 状态迁移携带的字段更新；`None` 字段不写入。 +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct StatusTransitionUpdate { + pub size_bytes: Option, + pub sha256: Option, + pub error_code: Option, + pub error_message: Option, + pub completed_at: Option>, +} + +/// 备份配置、任务与计划游标的 PostgreSQL 仓储端口。 +#[async_trait] +pub trait BackupRepository: Send + Sync { + /// 读取完整设置（含 Secret）。 + async fn load_settings(&self) -> AdminStoreResult; + + /// 同事务更新 S3 存储配置、递增 revision、清空验证状态并写审计事件。 + /// + /// 已有记录时禁止切换 endpoint/region/bucket/path-style（存储身份锁定）。 + /// 返回更新后的设置与新的全局 `config_revision`。 + async fn update_storage_settings( + &self, + command: UpdateBackupStorageCommand, + context: &MutationContext, + ) -> AdminStoreResult<(BackupSettings, Revision)>; + + /// 同事务更新调度配置、推进计划游标并写审计事件。 + async fn update_schedule_settings( + &self, + command: UpdateBackupScheduleCommand, + next_run_at: Option>, + context: &MutationContext, + ) -> AdminStoreResult; + + /// CAS 记录探针成功；`storage_revision` 与当前配置不一致时丢弃结果（返回 `false`）。 + async fn record_verification( + &self, + storage_revision: u64, + at: DateTime, + ) -> AdminStoreResult; + + /// 插入 queued 任务；存在活跃任务时返回 `backup.active_task_conflict`。 + async fn insert_backup_record(&self, seed: BackupRecordSeed) -> AdminStoreResult; + + /// 幂等插入 scheduled 任务；同一时间点或活跃冲突时返回 `false`（调用方推进游标即可）。 + async fn insert_scheduled_record(&self, seed: BackupRecordSeed) -> AdminStoreResult; + + /// 分页查询记录。 + async fn list_backup_records( + &self, + query: BackupRecordListQuery, + ) -> AdminStoreResult; + + /// 读取单条记录。 + async fn load_backup_record(&self, id: &str) -> AdminStoreResult>; + + /// 读取中间状态记录（dumping/uploading），供启动恢复。 + async fn list_intermediate_records(&self) -> AdminStoreResult>; + + /// 读取等待完成墓碑的记录（status = deleting）。 + async fn list_pending_deletions(&self, limit: u32) -> AdminStoreResult>; + + /// 读取 `expires_at` 已到期且非活跃的记录（completed/failed），供保留清理。 + async fn list_expired_records(&self, limit: u32) -> AdminStoreResult>; + + /// 原子领取最旧的 queued 任务并迁移到 dumping；无任务返回 `None`。 + async fn claim_next_queued(&self, now: DateTime) + -> AdminStoreResult>; + + /// 条件迁移状态：要求当前状态一致；不匹配返回 `None`。 + async fn transition_status( + &self, + id: &str, + from: BackupStatus, + to: BackupStatus, + update: StatusTransitionUpdate, + now: DateTime, + ) -> AdminStoreResult>; + + /// 迁移到删除流程（completed/failed → deleting）；供管理员与保留策略调用。 + async fn transition_to_deleting( + &self, + id: &str, + now: DateTime, + ) -> AdminStoreResult>; + + /// 远端删除成功后硬删除记录行。 + async fn delete_record(&self, id: &str) -> AdminStoreResult<()>; + + /// 条件推进计划游标；仅当计划表达式和时区仍是 `expected_*` 时才写入，返回是否推进。 + async fn advance_schedule_cursor( + &self, + next_run_at: DateTime, + expected_cron: &str, + expected_timezone: &str, + ) -> AdminStoreResult; + + /// 按完成时间倒序读取 completed 计划备份，供保留策略扫描（最多 `limit` 条）。 + async fn list_scheduled_completed_desc( + &self, + limit: u32, + ) -> AdminStoreResult>; +} + +/// `pg_dump` 导出与本地暂存端口。 +#[async_trait] +pub trait DatabaseDumpPort: Send + Sync { + /// 创建归档；完成后原子落盘。取消时终止子进程并清理部分文件。 + async fn dump(&self, request: DumpRequest) -> Result; + + /// 检查暂存目录是否存在完整归档。 + async fn inspect_staging(&self, backup_id: &str) + -> Result, BackupError>; + + /// 清理暂存目录中该备份的全部文件。 + async fn cleanup_staging(&self, backup_id: &str) -> Result<(), BackupError>; +} + +/// 一次 `pg_dump` 请求。 +#[derive(Debug, Clone)] +pub struct DumpRequest { + pub backup_id: String, + pub cancellation: gateway_core::engine::CancellationToken, +} + +/// 已完成并落盘的归档事实。 +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct DumpArtifact { + pub path: PathBuf, + pub size_bytes: u64, + pub sha256: String, +} + +/// 暂存目录中已存在的完整归档。 +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct StagedArtifact { + pub path: PathBuf, + pub size_bytes: u64, + pub sha256: String, +} + +/// S3 兼容对象存储适配器端口。 +/// +/// 方法接受已经校验的存储配置；SDK client 的构造与缓存属于 adapter 实现细节。 +#[async_trait] +pub trait BackupObjectStorePort: Send + Sync { + /// 对专用探针对象执行 Put/Head/Get/Delete。 + async fn test_connection( + &self, + config: &BackupStorageConfig, + ) -> Result; + + /// 分片流式上传本地归档；取消时中止 multipart 并返回 `backup.cancelled`。 + async fn upload_file( + &self, + config: &BackupStorageConfig, + request: UploadObjectRequest, + ) -> Result<(), BackupError>; + + /// 读取对象大小与 metadata；对象不存在返回 `Ok(None)`。 + async fn head_object( + &self, + config: &BackupStorageConfig, + object_key: &str, + ) -> Result, BackupError>; + + /// 删除对象；对象不存在视为成功。 + async fn delete_object( + &self, + config: &BackupStorageConfig, + object_key: &str, + ) -> Result<(), BackupError>; + + /// 创建短时预签名下载 URL。 + async fn presigned_download( + &self, + config: &BackupStorageConfig, + object_key: &str, + file_name: &str, + ttl: Duration, + ) -> Result; +} + +/// 一次对象上传请求。 +#[derive(Debug, Clone)] +pub struct UploadObjectRequest { + pub object_key: String, + pub source: PathBuf, + pub metadata: BackupObjectMetadata, + pub cancellation: gateway_core::engine::CancellationToken, +} + +/// 备份控制面所需的全部 Store 能力集合。 +#[derive(Clone)] +pub struct BackupStorePorts { + repository: Arc, + dump: Arc, + object_store: Arc, +} + +impl BackupStorePorts { + /// 组合备份端口。 + #[must_use] + pub fn new( + repository: Arc, + dump: Arc, + object_store: Arc, + ) -> Self { + Self { + repository, + dump, + object_store, + } + } + + /// 备份任务/配置仓储。 + #[must_use] + pub fn repository(&self) -> Arc { + self.repository.clone() + } + + /// `pg_dump` 导出与暂存。 + #[must_use] + pub fn dump(&self) -> Arc { + self.dump.clone() + } + + /// S3/R2 对象存储适配器。 + #[must_use] + pub fn object_store(&self) -> Arc { + self.object_store.clone() + } + + /// 测试与不可用路径使用的空实现；任何调用都返回不可用错误。 + #[doc(hidden)] + #[must_use] + pub fn disabled() -> Self { + Self { + repository: Arc::new(UnavailableBackupRepository), + dump: Arc::new(UnavailableDumpPort), + object_store: Arc::new(UnavailableObjectStore), + } + } +} + +struct UnavailableBackupRepository; + +#[async_trait] +impl BackupRepository for UnavailableBackupRepository { + async fn load_settings(&self) -> AdminStoreResult { + Err(disabled_store()) + } + async fn update_storage_settings( + &self, + _command: UpdateBackupStorageCommand, + _context: &MutationContext, + ) -> AdminStoreResult<(BackupSettings, Revision)> { + Err(disabled_store()) + } + async fn update_schedule_settings( + &self, + _command: UpdateBackupScheduleCommand, + _next_run_at: Option>, + _context: &MutationContext, + ) -> AdminStoreResult { + Err(disabled_store()) + } + async fn record_verification( + &self, + _storage_revision: u64, + _at: DateTime, + ) -> AdminStoreResult { + Err(disabled_store()) + } + async fn insert_backup_record( + &self, + _seed: BackupRecordSeed, + ) -> AdminStoreResult { + Err(disabled_store()) + } + async fn insert_scheduled_record(&self, _seed: BackupRecordSeed) -> AdminStoreResult { + Err(disabled_store()) + } + async fn list_backup_records( + &self, + _query: BackupRecordListQuery, + ) -> AdminStoreResult { + Err(disabled_store()) + } + async fn load_backup_record(&self, _id: &str) -> AdminStoreResult> { + Err(disabled_store()) + } + async fn list_intermediate_records(&self) -> AdminStoreResult> { + Err(disabled_store()) + } + async fn list_pending_deletions(&self, _limit: u32) -> AdminStoreResult> { + Err(disabled_store()) + } + async fn list_expired_records(&self, _limit: u32) -> AdminStoreResult> { + Err(disabled_store()) + } + async fn claim_next_queued( + &self, + _now: DateTime, + ) -> AdminStoreResult> { + Err(disabled_store()) + } + async fn transition_status( + &self, + _id: &str, + _from: BackupStatus, + _to: BackupStatus, + _update: StatusTransitionUpdate, + _now: DateTime, + ) -> AdminStoreResult> { + Err(disabled_store()) + } + async fn transition_to_deleting( + &self, + _id: &str, + _now: DateTime, + ) -> AdminStoreResult> { + Err(disabled_store()) + } + async fn delete_record(&self, _id: &str) -> AdminStoreResult<()> { + Err(disabled_store()) + } + async fn advance_schedule_cursor( + &self, + _next_run_at: DateTime, + _expected_cron: &str, + _expected_timezone: &str, + ) -> AdminStoreResult { + Err(disabled_store()) + } + async fn list_scheduled_completed_desc( + &self, + _limit: u32, + ) -> AdminStoreResult> { + Err(disabled_store()) + } +} + +struct UnavailableDumpPort; + +#[async_trait] +impl DatabaseDumpPort for UnavailableDumpPort { + async fn dump(&self, _request: DumpRequest) -> Result { + Err(disabled_infra()) + } + async fn inspect_staging( + &self, + _backup_id: &str, + ) -> Result, BackupError> { + Err(disabled_infra()) + } + async fn cleanup_staging(&self, _backup_id: &str) -> Result<(), BackupError> { + Err(disabled_infra()) + } +} + +struct UnavailableObjectStore; + +#[async_trait] +impl BackupObjectStorePort for UnavailableObjectStore { + async fn test_connection( + &self, + _config: &BackupStorageConfig, + ) -> Result { + Err(disabled_infra()) + } + async fn upload_file( + &self, + _config: &BackupStorageConfig, + _request: UploadObjectRequest, + ) -> Result<(), BackupError> { + Err(disabled_infra()) + } + async fn head_object( + &self, + _config: &BackupStorageConfig, + _object_key: &str, + ) -> Result, BackupError> { + Err(disabled_infra()) + } + async fn delete_object( + &self, + _config: &BackupStorageConfig, + _object_key: &str, + ) -> Result<(), BackupError> { + Err(disabled_infra()) + } + async fn presigned_download( + &self, + _config: &BackupStorageConfig, + _object_key: &str, + _file_name: &str, + _ttl: std::time::Duration, + ) -> Result { + Err(disabled_infra()) + } +} + +fn disabled_store() -> crate::ports::store::AdminStoreError { + crate::ports::store::AdminStoreError::new( + crate::ports::store::AdminStoreErrorKind::Unavailable, + "backup", + "backup store is disabled in this configuration", + ) +} + +fn disabled_infra() -> BackupError { + BackupError::new( + crate::model::backup::code::STORE_UNAVAILABLE, + "backup infrastructure is disabled in this configuration".to_owned(), + ) +} diff --git a/backend/crates/gateway-admin/src/ports/mod.rs b/backend/crates/gateway-admin/src/ports/mod.rs index 8de9dd195..4169383ff 100644 --- a/backend/crates/gateway-admin/src/ports/mod.rs +++ b/backend/crates/gateway-admin/src/ports/mod.rs @@ -1,5 +1,6 @@ //! 管理用例依赖的外部能力端口。 +pub mod backup; pub mod provider; pub mod store; pub mod system; diff --git a/backend/crates/gateway-admin/src/ports/store.rs b/backend/crates/gateway-admin/src/ports/store.rs index 70bd42871..7e8584fac 100644 --- a/backend/crates/gateway-admin/src/ports/store.rs +++ b/backend/crates/gateway-admin/src/ports/store.rs @@ -7,6 +7,7 @@ use std::sync::Arc; use async_trait::async_trait; use chrono::{DateTime, Utc}; +use super::backup::BackupStorePorts; use crate::model::{ MutationContext, Revision, accounts::{ @@ -300,6 +301,7 @@ pub struct AdminStorePorts { client_keys: Arc, observability: Arc, settings: Arc, + backup: BackupStorePorts, } impl AdminStorePorts { @@ -310,6 +312,7 @@ impl AdminStorePorts { client_keys: Arc, observability: Arc, settings: Arc, + backup: BackupStorePorts, ) -> Self { Self { accounts, @@ -317,6 +320,7 @@ impl AdminStorePorts { client_keys, observability, settings, + backup, } } @@ -344,4 +348,9 @@ impl AdminStorePorts { pub fn settings(&self) -> Arc { self.settings.clone() } + + #[must_use] + pub fn backup(&self) -> BackupStorePorts { + self.backup.clone() + } } diff --git a/backend/crates/gateway-admin/src/use_case/backup.rs b/backend/crates/gateway-admin/src/use_case/backup.rs new file mode 100644 index 000000000..2db0de26f --- /dev/null +++ b/backend/crates/gateway-admin/src/use_case/backup.rs @@ -0,0 +1,437 @@ +//! 备份配置、手动创建、下载与删除用例。 + +use std::sync::Arc; +use std::time::Duration; + +use async_trait::async_trait; +use chrono::{DateTime, Utc}; +use gateway_core::routing::snapshot::SnapshotControl; +use secrecy::ExposeSecret as _; +use uuid::Uuid; + +use crate::model::{ + AdminError, AdminErrorKind, MutationContext, + auth::{AdminAuditEvent, AuditActorKind}, + backup::{ + BackupError, BackupRecord, BackupRecordListQuery, BackupRecordPage, BackupSettings, + BackupStatus, BackupStorageConfig, BackupTriggerKind, ConnectionTestResult, + DownloadUrlResult, UpdateBackupScheduleCommand, UpdateBackupStorageCommand, + build_backup_seed, code, + }, +}; +use crate::ports::{ + backup::{BackupObjectStorePort, BackupRepository}, + store::AuthStore, +}; + +use super::map_store_error; + +/// 默认下载地址有效期。 +const DOWNLOAD_TTL: Duration = Duration::from_secs(5 * 60); + +/// API 消费的备份管理服务。 +#[async_trait] +pub trait BackupService: Send + Sync { + async fn load_settings(&self) -> Result; + + async fn update_storage( + &self, + context: &MutationContext, + command: UpdateBackupStorageCommand, + ) -> Result; + + async fn update_schedule( + &self, + context: &MutationContext, + command: UpdateBackupScheduleCommand, + ) -> Result; + + async fn test_connection( + &self, + context: &MutationContext, + ) -> Result; + + async fn create_backup( + &self, + context: &MutationContext, + expires_at: Option>, + ) -> Result; + + async fn list_records( + &self, + query: BackupRecordListQuery, + ) -> Result; + + async fn download_url( + &self, + context: &MutationContext, + backup_id: &str, + ) -> Result; + + async fn delete_backup( + &self, + context: &MutationContext, + backup_id: &str, + ) -> Result; +} + +pub(crate) struct DefaultBackupService { + repository: Arc, + object_store: Arc, + auth: Arc, + snapshot: Arc, +} + +impl DefaultBackupService { + #[must_use] + pub(crate) fn new( + repository: Arc, + object_store: Arc, + auth: Arc, + snapshot: Arc, + ) -> Self { + Self { + repository, + object_store, + auth, + snapshot, + } + } +} + +#[async_trait] +impl BackupService for DefaultBackupService { + async fn load_settings(&self) -> Result { + self.repository + .load_settings() + .await + .map_err(|error| map_store_error(error, "backup settings")) + } + + async fn update_storage( + &self, + context: &MutationContext, + command: UpdateBackupStorageCommand, + ) -> Result { + validate_storage(&command)?; + let (settings, revision) = self + .repository + .update_storage_settings(command, context) + .await + .map_err(|error| map_store_error(error, "backup storage"))?; + super::publish_committed(self.snapshot.as_ref(), revision).await?; + Ok(settings) + } + + async fn update_schedule( + &self, + context: &MutationContext, + command: UpdateBackupScheduleCommand, + ) -> Result { + validate_schedule(&command)?; + let next_run_at = if command.schedule_enabled { + Some( + crate::backup::policy::BackupSchedule::parse( + &command.cron_expression, + &command.schedule_timezone, + ) + .map_err(map_backup_error)? + .next_after(Utc::now()) + .ok_or_else(|| AdminError::bad_request("无法从计划推导下次执行时间"))?, + ) + } else { + None + }; + self.repository + .update_schedule_settings(command, next_run_at, context) + .await + .map_err(|error| map_store_error(error, "backup schedule")) + } + + async fn test_connection( + &self, + context: &MutationContext, + ) -> Result { + let settings = self + .repository + .load_settings() + .await + .map_err(|error| map_store_error(error, "backup settings"))?; + let Some(config) = BackupStorageConfig::from_settings(&settings) else { + return Ok(ConnectionTestResult { + ok: false, + stage: "putObject", + code: Some(code::INVALID_CONFIG), + message: "备份存储配置不完整，无法执行连接测试".to_owned(), + }); + }; + let result = self + .object_store + .test_connection(&config) + .await + .map_err(map_backup_error)?; + if result.ok { + let _ = self + .repository + .record_verification(config.storage_revision, Utc::now()) + .await + .map_err(|error| map_store_error(error, "backup settings"))?; + } + self.append_audit( + context, + "backup.s3_connection_tested", + "backup_settings", + vec![], + ) + .await?; + Ok(result) + } + + async fn create_backup( + &self, + context: &MutationContext, + expires_at: Option>, + ) -> Result { + let settings = self + .repository + .load_settings() + .await + .map_err(|error| map_store_error(error, "backup settings"))?; + if !settings.storage_configured() { + return Err(AdminError::bad_request( + "备份存储配置不完整，请先保存存储配置", + )); + } + if !settings.storage_verified() { + return Err(AdminError::bad_request( + "备份存储尚未通过连接测试，请先测试存储配置", + )); + } + let prefix = settings.prefix.as_deref().unwrap_or_default(); + let seed = build_backup_seed( + BackupTriggerKind::Manual, + None, + prefix, + Utc::now(), + expires_at, + ) + .map_err(map_backup_error)?; + let record = self + .repository + .insert_backup_record(seed) + .await + .map_err(|error| map_store_error(error, "backup record"))?; + self.append_audit(context, "backup.created", "backup", vec![record.id.clone()]) + .await?; + Ok(record) + } + + async fn list_records( + &self, + query: BackupRecordListQuery, + ) -> Result { + self.repository + .list_backup_records(query) + .await + .map_err(|error| map_store_error(error, "backup records")) + } + + async fn download_url( + &self, + context: &MutationContext, + backup_id: &str, + ) -> Result { + let record = self.require_record(backup_id).await?; + if record.status != BackupStatus::Completed { + return Err(AdminError::conflict( + "只有 completed 的备份才能创建下载地址", + )); + } + let settings = self + .repository + .load_settings() + .await + .map_err(|error| map_store_error(error, "backup settings"))?; + let Some(config) = BackupStorageConfig::from_settings(&settings) else { + return Err(AdminError::bad_request("备份存储配置不完整")); + }; + let file_name = crate::model::backup::build_download_file_name(backup_id); + let url = self + .object_store + .presigned_download(&config, &record.object_key, &file_name, DOWNLOAD_TTL) + .await + .map_err(map_backup_error)?; + self.append_audit( + context, + "backup.download_url_created", + "backup", + vec![backup_id.to_owned()], + ) + .await?; + Ok(DownloadUrlResult { + url, + file_name, + expires_in: DOWNLOAD_TTL, + }) + } + + async fn delete_backup( + &self, + context: &MutationContext, + backup_id: &str, + ) -> Result { + let record = self.require_record(backup_id).await?; + if !record.status.can_be_deleted() { + return Err(AdminError::conflict( + "只有 completed 或 failed 的备份才能删除", + )); + } + let now = Utc::now(); + let deleting = self + .repository + .transition_to_deleting(backup_id, now) + .await + .map_err(|error| map_store_error(error, "backup record"))? + .ok_or_else(|| AdminError::conflict("备份记录已进入删除流程"))?; + self.append_audit( + context, + "backup.delete_requested", + "backup", + vec![backup_id.to_owned()], + ) + .await?; + Ok(deleting) + } +} + +impl DefaultBackupService { + async fn require_record(&self, backup_id: &str) -> Result { + self.repository + .load_backup_record(backup_id) + .await + .map_err(|error| map_store_error(error, "backup record"))? + .ok_or_else(|| AdminError::not_found("备份记录不存在")) + } + + async fn append_audit( + &self, + context: &MutationContext, + action: &str, + entity_ref: &str, + changed_fields: Vec, + ) -> Result<(), AdminError> { + let actor_kind = match context.actor { + crate::model::MutationActor::AdminSession { .. } => AuditActorKind::AdminSession, + crate::model::MutationActor::AdminApiKey => AuditActorKind::AdminApiKey, + crate::model::MutationActor::System => AuditActorKind::System, + }; + let actor_ref = match &context.actor { + crate::model::MutationActor::AdminSession { admin_user_id } => admin_user_id.clone(), + crate::model::MutationActor::AdminApiKey => "admin_api_key".to_owned(), + crate::model::MutationActor::System => "system".to_owned(), + }; + let actor_admin_user_id = match &context.actor { + crate::model::MutationActor::AdminSession { admin_user_id } => { + Some(admin_user_id.clone()) + } + _ => None, + }; + let event = AdminAuditEvent { + id: format!("audit_{}", Uuid::now_v7().simple()), + actor_kind, + actor_admin_user_id, + actor_ref, + request_id: Some(context.request_id.clone()), + action: action.to_owned(), + entity_kind: "backup".to_owned(), + entity_ref: entity_ref.to_owned(), + config_revision: None, + changed_fields, + occurred_at: Utc::now(), + }; + self.auth + .append_audit_event(event) + .await + .map_err(|error| map_store_error(error, "backup audit")) + } +} + +/// 校验存储配置；空串视为非法，Secret 缺省表示保留旧值。 +fn validate_storage(command: &UpdateBackupStorageCommand) -> Result<(), AdminError> { + validate_endpoint(&command.endpoint)?; + validate_nonempty("region", &command.region)?; + validate_nonempty("bucket", &command.bucket)?; + validate_nonempty("accessKeyId", &command.access_key_id)?; + if let Some(secret) = command.secret_access_key.as_ref() { + let value = secret.expose_secret(); + if value.is_empty() { + return Err(AdminError::bad_request( + "secretAccessKey 不能为空字符串；缺省表示保留旧值", + )); + } + validate_length("secretAccessKey", value, 512)?; + } + crate::model::backup::normalize_prefix(&command.prefix).map_err(map_backup_error)?; + Ok(()) +} + +fn validate_endpoint(value: &str) -> Result<(), AdminError> { + if value.is_empty() { + return Err(AdminError::bad_request("endpoint 不能为空")); + } + if !value.starts_with("https://") { + return Err(AdminError::bad_request( + "endpoint 必须使用 HTTPS；仅测试或显式开发配置允许 HTTP", + )); + } + if value.contains('@') || value.bytes().any(|byte| byte.is_ascii_control()) { + return Err(AdminError::bad_request("endpoint 格式非法")); + } + validate_length("endpoint", value, 2048) +} + +fn validate_nonempty(field: &str, value: &str) -> Result<(), AdminError> { + if value.is_empty() { + return Err(AdminError::bad_request(format!("{field} 不能为空"))); + } + validate_length(field, value, 255) +} + +fn validate_length(field: &str, value: &str, max: usize) -> Result<(), AdminError> { + if value.len() > max || value.bytes().any(|byte| byte.is_ascii_control()) { + return Err(AdminError::bad_request(format!( + "{field} 超出长度或包含非法字符" + ))); + } + Ok(()) +} + +/// 校验调度配置并解析 Cron 与时区。 +fn validate_schedule(command: &UpdateBackupScheduleCommand) -> Result<(), AdminError> { + if command.cron_expression.trim().is_empty() { + return Err(AdminError::bad_request("cronExpression 不能为空")); + } + crate::backup::policy::BackupSchedule::parse( + &command.cron_expression, + &command.schedule_timezone, + ) + .map_err(map_backup_error)?; + Ok(()) +} + +/// 备份基础设施错误到既有 `AdminErrorKind` 的唯一边界映射。 +fn map_backup_error(error: BackupError) -> AdminError { + match error.code() { + code::INVALID_CONFIG | code::INVALID_CRON | code::INVALID_TIMEZONE => { + AdminError::bad_request(error.message().to_owned()) + } + code::RECORD_NOT_FOUND => AdminError::not_found(error.message().to_owned()), + code::ACTIVE_TASK_CONFLICT | code::STATE_CONFLICT | code::STORAGE_IDENTITY_LOCKED => { + AdminError::conflict(error.message().to_owned()) + } + code::STORE_UNAVAILABLE => { + AdminError::new(AdminErrorKind::Unavailable, error.message().to_owned()) + } + _ => AdminError::bad_gateway(error.message().to_owned()), + } +} diff --git a/backend/crates/gateway-admin/src/use_case/mod.rs b/backend/crates/gateway-admin/src/use_case/mod.rs index 376541286..9e048b561 100644 --- a/backend/crates/gateway-admin/src/use_case/mod.rs +++ b/backend/crates/gateway-admin/src/use_case/mod.rs @@ -2,6 +2,7 @@ pub mod accounts; pub mod auth; +pub mod backup; pub mod client_keys; pub mod observability; pub mod openai; diff --git a/backend/crates/gateway-admin/tests/backup.rs b/backend/crates/gateway-admin/tests/backup.rs new file mode 100644 index 000000000..36c508317 --- /dev/null +++ b/backend/crates/gateway-admin/tests/backup.rs @@ -0,0 +1,271 @@ +//! 备份领域模型与纯策略测试（生产 src 禁止内嵌测试，见 architecture.rs）。 + +use chrono::{Duration, TimeZone as _, Utc}; +use secrecy::SecretString; + +use gateway_admin::{ + backup::policy::{BackupSchedule, RetentionReason, decide_retention}, + model::backup::{ + BackupRecord, BackupSettings, BackupStatus, BackupStorageConfig, BackupTriggerKind, + build_object_key, + }, +}; + +fn record( + id: &str, + trigger: BackupTriggerKind, + status: BackupStatus, + created_at: chrono::DateTime, + completed_at: Option>, +) -> BackupRecord { + BackupRecord { + id: id.to_owned(), + trigger_kind: trigger, + status, + scheduled_at: if trigger == BackupTriggerKind::Scheduled { + Some(created_at) + } else { + None + }, + object_key: format!("k/{id}"), + size_bytes: Some(1), + sha256: Some("a".repeat(64)), + attempt_count: 0, + error_code: None, + error_message: None, + started_at: Some(created_at), + completed_at, + expires_at: None, + created_at, + updated_at: created_at, + } +} + +#[test] +fn status_machine_rejects_invalid_transitions() { + assert!(BackupStatus::Queued.allows_transition_to(BackupStatus::Dumping)); + assert!(BackupStatus::Dumping.allows_transition_to(BackupStatus::Uploading)); + assert!(BackupStatus::Uploading.allows_transition_to(BackupStatus::Completed)); + assert!(BackupStatus::Completed.allows_transition_to(BackupStatus::Deleting)); + assert!(!BackupStatus::Queued.allows_transition_to(BackupStatus::Completed)); + assert!(!BackupStatus::Completed.allows_transition_to(BackupStatus::Dumping)); + assert!(!BackupStatus::Deleting.allows_transition_to(BackupStatus::Failed)); + assert!(!BackupStatus::Deleting.allows_transition_to(BackupStatus::Completed)); +} + +#[test] +fn status_classification() { + assert!(BackupStatus::Queued.is_active()); + assert!(BackupStatus::Uploading.is_active()); + assert!(!BackupStatus::Completed.is_active()); + assert!(BackupStatus::Completed.can_be_deleted()); + assert!(BackupStatus::Failed.can_be_deleted()); + assert!(!BackupStatus::Queued.can_be_deleted()); +} + +#[test] +fn object_key_shape() { + let at = Utc.with_ymd_and_hms(2026, 8, 1, 12, 30, 0).unwrap(); + let key = build_object_key("production/backups", "backup_abc", at).unwrap(); + assert_eq!( + key, + "production/backups/2026/08/01/codex-proxy-rs_20260801_123000_abc.dump" + ); +} + +#[test] +fn object_key_rejects_dangerous_prefixes() { + let at = Utc::now(); + assert!(build_object_key("/leading", "backup_x", at).is_err()); + assert!(build_object_key("a/../b", "backup_x", at).is_err()); + assert!(build_object_key("a\\b", "backup_x", at).is_err()); + assert!(build_object_key("a\u{0001}b", "backup_x", at).is_err()); + assert!(build_object_key("", "backup_x", at).is_err()); + assert!(build_object_key("trailing//", "backup_x", at).is_ok()); +} + +#[test] +fn trigger_and_status_roundtrip() { + for status in [ + BackupStatus::Queued, + BackupStatus::Dumping, + BackupStatus::Uploading, + BackupStatus::Completed, + BackupStatus::Failed, + BackupStatus::Deleting, + ] { + assert_eq!(BackupStatus::parse(status.as_str()), Some(status)); + } + assert_eq!( + BackupTriggerKind::parse("manual"), + Some(BackupTriggerKind::Manual) + ); + assert_eq!( + BackupTriggerKind::parse("scheduled"), + Some(BackupTriggerKind::Scheduled) + ); + assert_eq!(BackupTriggerKind::parse("unknown"), None); + assert_eq!(BackupStatus::parse("skipped"), None); + assert_eq!(BackupStatus::parse("deleted"), None); + assert_eq!(BackupStatus::parse("unknown"), None); +} + +#[test] +fn config_from_settings_redacts_secret_in_debug() { + let mut settings = BackupSettings { + storage_revision: 3, + endpoint: Some("https://example.com".to_owned()), + region: Some("auto".to_owned()), + bucket: Some("b".to_owned()), + access_key_id: Some("ak".to_owned()), + secret_access_key: Some(SecretString::from("sk-secret")), + prefix: Some("p".to_owned()), + force_path_style: false, + schedule_enabled: false, + cron_expression: None, + schedule_timezone: None, + retention_days: 0, + retention_count: 0, + next_run_at: None, + last_verified_at: None, + updated_at: Utc::now(), + }; + assert!(!format!("{:?}", settings).contains("sk-secret")); + let config = BackupStorageConfig::from_settings(&settings).unwrap(); + assert!(!format!("{config:?}").contains("sk-secret")); + settings.endpoint = None; + assert!(BackupStorageConfig::from_settings(&settings).is_none()); +} + +#[test] +fn parse_rejects_bad_cron_and_timezone() { + assert!(BackupSchedule::parse("61 * * * *", "Asia/Shanghai").is_err()); + assert!(BackupSchedule::parse("0 2 * * *", "Not/AZone").is_err()); + assert!(BackupSchedule::parse("0 2 * * *", "Asia/Shanghai").is_ok()); +} + +#[test] +fn next_after_is_strictly_later() { + let schedule = BackupSchedule::parse("0 2 * * *", "Asia/Shanghai").unwrap(); + let at = Utc.with_ymd_and_hms(2026, 8, 1, 18, 0, 0).unwrap(); + let next = schedule.next_after(at).unwrap(); + assert_eq!(next, Utc.with_ymd_and_hms(2026, 8, 2, 18, 0, 0).unwrap()); +} + +#[test] +fn last_firing_is_at_or_before() { + let schedule = BackupSchedule::parse("0 2 * * *", "Asia/Shanghai").unwrap(); + // 北京时间 02:00 = 前一日 18:00 UTC。 + let before = Utc.with_ymd_and_hms(2026, 8, 1, 17, 59, 0).unwrap(); + assert_eq!( + schedule.last_firing_at_or_before(before).unwrap(), + Utc.with_ymd_and_hms(2026, 7, 31, 18, 0, 0).unwrap() + ); + let exact = Utc.with_ymd_and_hms(2026, 8, 1, 18, 0, 0).unwrap(); + assert_eq!(schedule.last_firing_at_or_before(exact).unwrap(), exact); + let after = Utc.with_ymd_and_hms(2026, 8, 1, 19, 0, 0).unwrap(); + assert_eq!(schedule.last_firing_at_or_before(after).unwrap(), exact); +} + +#[test] +fn dst_spring_forward_skips_missing_hour() { + let schedule = BackupSchedule::parse("30 2 * * *", "America/New_York").unwrap(); + let at = Utc.with_ymd_and_hms(2026, 3, 8, 6, 29, 0).unwrap(); // 01:29 EST + let next = schedule.next_after(at).unwrap(); + assert_eq!(next, Utc.with_ymd_and_hms(2026, 3, 9, 6, 30, 0).unwrap()); + let at_after_spring = Utc.with_ymd_and_hms(2026, 3, 8, 10, 0, 0).unwrap(); // 06:00 EDT + let last = schedule.last_firing_at_or_before(at_after_spring).unwrap(); + assert_eq!(last, Utc.with_ymd_and_hms(2026, 3, 7, 7, 30, 0).unwrap()); +} + +#[test] +fn retention_count_keeps_newest_n() { + let now = Utc::now(); + let base = now - Duration::days(10); + let records = vec![ + record( + "newest", + BackupTriggerKind::Scheduled, + BackupStatus::Completed, + base + Duration::days(2), + Some(base + Duration::days(2)), + ), + record( + "middle", + BackupTriggerKind::Scheduled, + BackupStatus::Completed, + base + Duration::days(1), + Some(base + Duration::days(1)), + ), + record( + "oldest", + BackupTriggerKind::Scheduled, + BackupStatus::Completed, + base, + Some(base), + ), + ]; + let decisions = decide_retention(0, 2, now, &records); + let ids: Vec<_> = decisions.iter().map(|d| d.record_id.as_str()).collect(); + assert_eq!(ids, vec!["oldest"]); + assert_eq!(decisions[0].reason, RetentionReason::Count); +} + +#[test] +fn retention_days_deletes_old_scheduled_but_keeps_newest() { + let now = Utc::now(); + let records = vec![ + record( + "recent", + BackupTriggerKind::Scheduled, + BackupStatus::Completed, + now - Duration::days(1), + Some(now - Duration::days(1)), + ), + record( + "ancient", + BackupTriggerKind::Scheduled, + BackupStatus::Completed, + now - Duration::days(40), + Some(now - Duration::days(40)), + ), + ]; + let decisions = decide_retention(30, 0, now, &records); + let ids: Vec<_> = decisions.iter().map(|d| d.record_id.as_str()).collect(); + assert_eq!(ids, vec!["ancient"]); +} + +#[test] +fn newest_scheduled_never_auto_deleted_even_when_ancient() { + let now = Utc::now(); + let records = vec![record( + "sole", + BackupTriggerKind::Scheduled, + BackupStatus::Completed, + now - Duration::days(400), + Some(now - Duration::days(400)), + )]; + assert!(decide_retention(30, 1, now, &records).is_empty()); +} + +#[test] +fn disabled_thresholds_delete_nothing() { + let now = Utc::now(); + let records = vec![ + record( + "newest", + BackupTriggerKind::Scheduled, + BackupStatus::Completed, + now - Duration::days(2), + Some(now - Duration::days(2)), + ), + record( + "old", + BackupTriggerKind::Scheduled, + BackupStatus::Completed, + now - Duration::days(90), + Some(now - Duration::days(90)), + ), + ]; + assert!(decide_retention(0, 0, now, &records).is_empty()); +} diff --git a/backend/crates/gateway-admin/tests/backup_runtime.rs b/backend/crates/gateway-admin/tests/backup_runtime.rs new file mode 100644 index 000000000..ea322faf5 --- /dev/null +++ b/backend/crates/gateway-admin/tests/backup_runtime.rs @@ -0,0 +1,800 @@ +//! BackupTask 执行链与 BackupService 用例的端口级测试。 +//! +//! 按参考实现 sub2api 的方式：用内存 fake 端口驱动编排层，不依赖 MinIO/真实 S3。 +//! s3.rs / pg_dump.rs 是 SDK 薄适配器，协议正确性交给 aws-sdk / pg_dump 保证。 + +use std::sync::{Arc, Mutex}; + +use async_trait::async_trait; +use chrono::{DateTime, Duration, Utc}; +use gateway_admin::{ + backup::task::BackupTask, + model::{ + MutationActor, MutationContext, Revision, + auth::AdminAuditEvent, + backup::{ + BackupObjectMetadata, BackupRecord, BackupRecordListQuery, BackupRecordPage, + BackupRecordSeed, BackupSettings, BackupStatus, BackupStorageConfig, BackupTriggerKind, + ConnectionTestResult, UpdateBackupScheduleCommand, UpdateBackupStorageCommand, + }, + }, + ports::{ + backup::{ + BackupObjectStorePort, BackupRepository, DatabaseDumpPort, DumpArtifact, DumpRequest, + StagedArtifact, StatusTransitionUpdate, UploadObjectRequest, + }, + store::{AdminStoreError, AdminStoreErrorKind, AdminStoreResult, AuthStore}, + }, +}; +use gateway_core::engine::CancellationToken; +use secrecy::SecretString; +use sha2::{Digest as _, Sha256}; +use tempfile::TempDir; + +/// 假归档内容；dump 与上传校验共用同一字节与校验值。 +pub(crate) const DUMP_CONTENT: &[u8] = b"custom-format-archive-bytes-0123456789"; + +/// 计算一段内容的 SHA-256 十六进制。 +pub(crate) fn sha256_hex(content: &[u8]) -> String { + hex::encode(Sha256::digest(content)) +} + +/// 返回 32 位十六进制的合法备份记录 id。 +pub(crate) fn backup_id(suffix: &str) -> String { + format!("backup_{}", sha256_hex(suffix.as_bytes())) +} + +/// 已配置且已通过连接测试的设置快照。 +pub(crate) fn configured_settings() -> BackupSettings { + let now = Utc::now(); + BackupSettings { + storage_revision: 1, + endpoint: Some("https://s3.example.com".to_owned()), + region: Some("auto".to_owned()), + bucket: Some("backup-bucket".to_owned()), + access_key_id: Some("ak".to_owned()), + secret_access_key: Some(SecretString::from("sk-value")), + prefix: Some("codex".to_owned()), + force_path_style: false, + schedule_enabled: false, + cron_expression: None, + schedule_timezone: None, + retention_days: 0, + retention_count: 0, + next_run_at: None, + last_verified_at: Some(now), + updated_at: now, + } +} + +/// 从 seed 构造 queued 记录。 +pub(crate) fn queued_record(seed: &BackupRecordSeed) -> BackupRecord { + let now = Utc::now(); + BackupRecord { + id: seed.id.clone(), + trigger_kind: seed.trigger_kind, + status: BackupStatus::Queued, + scheduled_at: seed.scheduled_at, + object_key: seed.object_key.clone(), + size_bytes: None, + sha256: None, + attempt_count: 0, + error_code: None, + error_message: None, + started_at: None, + completed_at: None, + expires_at: seed.expires_at, + created_at: now, + updated_at: now, + } +} + +/// 内存版备份仓储。 +pub(crate) struct FakeBackupRepository { + settings: Mutex, + records: Mutex>, +} + +impl FakeBackupRepository { + pub(crate) fn new(settings: BackupSettings) -> Self { + Self { + settings: Mutex::new(settings), + records: Mutex::new(Vec::new()), + } + } + + /// 测试辅助：读取当前全部记录。 + pub(crate) fn all_records(&self) -> Vec { + self.records.lock().expect("records").clone() + } + + /// 测试辅助：强制把一条记录标记为指定时间完成的 completed。 + pub(crate) fn set_completed(&self, id: &str, completed_at: DateTime) { + let mut records = self.records.lock().expect("records"); + if let Some(record) = records.iter_mut().find(|record| record.id == id) { + record.status = BackupStatus::Completed; + record.completed_at = Some(completed_at); + record.updated_at = Utc::now(); + } + } + + /// 测试辅助：强制把一条记录迁移到指定状态。 + pub(crate) fn force_status(&self, id: &str, status: BackupStatus) { + let mut records = self.records.lock().expect("records"); + if let Some(record) = records.iter_mut().find(|record| record.id == id) { + record.status = status; + if matches!(status, BackupStatus::Completed | BackupStatus::Failed) { + record.completed_at = Some(Utc::now()); + } + record.updated_at = Utc::now(); + } + } +} + +#[async_trait] +impl BackupRepository for FakeBackupRepository { + async fn load_settings(&self) -> AdminStoreResult { + Ok(self.settings.lock().expect("settings").clone()) + } + + async fn update_storage_settings( + &self, + command: UpdateBackupStorageCommand, + _context: &MutationContext, + ) -> AdminStoreResult<(BackupSettings, Revision)> { + let mut settings = self.settings.lock().expect("settings"); + settings.storage_revision += 1; + settings.endpoint = Some(command.endpoint); + settings.region = Some(command.region); + settings.bucket = Some(command.bucket); + settings.access_key_id = Some(command.access_key_id); + if let Some(secret) = command.secret_access_key { + settings.secret_access_key = Some(secret); + } + settings.prefix = Some(command.prefix); + settings.force_path_style = command.force_path_style; + settings.last_verified_at = None; + settings.updated_at = Utc::now(); + let revision = Revision::new(settings.storage_revision).map_err(|_| { + AdminStoreError::new(AdminStoreErrorKind::Invalid, "backup", "zero revision") + })?; + Ok((settings.clone(), revision)) + } + + async fn update_schedule_settings( + &self, + command: UpdateBackupScheduleCommand, + next_run_at: Option>, + _context: &MutationContext, + ) -> AdminStoreResult { + let mut settings = self.settings.lock().expect("settings"); + settings.schedule_enabled = command.schedule_enabled; + settings.cron_expression = Some(command.cron_expression); + settings.schedule_timezone = Some(command.schedule_timezone); + settings.retention_days = command.retention_days; + settings.retention_count = command.retention_count; + settings.next_run_at = next_run_at; + settings.updated_at = Utc::now(); + Ok(settings.clone()) + } + + async fn record_verification( + &self, + storage_revision: u64, + at: DateTime, + ) -> AdminStoreResult { + let mut settings = self.settings.lock().expect("settings"); + if settings.storage_revision != storage_revision { + return Ok(false); + } + settings.last_verified_at = Some(at); + Ok(true) + } + + async fn insert_backup_record(&self, seed: BackupRecordSeed) -> AdminStoreResult { + let mut records = self.records.lock().expect("records"); + if records.iter().any(|record| record.status.is_active()) { + return Err(AdminStoreError::new( + AdminStoreErrorKind::Conflict, + "backup record", + "an active backup task already exists", + )); + } + let record = queued_record(&seed); + records.push(record.clone()); + Ok(record) + } + + async fn insert_scheduled_record(&self, seed: BackupRecordSeed) -> AdminStoreResult { + let mut records = self.records.lock().expect("records"); + if records.iter().any(|record| record.status.is_active()) + || records.iter().any(|record| { + record.trigger_kind == BackupTriggerKind::Scheduled + && record.scheduled_at == seed.scheduled_at + }) + { + return Ok(false); + } + records.push(queued_record(&seed)); + Ok(true) + } + + async fn list_backup_records( + &self, + query: BackupRecordListQuery, + ) -> AdminStoreResult { + let records = self.records.lock().expect("records"); + let mut items = records + .iter() + .filter(|record| { + query.status.is_none_or(|status| record.status == status) + && query + .trigger + .is_none_or(|trigger| record.trigger_kind == trigger) + }) + .cloned() + .collect::>(); + items.sort_by(|a, b| { + b.created_at + .cmp(&a.created_at) + .then_with(|| b.id.cmp(&a.id)) + }); + let total = items.len() as u64; + let offset = + (u64::from(query.page.saturating_sub(1)) * u64::from(query.page_size.get())) as usize; + let limit = usize::from(query.page_size.get()); + items = items.into_iter().skip(offset).take(limit).collect(); + Ok(BackupRecordPage { + items, + total, + page: query.page, + page_size: query.page_size, + }) + } + + async fn load_backup_record(&self, id: &str) -> AdminStoreResult> { + Ok(self + .records + .lock() + .expect("records") + .iter() + .find(|record| record.id == id) + .cloned()) + } + + async fn list_intermediate_records(&self) -> AdminStoreResult> { + Ok(self + .records + .lock() + .expect("records") + .iter() + .filter(|record| { + matches!( + record.status, + BackupStatus::Dumping | BackupStatus::Uploading + ) + }) + .cloned() + .collect()) + } + + async fn list_pending_deletions(&self, _limit: u32) -> AdminStoreResult> { + Ok(self + .records + .lock() + .expect("records") + .iter() + .filter(|record| record.status == BackupStatus::Deleting) + .cloned() + .collect()) + } + + async fn list_expired_records(&self, _limit: u32) -> AdminStoreResult> { + let now = Utc::now(); + Ok(self + .records + .lock() + .expect("records") + .iter() + .filter(|record| { + record + .expires_at + .is_some_and(|expires_at| expires_at <= now) + && matches!( + record.status, + BackupStatus::Completed | BackupStatus::Failed + ) + }) + .cloned() + .collect()) + } + + async fn claim_next_queued( + &self, + now: DateTime, + ) -> AdminStoreResult> { + let mut records = self.records.lock().expect("records"); + let Some(record) = records + .iter_mut() + .find(|record| record.status == BackupStatus::Queued) + else { + return Ok(None); + }; + record.status = BackupStatus::Dumping; + record.started_at = Some(now); + record.attempt_count += 1; + record.updated_at = now; + Ok(Some(record.clone())) + } + + async fn transition_status( + &self, + id: &str, + from: BackupStatus, + to: BackupStatus, + update: StatusTransitionUpdate, + now: DateTime, + ) -> AdminStoreResult> { + let mut records = self.records.lock().expect("records"); + let Some(record) = records + .iter_mut() + .find(|record| record.id == id && record.status == from) + else { + return Ok(None); + }; + record.status = to; + if let Some(size) = update.size_bytes { + record.size_bytes = Some(size); + } + if let Some(sha256) = update.sha256 { + record.sha256 = Some(sha256); + } + if let Some(code) = update.error_code { + record.error_code = Some(code); + } + if let Some(message) = update.error_message { + record.error_message = Some(message); + } + record.completed_at = update.completed_at.or_else(|| { + matches!(to, BackupStatus::Completed | BackupStatus::Failed).then_some(now) + }); + record.updated_at = now; + Ok(Some(record.clone())) + } + + async fn transition_to_deleting( + &self, + id: &str, + now: DateTime, + ) -> AdminStoreResult> { + let mut records = self.records.lock().expect("records"); + let Some(record) = records + .iter_mut() + .find(|record| record.id == id && record.status.can_be_deleted()) + else { + return Ok(None); + }; + record.status = BackupStatus::Deleting; + record.updated_at = now; + Ok(Some(record.clone())) + } + + async fn delete_record(&self, id: &str) -> AdminStoreResult<()> { + let mut records = self.records.lock().expect("records"); + records.retain(|record| record.id != id); + Ok(()) + } + + async fn advance_schedule_cursor( + &self, + next_run_at: DateTime, + _expected_cron: &str, + _expected_timezone: &str, + ) -> AdminStoreResult { + let mut settings = self.settings.lock().expect("settings"); + settings.next_run_at = Some(next_run_at); + Ok(true) + } + + async fn list_scheduled_completed_desc( + &self, + _limit: u32, + ) -> AdminStoreResult> { + let mut records = self + .records + .lock() + .expect("records") + .iter() + .filter(|record| { + record.trigger_kind == BackupTriggerKind::Scheduled + && record.status == BackupStatus::Completed + }) + .cloned() + .collect::>(); + records.sort_by(|a, b| { + b.completed_at + .cmp(&a.completed_at) + .then_with(|| b.id.cmp(&a.id)) + }); + Ok(records) + } +} + +/// 假 `pg_dump` 端口：把固定内容写入临时文件作为归档；可注入失败。 +pub(crate) struct FakeDumpPort { + dir: TempDir, + fail_dump: bool, +} + +impl FakeDumpPort { + pub(crate) fn new() -> Self { + Self { + dir: tempfile::tempdir().expect("temp dir"), + fail_dump: false, + } + } + + /// 让下一次 `dump` 返回 `backup.pg_dump_failed`。 + pub(crate) fn fail_next_dump(mut self) -> Self { + self.fail_dump = true; + self + } + + fn artifact(&self, backup_id: &str) -> DumpArtifact { + let path = self.dir.path().join(format!("{backup_id}.dump")); + std::fs::write(&path, DUMP_CONTENT).expect("write artifact"); + DumpArtifact { + path, + size_bytes: DUMP_CONTENT.len() as u64, + sha256: sha256_hex(DUMP_CONTENT), + } + } +} + +#[async_trait] +impl DatabaseDumpPort for FakeDumpPort { + async fn dump( + &self, + request: DumpRequest, + ) -> Result { + if self.fail_dump { + return Err(gateway_admin::model::backup::BackupError::new( + gateway_admin::model::backup::code::PG_DUMP_FAILED, + "pg_dump failed".to_owned(), + )); + } + if request.cancellation.is_cancelled() { + return Err(gateway_admin::model::backup::BackupError::new( + gateway_admin::model::backup::code::CANCELLED, + "cancelled".to_owned(), + )); + } + Ok(self.artifact(&request.backup_id)) + } + + async fn inspect_staging( + &self, + backup_id: &str, + ) -> Result, gateway_admin::model::backup::BackupError> { + let path = self.dir.path().join(format!("{backup_id}.dump")); + if !path.exists() { + return Ok(None); + } + Ok(Some(StagedArtifact { + path, + size_bytes: DUMP_CONTENT.len() as u64, + sha256: sha256_hex(DUMP_CONTENT), + })) + } + + async fn cleanup_staging( + &self, + _backup_id: &str, + ) -> Result<(), gateway_admin::model::backup::BackupError> { + Ok(()) + } +} + +/// 内存版对象存储：把上传内容按 key 保存并支持 Head/Delete/预签名。 +pub(crate) struct FakeObjectStore { + objects: Mutex>>, +} + +impl FakeObjectStore { + pub(crate) fn new() -> Self { + Self { + objects: Mutex::new(std::collections::HashMap::new()), + } + } + + pub(crate) fn object(&self, key: &str) -> Option> { + self.objects.lock().expect("objects").get(key).cloned() + } +} + +#[async_trait] +impl BackupObjectStorePort for FakeObjectStore { + async fn test_connection( + &self, + _config: &BackupStorageConfig, + ) -> Result { + Ok(ConnectionTestResult { + ok: true, + stage: "deleteObject", + code: None, + message: "连接测试成功".to_owned(), + }) + } + + async fn upload_file( + &self, + _config: &BackupStorageConfig, + request: UploadObjectRequest, + ) -> Result<(), gateway_admin::model::backup::BackupError> { + let content = std::fs::read(&request.source).map_err(|_| { + gateway_admin::model::backup::BackupError::new( + gateway_admin::model::backup::code::S3_UPLOAD_FAILED, + "read source".to_owned(), + ) + })?; + self.objects + .lock() + .expect("objects") + .insert(request.object_key, content); + Ok(()) + } + + async fn head_object( + &self, + _config: &BackupStorageConfig, + object_key: &str, + ) -> Result, gateway_admin::model::backup::BackupError> { + let Some(content) = self.object(object_key) else { + return Ok(None); + }; + Ok(Some(BackupObjectMetadata::new( + "backup-test".to_owned(), + sha256_hex(&content), + Utc::now(), + content.len() as u64, + ))) + } + + async fn delete_object( + &self, + _config: &BackupStorageConfig, + object_key: &str, + ) -> Result<(), gateway_admin::model::backup::BackupError> { + self.objects.lock().expect("objects").remove(object_key); + Ok(()) + } + + async fn presigned_download( + &self, + _config: &BackupStorageConfig, + object_key: &str, + _file_name: &str, + _ttl: std::time::Duration, + ) -> Result { + Ok(format!("https://presigned.example.com/{object_key}")) + } +} + +/// 记录审计事件的假 AuthStore。 +pub(crate) struct FakeAuthStore { + audit: Mutex>, +} + +impl FakeAuthStore { + pub(crate) fn new() -> Self { + Self { + audit: Mutex::new(Vec::new()), + } + } + + pub(crate) fn audit_actions(&self) -> Vec { + self.audit + .lock() + .expect("audit") + .iter() + .map(|event| event.action.clone()) + .collect() + } +} + +#[async_trait] +impl AuthStore for FakeAuthStore { + async fn load_password_hash(&self, _admin_user_id: &str) -> AdminStoreResult> { + Ok(None) + } + async fn create_password_hash_if_absent( + &self, + _admin_user_id: &str, + _password_hash: &str, + ) -> AdminStoreResult { + Ok(true) + } + async fn load_admin_api_key( + &self, + ) -> AdminStoreResult> { + Ok(None) + } + async fn load_session( + &self, + _session_id: &str, + ) -> AdminStoreResult> { + Ok(None) + } + async fn store_session( + &self, + _session_id: &str, + _session: &gateway_admin::model::auth::AdminSession, + ) -> AdminStoreResult<()> { + Ok(()) + } + async fn delete_session( + &self, + _session_id: &str, + ) -> AdminStoreResult> { + Ok(None) + } + async fn append_audit_event(&self, event: AdminAuditEvent) -> AdminStoreResult<()> { + self.audit.lock().expect("audit").push(event); + Ok(()) + } +} + +/// 系统发起者上下文。 +pub(crate) fn system_context() -> MutationContext { + MutationContext { + actor: MutationActor::System, + request_id: "backup-runtime-test".to_owned(), + } +} + +#[tokio::test] +async fn backup_task_runs_full_dump_upload_verify_pipeline() { + let repository = Arc::new(FakeBackupRepository::new(configured_settings())); + let dump = Arc::new(FakeDumpPort::new()); + let object_store = Arc::new(FakeObjectStore::new()); + + let seed = BackupRecordSeed { + id: backup_id("manual"), + trigger_kind: BackupTriggerKind::Manual, + scheduled_at: None, + object_key: "codex/2026/08/01/manual.dump".to_owned(), + expires_at: None, + }; + repository + .insert_backup_record(seed.clone()) + .await + .expect("insert queued"); + + let task = BackupTask::new(repository.clone(), dump.clone(), object_store.clone()); + task.run_cycle(&CancellationToken::new()) + .await + .expect("run cycle"); + + let records = repository.all_records(); + assert_eq!(records.len(), 1); + let finished = &records[0]; + assert_eq!(finished.status, BackupStatus::Completed); + assert_eq!(finished.size_bytes, Some(DUMP_CONTENT.len() as u64)); + assert_eq!( + finished.sha256.as_deref(), + Some(sha256_hex(DUMP_CONTENT).as_str()) + ); + + // 对象已上传且内容与本地归档一致。 + let stored = object_store + .object(&seed.object_key) + .expect("uploaded object exists"); + assert_eq!(stored.as_slice(), DUMP_CONTENT); +} + +#[tokio::test] +async fn backup_task_marks_failed_when_dump_port_fails() { + let repository = Arc::new(FakeBackupRepository::new(configured_settings())); + let dump = Arc::new(FakeDumpPort::new().fail_next_dump()); + let object_store = Arc::new(FakeObjectStore::new()); + + let seed = BackupRecordSeed { + id: backup_id("fail"), + trigger_kind: BackupTriggerKind::Manual, + scheduled_at: None, + object_key: "codex/2026/08/01/fail.dump".to_owned(), + expires_at: None, + }; + repository + .insert_backup_record(seed.clone()) + .await + .expect("insert queued"); + + let task = BackupTask::new(repository.clone(), dump.clone(), object_store.clone()); + task.run_cycle(&CancellationToken::new()) + .await + .expect("run cycle"); + + let records = repository.all_records(); + let failed = &records[0]; + assert_eq!(failed.status, BackupStatus::Failed); + assert_eq!(failed.error_code.as_deref(), Some("backup.pg_dump_failed")); + assert!(failed.completed_at.is_some()); + // 归档未上传，对象不存在。 + assert!(object_store.object(&seed.object_key).is_none()); +} + +#[tokio::test] +async fn retention_cleans_expired_scheduled_backups() { + let repository = Arc::new(FakeBackupRepository::new(configured_settings())); + let dump = Arc::new(FakeDumpPort::new()); + let object_store = Arc::new(FakeObjectStore::new()); + + // 造一条旧 completed 计划备份（超过 retention_days）与一条新备份。 + let old_seed = BackupRecordSeed { + id: backup_id("old"), + trigger_kind: BackupTriggerKind::Scheduled, + scheduled_at: Some(Utc::now() - Duration::days(40)), + object_key: "codex/2026/06/01/old.dump".to_owned(), + expires_at: None, + }; + let new_seed = BackupRecordSeed { + id: backup_id("new"), + trigger_kind: BackupTriggerKind::Scheduled, + scheduled_at: Some(Utc::now() - Duration::days(1)), + object_key: "codex/2026/07/01/new.dump".to_owned(), + expires_at: None, + }; + repository + .insert_scheduled_record(old_seed.clone()) + .await + .expect("insert old"); + // 旧备份 40 天前完成（释放活跃名额），再插入新备份。 + let now = Utc::now(); + repository.set_completed(&backup_id("old"), now - Duration::days(40)); + repository + .insert_scheduled_record(new_seed.clone()) + .await + .expect("insert new"); + // 新备份 1 天前完成。 + repository.set_completed(&backup_id("new"), now - Duration::days(1)); + // 两个对象都已上传。 + object_store + .objects + .lock() + .expect("objects") + .insert(old_seed.object_key.clone(), DUMP_CONTENT.to_vec()); + object_store + .objects + .lock() + .expect("objects") + .insert(new_seed.object_key.clone(), DUMP_CONTENT.to_vec()); + + // 设置 retention_days = 30。 + repository + .update_schedule_settings( + UpdateBackupScheduleCommand { + schedule_enabled: false, + cron_expression: "0 2 * * *".to_owned(), + schedule_timezone: "Asia/Shanghai".to_owned(), + retention_days: 30, + retention_count: 0, + }, + None, + &system_context(), + ) + .await + .expect("set retention"); + + let task = BackupTask::new(repository.clone(), dump.clone(), object_store.clone()); + task.run_cycle(&CancellationToken::new()) + .await + .expect("run cycle"); + + // 旧备份被硬删除，对象被删除；新备份保留。 + let records = repository.all_records(); + let ids: Vec<_> = records.iter().map(|record| record.id.as_str()).collect(); + assert_eq!(ids, vec![backup_id("new").as_str()]); + assert!(object_store.object(&old_seed.object_key).is_none()); + assert!(object_store.object(&new_seed.object_key).is_some()); +} diff --git a/backend/crates/gateway-admin/tests/main.rs b/backend/crates/gateway-admin/tests/main.rs index 02a84f7dd..e3565c53d 100644 --- a/backend/crates/gateway-admin/tests/main.rs +++ b/backend/crates/gateway-admin/tests/main.rs @@ -17,6 +17,8 @@ use gateway_admin::{ ports::store::{AccountStore, AuthStore, ClientKeyStore, ObservabilityStore, SettingsStore}, }; +mod backup; +mod backup_runtime; mod use_case; #[test] diff --git a/backend/crates/gateway-admin/tests/use_case/backup.rs b/backend/crates/gateway-admin/tests/use_case/backup.rs new file mode 100644 index 000000000..c4faa0d9f --- /dev/null +++ b/backend/crates/gateway-admin/tests/use_case/backup.rs @@ -0,0 +1,104 @@ +//! BackupService 用例测试：通过 AdminHarness 注入 fake 备份端口构造真实服务。 + +use std::sync::Arc; + +use gateway_admin::model::backup::BackupStatus; +use gateway_admin::ports::backup::BackupStorePorts; + +use crate::backup_runtime::{ + FakeAuthStore, FakeBackupRepository, FakeDumpPort, FakeObjectStore, configured_settings, + system_context, +}; + +use super::AdminHarness; + +#[tokio::test] +async fn backup_service_creates_downloads_and_deletes_with_audit() { + let repository = Arc::new(FakeBackupRepository::new(configured_settings())); + let dump = Arc::new(FakeDumpPort::new()); + let object_store = Arc::new(FakeObjectStore::new()); + let auth = Arc::new(FakeAuthStore::new()); + + let services = AdminHarness::new() + .auth(auth.clone()) + .backup(BackupStorePorts::new( + repository.clone(), + dump, + object_store, + )) + .build() + .await; + + // 创建手动备份 → queued + 审计。 + let created = services + .backups() + .create_backup(&system_context(), None) + .await + .expect("create backup"); + assert_eq!(created.status, BackupStatus::Queued); + assert!(created.object_key.starts_with("codex/")); + assert!( + auth.audit_actions() + .iter() + .any(|action| action == "backup.created") + ); + + // 下载地址要求 completed 记录。 + repository.force_status(&created.id, BackupStatus::Completed); + let download = services + .backups() + .download_url(&system_context(), &created.id) + .await + .expect("download url"); + assert!(download.url.starts_with("https://presigned.example.com/")); + assert!( + auth.audit_actions() + .iter() + .any(|action| action == "backup.download_url_created") + ); + + // 删除 → deleting + 审计。 + let deleting = services + .backups() + .delete_backup(&system_context(), &created.id) + .await + .expect("delete backup"); + assert_eq!(deleting.status, BackupStatus::Deleting); + assert!( + auth.audit_actions() + .iter() + .any(|action| action == "backup.delete_requested") + ); +} + +#[tokio::test] +async fn backup_service_rejects_download_of_non_completed_record() { + let repository = Arc::new(FakeBackupRepository::new(configured_settings())); + let dump = Arc::new(FakeDumpPort::new()); + let object_store = Arc::new(FakeObjectStore::new()); + let auth = Arc::new(FakeAuthStore::new()); + + let services = AdminHarness::new() + .auth(auth.clone()) + .backup(BackupStorePorts::new( + repository.clone(), + dump, + object_store, + )) + .build() + .await; + + let created = services + .backups() + .create_backup(&system_context(), None) + .await + .expect("create backup"); + + // queued 记录不能创建下载地址。 + let error = services + .backups() + .download_url(&system_context(), &created.id) + .await + .expect_err("non-completed record must be rejected"); + assert_eq!(error.kind(), gateway_admin::model::AdminErrorKind::Conflict); +} diff --git a/backend/crates/gateway-admin/tests/use_case/mod.rs b/backend/crates/gateway-admin/tests/use_case/mod.rs index abecdf741..576ff5ae5 100644 --- a/backend/crates/gateway-admin/tests/use_case/mod.rs +++ b/backend/crates/gateway-admin/tests/use_case/mod.rs @@ -1,5 +1,6 @@ mod accounts; mod auth; +mod backup; mod client_keys; mod observability; mod openai; @@ -46,6 +47,7 @@ use gateway_admin::{ system::{SystemOperationAccepted, SystemUpdateDetail, SystemUpdateStatus, SystemVersion}, }, ports::{ + backup::BackupStorePorts, provider::{ProviderAdmin, ProviderAdminError, ProviderAdminErrorKind}, store::{ AccountStore, AdminStoreError, AdminStoreErrorKind, AdminStorePorts, AdminStoreResult, @@ -75,6 +77,7 @@ pub(super) struct AdminHarness { client_keys: Arc, observability: Arc, settings: Arc, + backup: BackupStorePorts, providers: Vec>, probe: Arc, system: Arc, @@ -91,6 +94,7 @@ impl AdminHarness { client_keys: unavailable.clone(), observability: unavailable.clone(), settings: unavailable, + backup: BackupStorePorts::disabled(), providers: vec![ Arc::new(UnavailableProvider::new("openai")), Arc::new(UnavailableProvider::new("xai")), @@ -135,6 +139,11 @@ impl AdminHarness { self } + pub(super) fn backup(mut self, backup: BackupStorePorts) -> Self { + self.backup = backup; + self + } + pub(super) fn provider(mut self, provider: Arc) -> Self { self.providers .retain(|registered| registered.provider_kind() != provider.provider_kind()); @@ -165,6 +174,7 @@ impl AdminHarness { self.client_keys, self.observability, self.settings, + self.backup, ), self.providers, Arc::new(NoopSnapshot), diff --git a/backend/crates/gateway-api/Cargo.toml b/backend/crates/gateway-api/Cargo.toml index c7e5a30ee..a9ed466f2 100644 --- a/backend/crates/gateway-api/Cargo.toml +++ b/backend/crates/gateway-api/Cargo.toml @@ -20,6 +20,7 @@ futures = "0.3.32" gateway-admin = { path = "../gateway-admin" } gateway-core = { path = "../gateway-core" } gateway-protocol = { path = "../gateway-protocol" } +secrecy = "0.10" serde = { version = "1.0.228", features = ["derive"] } serde_json = { version = "1.0.150", features = ["arbitrary_precision", "preserve_order"] } thiserror = "2.0.18" diff --git a/backend/crates/gateway-api/src/admin/backups.rs b/backend/crates/gateway-api/src/admin/backups.rs new file mode 100644 index 000000000..23934f658 --- /dev/null +++ b/backend/crates/gateway-api/src/admin/backups.rs @@ -0,0 +1,459 @@ +//! 备份设置、计划、手动创建、记录、下载与删除路由。 +//! +//! 路径统一位于 `/api/admin/settings/backups/*`，内部由独立 `BackupService` 承担业务。 + +use axum::{ + Json, Router, + extract::{Query, State}, + http::StatusCode, + response::IntoResponse, + routing::{get, post}, +}; +use chrono::{DateTime, Utc}; +use gateway_admin::model::backup::{ + BackupRecord, BackupSettings, BackupStatus, BackupTriggerKind, ConnectionTestResult, + UpdateBackupScheduleCommand, UpdateBackupStorageCommand, +}; +use secrecy::{ExposeSecret as _, SecretString}; +use serde::{Deserialize, Serialize}; + +use super::{ + AdminAuth, AdminEnvelope, AdminError, AdminPageData, AdminResponse, AdminSessionState, + PageMeta, wire::map_admin_service_error, +}; + +/// 备份设置视图；`secretAccessKey` 返回已保存的明文凭据，由前端掩码显示。 +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct BackupSettingsView { + pub storage_revision: u64, + pub endpoint: Option, + pub region: Option, + pub bucket: Option, + pub access_key_id: Option, + pub secret_access_key: Option, + pub prefix: Option, + pub force_path_style: bool, + pub verified: bool, + pub schedule_enabled: bool, + pub cron_expression: Option, + pub schedule_timezone: Option, + pub retention_days: u32, + pub retention_count: u32, + pub next_run_at: Option>, + pub last_verified_at: Option>, + pub updated_at: DateTime, +} + +impl From for BackupSettingsView { + fn from(settings: BackupSettings) -> Self { + let verified = settings.storage_verified(); + Self { + storage_revision: settings.storage_revision, + endpoint: settings.endpoint, + region: settings.region, + bucket: settings.bucket, + access_key_id: settings.access_key_id, + secret_access_key: settings + .secret_access_key + .map(|secret| secret.expose_secret().to_owned()), + prefix: settings.prefix, + force_path_style: settings.force_path_style, + verified, + schedule_enabled: settings.schedule_enabled, + cron_expression: settings.cron_expression, + schedule_timezone: settings.schedule_timezone, + retention_days: settings.retention_days, + retention_count: settings.retention_count, + next_run_at: settings.next_run_at, + last_verified_at: settings.last_verified_at, + updated_at: settings.updated_at, + } + } +} + +/// 更新 S3 存储配置请求；`secretAccessKey` 缺省表示保留旧值。 +#[derive(Debug, Clone, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct UpdateBackupStorageRequest { + pub endpoint: String, + pub region: String, + pub bucket: String, + pub access_key_id: String, + #[serde(default)] + pub secret_access_key: Option, + pub prefix: String, + pub force_path_style: bool, +} + +/// 更新调度配置请求。 +#[derive(Debug, Clone, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct UpdateBackupScheduleRequest { + pub schedule_enabled: bool, + pub cron_expression: String, + pub schedule_timezone: String, + pub retention_days: u32, + pub retention_count: u32, +} + +/// 备份记录 wire 视图。 +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct BackupRecordView { + pub id: String, + pub trigger_kind: String, + pub status: String, + pub scheduled_at: Option>, + pub object_key: String, + pub size_bytes: Option, + pub sha256: Option, + pub attempt_count: u32, + pub error_code: Option, + pub error_message: Option, + pub started_at: Option>, + pub completed_at: Option>, + pub expires_at: Option>, + pub created_at: DateTime, + pub updated_at: DateTime, +} + +impl From for BackupRecordView { + fn from(record: BackupRecord) -> Self { + Self { + id: record.id, + trigger_kind: record.trigger_kind.to_string(), + status: record.status.to_string(), + scheduled_at: record.scheduled_at, + object_key: record.object_key, + size_bytes: record.size_bytes, + sha256: record.sha256, + attempt_count: record.attempt_count, + error_code: record.error_code, + error_message: record.error_message, + started_at: record.started_at, + completed_at: record.completed_at, + expires_at: record.expires_at, + created_at: record.created_at, + updated_at: record.updated_at, + } + } +} + +/// 记录列表查询参数。 +#[derive(Debug, Clone, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct BackupRecordsQuery { + pub page: Option, + pub page_size: Option, + pub status: Option, + pub trigger: Option, +} + +/// 下载地址请求。 +#[derive(Debug, Clone, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct BackupIdRequest { + pub backup_id: String, +} + +/// 下载地址响应。 +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct DownloadUrlView { + pub url: String, + pub file_name: String, + pub expires_in_seconds: u64, +} + +impl From for DownloadUrlView { + fn from(result: gateway_admin::model::backup::DownloadUrlResult) -> Self { + Self { + url: result.url, + file_name: result.file_name, + expires_in_seconds: result.expires_in.as_secs(), + } + } +} + +/// 连接测试响应。 +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct ConnectionTestView { + pub ok: bool, + pub stage: String, + pub code: Option, + pub message: String, +} + +impl From for ConnectionTestView { + fn from(result: ConnectionTestResult) -> Self { + Self { + ok: result.ok, + stage: result.stage.to_owned(), + code: result.code.map(str::to_owned), + message: result.message, + } + } +} + +/// 构造固定 GET/POST 备份路由。 +pub fn router() -> Router +where + S: AdminSessionState + Clone + Send + Sync + 'static, +{ + Router::new() + .route("/api/admin/settings/backups", get(backup_settings::)) + .route( + "/api/admin/settings/backups/storage/update", + post(update_backup_storage::), + ) + .route( + "/api/admin/settings/backups/storage/test", + post(test_backup_storage::), + ) + .route( + "/api/admin/settings/backups/schedule/update", + post(update_backup_schedule::), + ) + .route( + "/api/admin/settings/backups/records", + get(backup_records::), + ) + .route( + "/api/admin/settings/backups/create", + post(create_backup::), + ) + .route( + "/api/admin/settings/backups/download-url", + post(download_backup_url::), + ) + .route( + "/api/admin/settings/backups/delete", + post(delete_backup::), + ) +} + +async fn backup_settings( + _auth: AdminAuth, + State(state): State, +) -> Result +where + S: AdminSessionState + Send + Sync, +{ + let settings = state + .admin_services() + .backups() + .load_settings() + .await + .map_err(map_service_error)?; + Ok(AdminResponse::new( + StatusCode::OK, + AdminEnvelope::ok(BackupSettingsView::from(settings)), + )) +} + +async fn update_backup_storage( + auth: AdminAuth, + State(state): State, + Json(request): Json, +) -> Result +where + S: AdminSessionState + Send + Sync, +{ + let command = UpdateBackupStorageCommand { + endpoint: request.endpoint, + region: request.region, + bucket: request.bucket, + access_key_id: request.access_key_id, + secret_access_key: request.secret_access_key.map(SecretString::from), + prefix: request.prefix, + force_path_style: request.force_path_style, + }; + let settings = state + .admin_services() + .backups() + .update_storage(&auth.context().mutation_context(), command) + .await + .map_err(map_service_error)?; + Ok(AdminResponse::new( + StatusCode::OK, + AdminEnvelope::ok(BackupSettingsView::from(settings)), + )) +} + +async fn test_backup_storage( + auth: AdminAuth, + State(state): State, +) -> Result +where + S: AdminSessionState + Send + Sync, +{ + let result = state + .admin_services() + .backups() + .test_connection(&auth.context().mutation_context()) + .await + .map_err(map_service_error)?; + Ok(AdminResponse::new( + StatusCode::OK, + AdminEnvelope::ok(ConnectionTestView::from(result)), + )) +} + +async fn update_backup_schedule( + auth: AdminAuth, + State(state): State, + Json(request): Json, +) -> Result +where + S: AdminSessionState + Send + Sync, +{ + let command = UpdateBackupScheduleCommand { + schedule_enabled: request.schedule_enabled, + cron_expression: request.cron_expression, + schedule_timezone: request.schedule_timezone, + retention_days: request.retention_days, + retention_count: request.retention_count, + }; + let settings = state + .admin_services() + .backups() + .update_schedule(&auth.context().mutation_context(), command) + .await + .map_err(map_service_error)?; + Ok(AdminResponse::new( + StatusCode::OK, + AdminEnvelope::ok(BackupSettingsView::from(settings)), + )) +} + +async fn backup_records( + _auth: AdminAuth, + State(state): State, + Query(query): Query, +) -> Result +where + S: AdminSessionState + Send + Sync, +{ + let page_size = gateway_admin::model::PageSize::new(query.page_size.unwrap_or(20)) + .map_err(|_| AdminError::bad_request("Invalid pageSize"))?; + let page = query.page.unwrap_or(1); + if page == 0 { + return Err(AdminError::bad_request("Invalid page")); + } + let status = match query.status.as_deref() { + None => None, + Some(value) => BackupStatus::parse(value) + .map(Some) + .ok_or_else(|| AdminError::bad_request("Invalid status"))?, + }; + let trigger = match query.trigger.as_deref() { + None => None, + Some(value) => BackupTriggerKind::parse(value) + .map(Some) + .ok_or_else(|| AdminError::bad_request("Invalid trigger"))?, + }; + let domain_query = gateway_admin::model::backup::BackupRecordListQuery { + page, + page_size, + status, + trigger, + }; + let page_result = state + .admin_services() + .backups() + .list_records(domain_query) + .await + .map_err(map_service_error)?; + let meta = PageMeta::new( + page_result.page, + u32::from(page_result.page_size.get()), + page_result.total, + page_result.total_pages(), + ); + let items: Vec = page_result + .items + .into_iter() + .map(BackupRecordView::from) + .collect(); + Ok(AdminResponse::new( + StatusCode::OK, + AdminEnvelope::ok(AdminPageData::new(items, meta)), + )) +} + +async fn create_backup( + auth: AdminAuth, + State(state): State, + body: Option>, +) -> Result +where + S: AdminSessionState + Send + Sync, +{ + let expires_at = body + .and_then(|body| body.0.expires_in_days) + .filter(|&days| days > 0) + .map(|days| Utc::now() + chrono::Duration::days(i64::from(days))); + let record = state + .admin_services() + .backups() + .create_backup(&auth.context().mutation_context(), expires_at) + .await + .map_err(map_service_error)?; + Ok(AdminResponse::new( + StatusCode::ACCEPTED, + AdminEnvelope::ok(BackupRecordView::from(record)), + )) +} + +/// 创建备份请求；`expiresInDays` 为 0 或缺省表示不设置过期时间。 +#[derive(Debug, Clone, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct CreateBackupRequest { + pub expires_in_days: Option, +} + +async fn download_backup_url( + auth: AdminAuth, + State(state): State, + Json(request): Json, +) -> Result +where + S: AdminSessionState + Send + Sync, +{ + let result = state + .admin_services() + .backups() + .download_url(&auth.context().mutation_context(), &request.backup_id) + .await + .map_err(map_service_error)?; + Ok(AdminResponse::new( + StatusCode::OK, + AdminEnvelope::ok(DownloadUrlView::from(result)), + )) +} + +async fn delete_backup( + auth: AdminAuth, + State(state): State, + Json(request): Json, +) -> Result +where + S: AdminSessionState + Send + Sync, +{ + let record = state + .admin_services() + .backups() + .delete_backup(&auth.context().mutation_context(), &request.backup_id) + .await + .map_err(map_service_error)?; + Ok(AdminResponse::new( + StatusCode::OK, + AdminEnvelope::ok(BackupRecordView::from(record)), + )) +} + +fn map_service_error(error: gateway_admin::model::AdminError) -> AdminError { + map_admin_service_error(error, "Backup service unavailable") +} diff --git a/backend/crates/gateway-api/src/admin/mod.rs b/backend/crates/gateway-api/src/admin/mod.rs index 3fdb36e07..7e78713f6 100644 --- a/backend/crates/gateway-api/src/admin/mod.rs +++ b/backend/crates/gateway-api/src/admin/mod.rs @@ -9,6 +9,7 @@ use axum::{ pub mod accounts; pub mod auth; +pub mod backups; pub mod client_keys; pub mod observability; pub mod settings; @@ -29,6 +30,7 @@ where Router::new() .merge(accounts::router::()) .merge(auth::router::()) + .merge(backups::router::()) .merge(client_keys::router::()) .merge(observability::router::()) .merge(settings::router::()) diff --git a/backend/crates/gateway-api/tests/admin/mod.rs b/backend/crates/gateway-api/tests/admin/mod.rs index 86d615279..c08dfdc34 100644 --- a/backend/crates/gateway-api/tests/admin/mod.rs +++ b/backend/crates/gateway-api/tests/admin/mod.rs @@ -113,6 +113,7 @@ impl AdminTestFixture { client_keys.clone(), unused, settings.clone(), + gateway_admin::ports::backup::BackupStorePorts::disabled(), ); let providers: Vec> = vec![ Arc::new(UnusedProvider::new("openai")), diff --git a/backend/crates/gateway-api/tests/architecture.rs b/backend/crates/gateway-api/tests/architecture.rs index ff4125e71..284ea6c9d 100644 --- a/backend/crates/gateway-api/tests/architecture.rs +++ b/backend/crates/gateway-api/tests/architecture.rs @@ -35,6 +35,7 @@ fn source_tree_should_match_frozen_machine_manifest() { let mut expected = vec![ "src/admin/accounts.rs", "src/admin/auth.rs", + "src/admin/backups.rs", "src/admin/client_keys.rs", "src/admin/mod.rs", "src/admin/observability.rs", diff --git a/backend/crates/gateway-core/src/task/mod.rs b/backend/crates/gateway-core/src/task/mod.rs index 1e2bfa27d..7c2aad806 100644 --- a/backend/crates/gateway-core/src/task/mod.rs +++ b/backend/crates/gateway-core/src/task/mod.rs @@ -21,10 +21,11 @@ pub enum WorkerKind { StaleModelRequestRecovery, Retention, OpsFlush, + Backup, } impl WorkerKind { - pub const ALL: [Self; 7] = [ + pub const ALL: [Self; 8] = [ Self::OAuthRefresh, Self::QuotaCatalogHealth, Self::RuntimeSnapshotReconciliation, @@ -32,6 +33,7 @@ impl WorkerKind { Self::StaleModelRequestRecovery, Self::Retention, Self::OpsFlush, + Self::Backup, ]; #[must_use] @@ -44,6 +46,7 @@ impl WorkerKind { Self::StaleModelRequestRecovery => "stale_model_request_recovery", Self::Retention => "retention", Self::OpsFlush => "ops_flush", + Self::Backup => "backup", } } } diff --git a/backend/crates/gateway-host/tests/workers/mod.rs b/backend/crates/gateway-host/tests/workers/mod.rs index 7205e23ff..d5e96fa25 100644 --- a/backend/crates/gateway-host/tests/workers/mod.rs +++ b/backend/crates/gateway-host/tests/workers/mod.rs @@ -17,13 +17,14 @@ use gateway_core::task::{ use gateway_host::workers::{WorkerStartError, WorkerSupervisor}; use tokio::sync::Notify; -const ACTIVE_KINDS: [WorkerKind; 6] = [ +const ACTIVE_KINDS: [WorkerKind; 7] = [ WorkerKind::OAuthRefresh, WorkerKind::QuotaCatalogHealth, WorkerKind::RuntimeSnapshotReconciliation, WorkerKind::RuntimeChangeSubscription, WorkerKind::StaleModelRequestRecovery, WorkerKind::Retention, + WorkerKind::Backup, ]; #[derive(Clone)] diff --git a/backend/crates/gateway-store/Cargo.toml b/backend/crates/gateway-store/Cargo.toml index 8d29fede9..9fd8d13a0 100644 --- a/backend/crates/gateway-store/Cargo.toml +++ b/backend/crates/gateway-store/Cargo.toml @@ -13,24 +13,31 @@ path = "tests/main.rs" [dependencies] async-trait = "0.1.89" +aws-credential-types = "1.2" +aws-sdk-s3 = { version = "1.140", features = ["rt-tokio", "rustls"] } +aws-smithy-runtime-api = "1.14" +aws-smithy-types = "1.6" chrono = "0.4.45" +fs2 = "0.4" futures = "0.3.32" gateway-admin = { path = "../gateway-admin" } gateway-core = { path = "../gateway-core" } hex = "0.4.3" redis = { version = "1.3.0", default-features = false, features = ["tokio-comp", "connection-manager", "script"] } +secrecy = "0.10" serde = { version = "1.0.228", features = ["derive"] } serde_json = { version = "1.0.150", features = ["arbitrary_precision", "preserve_order"] } sha2 = "0.11.0" sqlx = { version = "0.9.0", default-features = false, features = ["runtime-tokio", "tls-rustls", "postgres", "chrono", "json", "macros", "migrate"] } thiserror = "2.0.18" -tokio = { version = "1.52.3", default-features = false, features = ["rt", "time"] } +tokio = { version = "1.52", default-features = false, features = ["fs", "io-std", "io-util", "macros", "process", "rt", "sync", "time"] } tracing = "0.1.44" url = "2.5.8" uuid = { version = "1.23.3", features = ["v4", "v7"] } [dev-dependencies] -tokio = { version = "1.52.3", default-features = false, features = ["macros", "rt-multi-thread"] } +tempfile = "3.23" +tokio = { version = "1.52.3", default-features = false, features = ["macros", "rt-multi-thread", "fs", "io-util", "process", "time"] } [lints] workspace = true diff --git a/backend/crates/gateway-store/src/backup/mod.rs b/backend/crates/gateway-store/src/backup/mod.rs new file mode 100644 index 000000000..38dd212ad --- /dev/null +++ b/backend/crates/gateway-store/src/backup/mod.rs @@ -0,0 +1,5 @@ +//! S3 对象存储与 `pg_dump` 基础设施适配器。 + +pub mod pg_dump; +pub mod s3; +pub mod staging; diff --git a/backend/crates/gateway-store/src/backup/pg_dump.rs b/backend/crates/gateway-store/src/backup/pg_dump.rs new file mode 100644 index 000000000..31a49a03e --- /dev/null +++ b/backend/crates/gateway-store/src/backup/pg_dump.rs @@ -0,0 +1,205 @@ +//! `pg_dump` 子进程与本地暂存适配器。 +//! +//! 数据库密码只通过 `PGPASSWORD` 环境变量传给子进程，绝不进入命令行参数或日志。 +//! 导出以有界内存流式写入暂存文件并计算 SHA-256；取消时终止子进程并清理部分文件。 + +use std::process::Stdio; +use std::sync::Arc; + +use async_trait::async_trait; +use sha2::{Digest as _, Sha256}; +use tokio::io::{AsyncReadExt as _, AsyncWriteExt as _}; +use tokio::process::Command; + +use gateway_admin::model::backup::{BackupError, code}; +use gateway_admin::ports::backup::{DatabaseDumpPort, DumpArtifact, DumpRequest, StagedArtifact}; + +use super::staging::StagingArea; + +/// `pg_dump` 导出适配器。 +pub struct PgDumpAdapter { + staging: Arc, + /// 不含密码的 PostgreSQL URL，用于 `--dbname`。 + database_url: String, + /// 数据库密码，仅通过 `PGPASSWORD` 注入。 + database_password: String, +} + +impl PgDumpAdapter { + /// 组合暂存区与数据库连接事实。 + #[must_use] + pub fn new( + staging: Arc, + database_url: impl Into, + database_password: impl Into, + ) -> Self { + Self { + staging, + database_url: database_url.into(), + database_password: database_password.into(), + } + } +} + +#[async_trait] +impl DatabaseDumpPort for PgDumpAdapter { + async fn dump(&self, request: DumpRequest) -> Result { + self.staging + .ensure_capacity() + .map_err(|_| staging_space_exhausted())?; + let partial = self.staging.partial_path(&request.backup_id); + let final_path = self.staging.final_path(&request.backup_id); + + let mut child = Command::new("pg_dump") + .arg("--format=custom") + .arg("--no-owner") + .arg("--no-privileges") + .arg("--dbname") + .arg(&self.database_url) + .env("PGPASSWORD", &self.database_password) + .env("PGCONNECT_TIMEOUT", "10") + .stdout(Stdio::piped()) + .stderr(Stdio::null()) + .spawn() + .map_err(|_| pg_dump_failed("无法启动 pg_dump 进程"))?; + let stdout = child + .stdout + .take() + .ok_or_else(|| pg_dump_failed("无法读取 pg_dump 输出"))?; + + let file = tokio::fs::OpenOptions::new() + .create(true) + .write(true) + .truncate(true) + .mode(0o600) + .open(&partial) + .await + .map_err(|_| pg_dump_failed("无法创建暂存文件"))?; + let mut writer = tokio::io::BufWriter::new(file); + let mut reader = tokio::io::BufReader::new(stdout); + let mut hasher = Sha256::new(); + let mut size: u64 = 0; + + let copy = async { + let mut buffer = vec![0_u8; 64 * 1024]; + loop { + let n = reader + .read(&mut buffer) + .await + .map_err(|_| pg_dump_failed("读取 pg_dump 输出失败"))?; + if n == 0 { + break; + } + size = size.saturating_add(n as u64); + if size > self.staging.max_archive_bytes() { + return Err(staging_space_exhausted()); + } + hasher.update(&buffer[..n]); + writer + .write_all(&buffer[..n]) + .await + .map_err(|_| pg_dump_failed("写入暂存文件失败"))?; + } + writer + .flush() + .await + .map_err(|_| pg_dump_failed("刷新暂存文件失败"))?; + Ok(()) + }; + + let copy_result = tokio::select! { + result = copy => result, + _ = request.cancellation.cancelled() => { + let _ = child.kill().await; + Err(cancelled()) + } + }; + + if let Err(error) = copy_result { + let _ = child.kill().await; + let _ = child.wait().await; + self.staging.cleanup(&request.backup_id); + return Err(error); + } + + let status = child + .wait() + .await + .map_err(|_| pg_dump_failed("等待 pg_dump 退出失败"))?; + if !status.success() { + self.staging.cleanup(&request.backup_id); + return Err(pg_dump_failed("pg_dump 非零退出")); + } + + tokio::fs::rename(&partial, &final_path) + .await + .map_err(|_| pg_dump_failed("暂存归档改名失败"))?; + let sha256 = hex::encode(hasher.finalize()); + Ok(DumpArtifact { + path: final_path, + size_bytes: size, + sha256, + }) + } + + async fn inspect_staging( + &self, + backup_id: &str, + ) -> Result, BackupError> { + let path = self.staging.final_path(backup_id); + let metadata = match tokio::fs::metadata(&path).await { + Ok(metadata) => metadata, + Err(_) => return Ok(None), + }; + if !metadata.is_file() { + return Ok(None); + } + let size_bytes = metadata.len(); + let sha256 = hash_file(&path).await?; + Ok(Some(StagedArtifact { + path, + size_bytes, + sha256, + })) + } + + async fn cleanup_staging(&self, backup_id: &str) -> Result<(), BackupError> { + self.staging.cleanup(backup_id); + Ok(()) + } +} + +/// 以有界内存读取文件并计算 SHA-256。 +async fn hash_file(path: &std::path::Path) -> Result { + let mut file = tokio::fs::File::open(path) + .await + .map_err(|_| pg_dump_failed("打开暂存归档失败"))?; + let mut hasher = Sha256::new(); + let mut buffer = vec![0_u8; 64 * 1024]; + loop { + let n = file + .read(&mut buffer) + .await + .map_err(|_| pg_dump_failed("读取暂存归档失败"))?; + if n == 0 { + break; + } + hasher.update(&buffer[..n]); + } + Ok(hex::encode(hasher.finalize())) +} + +fn pg_dump_failed(message: &'static str) -> BackupError { + BackupError::new(code::PG_DUMP_FAILED, message.to_owned()) +} + +fn staging_space_exhausted() -> BackupError { + BackupError::new( + code::STAGING_SPACE_EXHAUSTED, + "暂存磁盘空间不足或归档超过单任务上限".to_owned(), + ) +} + +fn cancelled() -> BackupError { + BackupError::new(code::CANCELLED, "备份导出已取消".to_owned()) +} diff --git a/backend/crates/gateway-store/src/backup/s3.rs b/backend/crates/gateway-store/src/backup/s3.rs new file mode 100644 index 000000000..e742ee8d0 --- /dev/null +++ b/backend/crates/gateway-store/src/backup/s3.rs @@ -0,0 +1,544 @@ +//! S3 兼容对象存储适配器（含 Cloudflare R2）。 +//! +//! 单个 `BackupObjectStorePort` 实现：方法接受已经校验的存储配置，内部按 +//! `storage_revision` 缓存 SDK client。上传采用分片流式读取，内存占用有界； +//! 取消时主动 `AbortMultipartUpload`。 + +use std::collections::HashMap; +use std::path::Path; +use std::sync::{ + Arc, Mutex, + atomic::{AtomicU64, Ordering}, +}; +use std::time::Duration; + +use aws_credential_types::Credentials; +use aws_sdk_s3::config::{BehaviorVersion, Builder as S3ConfigBuilder, Region}; +use aws_sdk_s3::primitives::ByteStream; +use aws_sdk_s3::types::{CompletedMultipartUpload, CompletedPart}; +use aws_sdk_s3::{Client as S3Client, presigning::PresigningConfig}; +use aws_smithy_runtime_api::client::orchestrator::HttpResponse; +use aws_smithy_runtime_api::client::result::SdkError; +use aws_smithy_types::error::metadata::ProvideErrorMetadata; +use aws_smithy_types::retry::RetryConfig; +use aws_smithy_types::timeout::TimeoutConfig; +use chrono::{DateTime, Utc}; +use secrecy::ExposeSecret as _; +use tokio::io::{AsyncReadExt as _, AsyncSeekExt as _, SeekFrom}; +use tokio::sync::Semaphore; +use uuid::Uuid; + +use gateway_admin::model::backup::{ + BackupError, BackupObjectMetadata, BackupStorageConfig, ConnectionTestResult, + ConnectionTestStage, code, +}; +use gateway_admin::ports::backup::{BackupObjectStorePort, UploadObjectRequest}; + +/// 分片大小：16 MiB。 +const PART_SIZE: usize = 16 * 1024 * 1024; +/// 并发上传分片数；最大额外内存约为 `PART_SIZE * 并发数`。 +const CONCURRENCY: usize = 4; +/// SDK 单次请求超时。 +const OPERATION_TIMEOUT: Duration = Duration::from_secs(300); + +/// 探针对象的固定 metadata 值。 +const PROBE_METADATA_KEY: &str = "probe"; +const PROBE_METADATA_VALUE: &str = "backup-connection-test"; + +/// S3 兼容对象存储适配器。 +pub struct S3ObjectStoreAdapter { + clients: Mutex>, +} + +impl Default for S3ObjectStoreAdapter { + fn default() -> Self { + Self::new() + } +} + +impl S3ObjectStoreAdapter { + /// 创建适配器；client 按 `storage_revision` 惰性构建并缓存。 + #[must_use] + pub fn new() -> Self { + Self { + clients: Mutex::new(HashMap::new()), + } + } + + fn client(&self, config: &BackupStorageConfig) -> S3Client { + if let Some(client) = self + .clients + .lock() + .ok() + .and_then(|guard| guard.get(&config.storage_revision).cloned()) + { + return client; + } + let client = build_client(config); + if let Ok(mut guard) = self.clients.lock() { + guard.insert(config.storage_revision, client.clone()); + } + client + } +} + +fn build_client(config: &BackupStorageConfig) -> S3Client { + let retry = RetryConfig::standard().with_max_attempts(3); + let timeout = TimeoutConfig::builder() + .connect_timeout(Duration::from_secs(10)) + .read_timeout(OPERATION_TIMEOUT) + .build(); + let s3_config = S3ConfigBuilder::new() + .behavior_version(BehaviorVersion::latest()) + .region(Region::new(config.region.clone())) + .endpoint_url(&config.endpoint) + .force_path_style(config.force_path_style) + .credentials_provider(Credentials::new( + config.access_key_id.clone(), + config.secret_access_key.expose_secret().to_string(), + None, + None, + "static", + )) + .retry_config(retry) + .timeout_config(timeout) + .build(); + S3Client::from_conf(s3_config) +} + +#[async_trait::async_trait] +impl BackupObjectStorePort for S3ObjectStoreAdapter { + async fn test_connection( + &self, + config: &BackupStorageConfig, + ) -> Result { + let client = self.client(config); + let probe_key = format!( + "{}/.probe/{}", + config.prefix.trim_end_matches('/'), + Uuid::new_v4() + ); + let body: Vec = (0_u8..64).collect(); + + if let Err(error) = client + .put_object() + .bucket(&config.bucket) + .key(&probe_key) + .body(ByteStream::from(body.clone())) + .metadata(PROBE_METADATA_KEY, PROBE_METADATA_VALUE) + .send() + .await + { + let _ = best_effort_delete(&client, config, &probe_key).await; + return Ok(failed(ConnectionTestStage::PutObject, &error)); + } + + match client + .head_object() + .bucket(&config.bucket) + .key(&probe_key) + .send() + .await + { + Ok(head) => { + let length_matches = head.content_length() == Some(body.len() as i64); + let metadata_matches = head + .metadata() + .and_then(|metadata| metadata.get(PROBE_METADATA_KEY)) + .is_some_and(|value| value == PROBE_METADATA_VALUE); + if !length_matches || !metadata_matches { + let _ = best_effort_delete(&client, config, &probe_key).await; + return Ok(ConnectionTestResult { + ok: false, + stage: ConnectionTestStage::HeadObject.as_str(), + code: Some(code::UPSTREAM_FAILURE), + message: "探针对象校验不一致".to_owned(), + }); + } + } + Err(error) => { + let _ = best_effort_delete(&client, config, &probe_key).await; + return Ok(failed(ConnectionTestStage::HeadObject, &error)); + } + } + + match client + .get_object() + .bucket(&config.bucket) + .key(&probe_key) + .send() + .await + { + Ok(get) => { + let received = get + .body + .collect() + .await + .map_err(|_| { + BackupError::new(code::UPSTREAM_FAILURE, "读取探针对象响应失败".to_owned()) + })? + .into_bytes(); + if received.as_ref() != body.as_slice() { + let _ = best_effort_delete(&client, config, &probe_key).await; + return Ok(ConnectionTestResult { + ok: false, + stage: ConnectionTestStage::GetObject.as_str(), + code: Some(code::UPSTREAM_FAILURE), + message: "探针对象内容不一致".to_owned(), + }); + } + } + Err(error) => { + let _ = best_effort_delete(&client, config, &probe_key).await; + return Ok(failed(ConnectionTestStage::GetObject, &error)); + } + } + + let _ = best_effort_delete(&client, config, &probe_key).await; + Ok(ConnectionTestResult { + ok: true, + stage: ConnectionTestStage::DeleteObject.as_str(), + code: None, + message: "连接测试成功".to_owned(), + }) + } + + async fn upload_file( + &self, + config: &BackupStorageConfig, + request: UploadObjectRequest, + ) -> Result<(), BackupError> { + let client = self.client(config); + let file_size = tokio::fs::metadata(&request.source) + .await + .map_err(|_| BackupError::new(code::S3_UPLOAD_FAILED, "读取暂存归档失败".to_owned()))? + .len(); + if file_size == 0 { + return Err(BackupError::new( + code::S3_UPLOAD_FAILED, + "暂存归档为空，拒绝上传".to_owned(), + )); + } + + let created_at = request.metadata.created_at.to_rfc3339(); + let create = client + .create_multipart_upload() + .bucket(&config.bucket) + .key(&request.object_key) + .metadata("backup-id", &request.metadata.backup_id) + .metadata("sha256", &request.metadata.sha256) + .metadata("created-at", &created_at) + .send() + .await + .map_err(map_s3_error)?; + let upload_id = create + .upload_id() + .ok_or_else(|| { + BackupError::new( + code::S3_UPLOAD_FAILED, + "S3 未返回 multipart upload id".to_owned(), + ) + })? + .to_owned(); + + let upload = upload_parts(&client, config, &request, upload_id.clone(), file_size); + let outcome = tokio::select! { + result = upload => result, + _ = request.cancellation.cancelled() => { + Err(BackupError::new(code::CANCELLED, "上传已取消".to_owned())) + } + }; + if let Err(error) = &outcome { + let _ = client + .abort_multipart_upload() + .bucket(&config.bucket) + .key(&request.object_key) + .upload_id(&upload_id) + .send() + .await; + if error.code() != code::CANCELLED { + let _ = client + .delete_object() + .bucket(&config.bucket) + .key(&request.object_key) + .send() + .await; + } + } + outcome + } + + async fn head_object( + &self, + config: &BackupStorageConfig, + object_key: &str, + ) -> Result, BackupError> { + let client = self.client(config); + match client + .head_object() + .bucket(&config.bucket) + .key(object_key) + .send() + .await + { + Ok(head) => { + let size_bytes = head.content_length().filter(|&value| value >= 0); + let metadata = head.metadata().cloned().unwrap_or_default(); + let sha256 = metadata.get("sha256").cloned().ok_or_else(|| { + BackupError::new( + code::REMOTE_VERIFICATION_FAILED, + "远端对象缺少 sha256 metadata".to_owned(), + ) + })?; + let backup_id = metadata.get("backup-id").cloned().unwrap_or_default(); + let created_at = metadata + .get("created-at") + .and_then(|value| DateTime::parse_from_rfc3339(value).ok()) + .map(|value| value.with_timezone(&Utc)) + .unwrap_or_else(Utc::now); + let Some(size_bytes) = size_bytes else { + return Err(BackupError::new( + code::REMOTE_VERIFICATION_FAILED, + "远端对象缺少大小".to_owned(), + )); + }; + Ok(Some(BackupObjectMetadata::new( + backup_id, + sha256, + created_at, + size_bytes as u64, + ))) + } + Err(error) if is_not_found(&error) => Ok(None), + Err(error) => Err(map_s3_error(error)), + } + } + + async fn delete_object( + &self, + config: &BackupStorageConfig, + object_key: &str, + ) -> Result<(), BackupError> { + let client = self.client(config); + client + .delete_object() + .bucket(&config.bucket) + .key(object_key) + .send() + .await + .map(|_| ()) + .map_err(map_s3_error) + } + + async fn presigned_download( + &self, + config: &BackupStorageConfig, + object_key: &str, + file_name: &str, + ttl: Duration, + ) -> Result { + let client = self.client(config); + let presigning = PresigningConfig::expires_in(ttl) + .map_err(|_| BackupError::new(code::INVALID_CONFIG, "下载地址有效期非法".to_owned()))?; + let request = client + .get_object() + .bucket(&config.bucket) + .key(object_key) + .response_content_disposition(format!("attachment; filename=\"{file_name}\"")) + .presigned(presigning) + .await + .map_err(map_s3_error)?; + Ok(request.uri().to_owned()) + } +} + +/// 并发分片上传 + CompleteMultipartUpload。 +async fn upload_parts( + client: &S3Client, + config: &BackupStorageConfig, + request: &UploadObjectRequest, + upload_id: String, + file_size: u64, +) -> Result<(), BackupError> { + let total_parts = file_size.div_ceil(u64::try_from(PART_SIZE).unwrap_or(u64::MAX)); + let next_part = Arc::new(AtomicU64::new(1)); + let semaphore = Arc::new(Semaphore::new(CONCURRENCY)); + let (sender, mut receiver) = tokio::sync::mpsc::channel::<(i32, String)>(CONCURRENCY * 2); + + let mut workers = Vec::with_capacity(CONCURRENCY); + for _ in 0..CONCURRENCY { + let client = client.clone(); + let semaphore = Arc::clone(&semaphore); + let next_part = Arc::clone(&next_part); + let sender = sender.clone(); + let source = request.source.clone(); + let bucket = config.bucket.clone(); + let key = request.object_key.clone(); + let upload_id = upload_id.clone(); + workers.push(tokio::spawn(async move { + loop { + let part = next_part.fetch_add(1, Ordering::Relaxed); + if part > total_parts { + break; + } + let _permit = semaphore.acquire().await.map_err(|_| { + BackupError::new(code::S3_UPLOAD_FAILED, "上传并发闸门关闭".to_owned()) + })?; + let offset = (part - 1) * (PART_SIZE as u64); + let bytes = read_part(&source, offset, PART_SIZE).await?; + let part_number = i32::try_from(part).map_err(|_| { + BackupError::new(code::S3_UPLOAD_FAILED, "分片号溢出".to_owned()) + })?; + let response = client + .upload_part() + .bucket(&bucket) + .key(&key) + .upload_id(&upload_id) + .part_number(part_number) + .body(ByteStream::from(bytes)) + .send() + .await + .map_err(map_s3_error)?; + let etag = response + .e_tag() + .ok_or_else(|| { + BackupError::new(code::S3_UPLOAD_FAILED, "S3 未返回分片 ETag".to_owned()) + })? + .to_owned(); + if sender.send((part_number, etag)).await.is_err() { + break; + } + } + Ok::<(), BackupError>(()) + })); + } + drop(sender); + + let mut parts = Vec::new(); + while let Some((part, etag)) = receiver.recv().await { + parts.push((part, etag)); + } + for worker in workers { + worker.await.map_err(|_| { + BackupError::new(code::S3_UPLOAD_FAILED, "上传分片任务崩溃".to_owned()) + })??; + } + parts.sort_by_key(|(part, _)| *part); + + let completed = CompletedMultipartUpload::builder() + .set_parts(Some( + parts + .into_iter() + .map(|(part, etag)| { + CompletedPart::builder() + .part_number(part) + .e_tag(etag) + .build() + }) + .collect(), + )) + .build(); + client + .complete_multipart_upload() + .bucket(&config.bucket) + .key(&request.object_key) + .upload_id(&upload_id) + .multipart_upload(completed) + .send() + .await + .map(|_| ()) + .map_err(map_s3_error) +} + +/// 从文件中读取一个分片（最多 `PART_SIZE` 字节）。 +async fn read_part(source: &Path, offset: u64, max_len: usize) -> Result, BackupError> { + let mut file = tokio::fs::File::open(source) + .await + .map_err(|_| BackupError::new(code::S3_UPLOAD_FAILED, "打开暂存归档失败".to_owned()))?; + file.seek(SeekFrom::Start(offset)) + .await + .map_err(|_| BackupError::new(code::S3_UPLOAD_FAILED, "定位暂存归档失败".to_owned()))?; + let mut buffer = vec![0_u8; max_len]; + let n = file + .read(&mut buffer) + .await + .map_err(|_| BackupError::new(code::S3_UPLOAD_FAILED, "读取暂存归档失败".to_owned()))?; + buffer.truncate(n); + Ok(buffer) +} + +async fn best_effort_delete( + client: &S3Client, + config: &BackupStorageConfig, + object_key: &str, +) -> Result<(), BackupError> { + client + .delete_object() + .bucket(&config.bucket) + .key(object_key) + .send() + .await + .map(|_| ()) + .map_err(map_s3_error) +} + +/// 探测失败结果。 +fn failed( + stage: ConnectionTestStage, + error: &SdkError, +) -> ConnectionTestResult { + let result = ConnectionTestResult { + ok: false, + stage: stage.as_str(), + code: Some(s3_error_code(error)), + message: s3_error_message(error), + }; + tracing::warn!( + stage = result.stage, + code = result.code.unwrap_or_default(), + message = %result.message, + "S3 连接探针失败" + ); + result +} + +/// 稳定错误码。 +fn s3_error_code(error: &SdkError) -> &'static str { + match status_of(error) { + Some(401) => code::S3_AUTH_FAILED, + Some(403) => code::S3_PERMISSION_DENIED, + Some(404) => code::UPSTREAM_FAILURE, + Some(500..=599) => code::UPSTREAM_FAILURE, + _ => code::UPSTREAM_FAILURE, + } +} + +fn s3_error_message(error: &SdkError) -> String { + // 只取稳定的 S3 错误码（如 AccessDenied / NoSuchBucket / SignatureDoesNotMatch）， + // 不拼入原始 message，避免泄露请求细节。 + let detail = error.code().unwrap_or("unknown"); + match status_of(error) { + Some(401) => format!("当前凭据认证失败（{detail}）"), + Some(403) => format!("当前凭据没有目标前缀的读写权限（{detail}）"), + Some(404) => format!("目标存储或对象不存在（{detail}）"), + Some(500..=599) => format!("上游对象存储服务返回失败响应（{detail}）"), + _ => format!("对象存储请求失败（{detail}）"), + } +} + +/// 提取 HTTP 状态码；非 ServiceError 返回 `None`。 +fn status_of(error: &SdkError) -> Option { + match error { + SdkError::ServiceError(service) => Some(service.raw().status().as_u16()), + _ => None, + } +} + +/// 对象不存在的判定（HeadObject 的 404）。 +fn is_not_found(error: &SdkError) -> bool { + status_of(error) == Some(404) +} + +fn map_s3_error(error: SdkError) -> BackupError { + BackupError::new(s3_error_code(&error), s3_error_message(&error)) +} diff --git a/backend/crates/gateway-store/src/backup/staging.rs b/backend/crates/gateway-store/src/backup/staging.rs new file mode 100644 index 000000000..fe8be28b1 --- /dev/null +++ b/backend/crates/gateway-store/src/backup/staging.rs @@ -0,0 +1,104 @@ +//! 受控暂存目录：权限校验、磁盘空间检查与原子改名。 + +use std::path::{Path, PathBuf}; + +use crate::{StoreError, StoreResult}; + +/// 单任务暂存归档上限（默认 64 GiB）。 +pub const DEFAULT_MAX_ARCHIVE_BYTES: u64 = 64 * 1024 * 1024 * 1024; +/// 暂存目录所需的最小剩余空间；与单归档上限解耦，避免小磁盘环境被 64 GiB 门槛误拒。 +const MIN_STAGING_FREE_BYTES: u64 = 1024 * 1024 * 1024; + +/// 备份暂存区。只负责文件系统；归档校验与上传由调用方负责。 +#[derive(Debug, Clone)] +pub struct StagingArea { + base_dir: PathBuf, + max_archive_bytes: u64, +} + +impl StagingArea { + /// 创建暂存区并确保目录存在、权限为仅运行用户可读写（0700）。 + /// + /// # Errors + /// + /// 目录创建或权限设置失败时返回 [`StoreError`]。 + pub fn open(base_dir: PathBuf, max_archive_bytes: u64) -> StoreResult { + std::fs::create_dir_all(&base_dir).map_err(|_| unavailable("create staging directory"))?; + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt as _; + std::fs::set_permissions(&base_dir, std::fs::Permissions::from_mode(0o700)) + .map_err(|_| unavailable("set staging directory permissions"))?; + } + let metadata = std::fs::metadata(&base_dir) + .map_err(|_| unavailable("read staging directory metadata"))?; + if !metadata.is_dir() { + return Err(invalid("staging path is not a directory")); + } + Ok(Self { + base_dir, + max_archive_bytes, + }) + } + + /// 未完成归档的暂存路径。 + #[must_use] + pub fn partial_path(&self, backup_id: &str) -> PathBuf { + self.base_dir.join(format!("{backup_id}.partial")) + } + + /// 完成归档的暂存路径。 + #[must_use] + pub fn final_path(&self, backup_id: &str) -> PathBuf { + self.base_dir.join(format!("{backup_id}.dump")) + } + + /// 单任务暂存归档上限。 + #[must_use] + pub const fn max_archive_bytes(&self) -> u64 { + self.max_archive_bytes + } + + /// 检查剩余磁盘空间是否足够暂存一个归档。 + /// + /// 只要求保留一个基本工作余量；单个归档的硬上限由 + /// [`Self::max_archive_bytes`] 在写入时兜底。 + /// + /// # Errors + /// + /// 剩余空间不足或读取失败时返回 [`crate::StoreError`]。 + pub fn ensure_capacity(&self) -> StoreResult<()> { + let free = fs2::available_space(&self.base_dir) + .map_err(|_| unavailable("read staging free space"))?; + if free < MIN_STAGING_FREE_BYTES { + return Err(StoreError::InvalidData { + entity: "backup staging", + message: "staging disk space is below 1 GiB".to_owned(), + }); + } + Ok(()) + } + + /// 清理该备份在暂存区的全部文件；不存在视为成功。 + pub fn cleanup(&self, backup_id: &str) { + let _ = std::fs::remove_file(self.partial_path(backup_id)); + let _ = std::fs::remove_file(self.final_path(backup_id)); + } + + /// 校验一个已完成归档的暂存路径归属本暂存区。 + #[must_use] + pub fn owns(&self, path: &Path) -> bool { + path.parent() == Some(self.base_dir.as_path()) + } +} + +fn unavailable(operation: &'static str) -> StoreError { + crate::postgres_unavailable(operation) +} + +fn invalid(message: &str) -> StoreError { + StoreError::InvalidData { + entity: "backup staging", + message: message.to_owned(), + } +} diff --git a/backend/crates/gateway-store/src/lib.rs b/backend/crates/gateway-store/src/lib.rs index 3fea0ff94..139a114b5 100644 --- a/backend/crates/gateway-store/src/lib.rs +++ b/backend/crates/gateway-store/src/lib.rs @@ -13,6 +13,7 @@ use gateway_admin::model::settings::{ RotationStrategy as AdminRotationStrategy, RuntimeSettings as AdminRuntimeSettings, }; use gateway_admin::model::{MutationActor, MutationContext, Revision as AdminRevision}; +use gateway_admin::ports::backup::BackupStorePorts; use gateway_admin::ports::store::{ AdminStoreError, AdminStoreErrorKind, AdminStorePorts, AdminStoreResult, AuthStore, SettingsStore, @@ -33,6 +34,7 @@ const REDIS_URL_ENV: &str = "CPR_REDIS_URL"; const DATABASE_PASSWORD_ENV: &str = "CPR_DATABASE_PASSWORD"; const REDIS_PASSWORD_ENV: &str = "CPR_REDIS_PASSWORD"; +pub mod backup; pub mod postgres; pub mod redis; @@ -316,6 +318,7 @@ pub async fn initialize(mut config: StoreConfig) -> StoreResult { Arc::new(AdminSettingsStoreAdapter { control_plane: postgres::PgControlPlaneRepository::new(pool.clone()), }), + backup_ports(pool.clone(), &config)?, ); let execution_repository = Arc::new(postgres::PgExecutionStore::new(pool.clone())); @@ -401,6 +404,22 @@ pub async fn initialize(mut config: StoreConfig) -> StoreResult { }) } +/// 构造备份控制面的仓储、导出器与对象存储适配器。 +fn backup_ports(pool: sqlx::PgPool, config: &StoreConfig) -> StoreResult { + let staging = Arc::new(backup::staging::StagingArea::open( + std::path::PathBuf::from(".runtime/data/backup-staging"), + backup::staging::DEFAULT_MAX_ARCHIVE_BYTES, + )?); + let repository = Arc::new(postgres::PgBackupRepository::new(pool)); + let dump = Arc::new(backup::pg_dump::PgDumpAdapter::new( + staging, + config.database.url.clone(), + config.database.password.clone(), + )); + let object_store = Arc::new(backup::s3::S3ObjectStoreAdapter::new()); + Ok(BackupStorePorts::new(repository, dump, object_store)) +} + fn store_worker_contributions( execution: Arc, execution_writer: postgres::ExecutionObservationWriter, diff --git a/backend/crates/gateway-store/src/postgres/backup.rs b/backend/crates/gateway-store/src/postgres/backup.rs new file mode 100644 index 000000000..0eec37823 --- /dev/null +++ b/backend/crates/gateway-store/src/postgres/backup.rs @@ -0,0 +1,870 @@ +//! `backup_settings` 与 `backup_records` 的 PostgreSQL owner。 +//! +//! 实现 `gateway-admin::ports::backup::BackupRepository`。本模块只执行事务与 +//! 条件更新，不决定保留策略或状态机语义。 + +use async_trait::async_trait; +use chrono::{DateTime, Utc}; +use secrecy::{ExposeSecret as _, SecretString}; +use sqlx::{PgPool, Postgres, Row as _, Transaction}; + +use gateway_admin::model::backup::{ + BackupRecord, BackupRecordListQuery, BackupRecordPage, BackupRecordSeed, BackupSettings, + BackupStatus, BackupTriggerKind, UpdateBackupScheduleCommand, UpdateBackupStorageCommand, +}; +use gateway_admin::model::{MutationContext, Revision as AdminRevision}; +use gateway_admin::ports::backup::{BackupRepository, StatusTransitionUpdate}; +use gateway_admin::ports::store::{AdminStoreError, AdminStoreErrorKind, AdminStoreResult}; + +use crate::{ + StoreError, StoreResult, + postgres::{ + admin_security_audit::AdminAuditEvent, + runtime_settings::bump_config_revision_in_transaction, + }, +}; + +/// `backup_records` 的最大分页大小。 +const BACKUP_PAGE_LIMIT: u32 = 200; + +/// 备份任务/配置仓储。 +#[derive(Clone)] +pub struct PgBackupRepository { + pool: PgPool, +} + +impl PgBackupRepository { + /// 包装连接池。 + #[must_use] + pub const fn new(pool: PgPool) -> Self { + Self { pool } + } +} + +#[async_trait] +impl BackupRepository for PgBackupRepository { + async fn load_settings(&self) -> AdminStoreResult { + load_backup_settings(&self.pool).await + } + + async fn update_storage_settings( + &self, + command: UpdateBackupStorageCommand, + context: &MutationContext, + ) -> AdminStoreResult<(BackupSettings, AdminRevision)> { + let mut transaction = self + .pool + .begin() + .await + .map_err(|_| store_unavailable("begin backup storage update"))?; + let result = async { + let current = lock_settings_in_transaction(&mut transaction).await?; + ensure_storage_identity_stable(¤t, &command, &mut transaction).await?; + + let secret = command + .secret_access_key + .as_ref() + .map(|secret| secret.expose_secret().to_owned()); + sqlx::query( + "update backup_settings + set endpoint = $1, + region = $2, + bucket = $3, + access_key_id = $4, + secret_access_key = coalesce($5, secret_access_key), + prefix = $6, + force_path_style = $7, + storage_revision = storage_revision + 1, + last_verified_at = null, + updated_at = now() + where id = 1", + ) + .bind(&command.endpoint) + .bind(&command.region) + .bind(&command.bucket) + .bind(&command.access_key_id) + .bind(secret.as_deref()) + .bind(&command.prefix) + .bind(command.force_path_style) + .execute(&mut *transaction) + .await + .map_err(|_| store_unavailable("update backup storage"))?; + + let store_revision = bump_config_revision_in_transaction(&mut transaction) + .await + .map_err(map_admin_error)?; + let revision = admin_revision(store_revision)?; + let changed = storage_changed_fields(¤t, &command); + let audit = crate::mutation_audit( + context, + "backup.s3_config_updated", + "backup_settings", + "1", + changed, + ); + append_audit_in_transaction(&mut transaction, audit, Some(revision)) + .await + .map_err(map_admin_error)?; + + let settings = load_settings_in_transaction(&mut transaction).await?; + Ok((settings, revision)) + } + .await; + match result { + Ok(result) => { + transaction + .commit() + .await + .map_err(|_| store_unavailable("commit backup storage update"))?; + Ok(result) + } + Err(error) => { + transaction + .rollback() + .await + .map_err(|_| store_unavailable("rollback backup storage update"))?; + Err(error) + } + } + } + + async fn update_schedule_settings( + &self, + command: UpdateBackupScheduleCommand, + next_run_at: Option>, + context: &MutationContext, + ) -> AdminStoreResult { + let mut transaction = self + .pool + .begin() + .await + .map_err(|_| store_unavailable("begin backup schedule update"))?; + let result = async { + sqlx::query( + "update backup_settings + set schedule_enabled = $1, + cron_expression = $2, + schedule_timezone = $3, + retention_days = $4, + retention_count = $5, + next_run_at = $6, + updated_at = now() + where id = 1", + ) + .bind(command.schedule_enabled) + .bind(&command.cron_expression) + .bind(&command.schedule_timezone) + .bind(i64::from(command.retention_days)) + .bind(i64::from(command.retention_count)) + .bind(next_run_at) + .execute(&mut *transaction) + .await + .map_err(|_| store_unavailable("update backup schedule"))?; + + let audit = crate::mutation_audit( + context, + "backup.schedule_updated", + "backup_settings", + "1", + vec![ + "schedule_enabled".to_owned(), + "cron_expression".to_owned(), + "schedule_timezone".to_owned(), + "retention_days".to_owned(), + "retention_count".to_owned(), + ], + ); + append_audit_in_transaction(&mut transaction, audit, None) + .await + .map_err(map_admin_error)?; + load_settings_in_transaction(&mut transaction).await + } + .await; + match result { + Ok(settings) => { + transaction + .commit() + .await + .map_err(|_| store_unavailable("commit backup schedule update"))?; + Ok(settings) + } + Err(error) => { + transaction + .rollback() + .await + .map_err(|_| store_unavailable("rollback backup schedule update"))?; + Err(error) + } + } + } + + async fn record_verification( + &self, + storage_revision: u64, + at: DateTime, + ) -> AdminStoreResult { + let result = sqlx::query( + "update backup_settings + set last_verified_at = $2, updated_at = now() + where id = 1 and storage_revision = $1", + ) + .bind(i64::try_from(storage_revision).map_err(|_| store_invalid("storage revision"))?) + .bind(at) + .execute(&self.pool) + .await + .map_err(|_| store_unavailable("record backup storage verification"))?; + Ok(result.rows_affected() == 1) + } + + async fn insert_backup_record(&self, seed: BackupRecordSeed) -> AdminStoreResult { + let mut transaction = self + .pool + .begin() + .await + .map_err(|_| store_unavailable("begin backup record insert"))?; + let result = async { + let _ = lock_settings_in_transaction(&mut transaction).await?; + insert_queued_in_transaction(&mut transaction, &seed).await + } + .await; + match result { + Ok(record) => { + transaction + .commit() + .await + .map_err(|_| store_unavailable("commit backup record insert"))?; + Ok(record) + } + Err(error) => { + transaction + .rollback() + .await + .map_err(|_| store_unavailable("rollback backup record insert"))?; + Err(error) + } + } + } + + async fn insert_scheduled_record(&self, seed: BackupRecordSeed) -> AdminStoreResult { + let mut transaction = self + .pool + .begin() + .await + .map_err(|_| store_unavailable("begin scheduled insert"))?; + match insert_queued_in_transaction(&mut transaction, &seed).await { + Ok(_) => { + transaction + .commit() + .await + .map_err(|_| store_unavailable("commit scheduled insert"))?; + Ok(true) + } + Err(error) if is_active_or_scheduled_conflict(&error) => { + transaction + .rollback() + .await + .map_err(|_| store_unavailable("rollback scheduled insert"))?; + Ok(false) + } + Err(error) => { + transaction + .rollback() + .await + .map_err(|_| store_unavailable("rollback scheduled insert"))?; + Err(error) + } + } + } + + async fn list_backup_records( + &self, + query: BackupRecordListQuery, + ) -> AdminStoreResult { + let limit = u32::from(query.page_size.get()).min(BACKUP_PAGE_LIMIT); + let offset = u64::from(query.page.saturating_sub(1)) * u64::from(limit); + + let status = query.status.map(BackupStatus::as_str); + let trigger = query.trigger.map(BackupTriggerKind::as_str); + + let total = sqlx::query_scalar::<_, i64>( + "select count(*) from backup_records + where ($1::text is null or status = $1) + and ($2::text is null or trigger_kind = $2)", + ) + .bind(status) + .bind(trigger) + .fetch_one(&self.pool) + .await + .map_err(|_| store_unavailable("count backup records"))?; + + let rows = sqlx::query( + "select id, trigger_kind, status, scheduled_at, object_key, size_bytes, sha256, + attempt_count, error_code, error_message, started_at, completed_at, expires_at, + created_at, updated_at + from backup_records + where ($1::text is null or status = $1) + and ($2::text is null or trigger_kind = $2) + order by created_at desc, id desc + limit $3 offset $4", + ) + .bind(status) + .bind(trigger) + .bind(i64::from(limit)) + .bind(i64::try_from(offset).map_err(|_| store_invalid("page offset"))?) + .fetch_all(&self.pool) + .await + .map_err(|_| store_unavailable("list backup records"))?; + + let items = rows + .iter() + .map(backup_record_from_row) + .collect::, _>>()?; + Ok(BackupRecordPage { + items, + total: u64::try_from(total).map_err(|_| store_invalid("record count"))?, + page: query.page, + page_size: query.page_size, + }) + } + + async fn load_backup_record(&self, id: &str) -> AdminStoreResult> { + sqlx::query( + "select id, trigger_kind, status, scheduled_at, object_key, size_bytes, sha256, + attempt_count, error_code, error_message, started_at, completed_at, expires_at, + created_at, updated_at + from backup_records where id = $1", + ) + .bind(id) + .fetch_optional(&self.pool) + .await + .map_err(|_| store_unavailable("load backup record"))? + .map(|row| backup_record_from_row(&row)) + .transpose() + } + + async fn list_intermediate_records(&self) -> AdminStoreResult> { + let rows = sqlx::query( + "select id, trigger_kind, status, scheduled_at, object_key, size_bytes, sha256, + attempt_count, error_code, error_message, started_at, completed_at, expires_at, + created_at, updated_at + from backup_records + where status in ('dumping', 'uploading') + order by created_at, id", + ) + .fetch_all(&self.pool) + .await + .map_err(|_| store_unavailable("list intermediate backup records"))?; + rows.iter() + .map(backup_record_from_row) + .collect::, _>>() + } + + async fn list_pending_deletions(&self, limit: u32) -> AdminStoreResult> { + let limit = limit.clamp(1, 100); + let rows = sqlx::query( + "select id, trigger_kind, status, scheduled_at, object_key, size_bytes, sha256, + attempt_count, error_code, error_message, started_at, completed_at, expires_at, + created_at, updated_at + from backup_records + where status = 'deleting' + order by created_at, id + limit $1", + ) + .bind(i64::from(limit)) + .fetch_all(&self.pool) + .await + .map_err(|_| store_unavailable("list pending deletions"))?; + rows.iter() + .map(backup_record_from_row) + .collect::, _>>() + } + + async fn list_expired_records(&self, limit: u32) -> AdminStoreResult> { + let limit = limit.clamp(1, 1000); + let now = Utc::now(); + let rows = sqlx::query( + "select id, trigger_kind, status, scheduled_at, object_key, size_bytes, sha256, + attempt_count, error_code, error_message, started_at, completed_at, expires_at, + created_at, updated_at + from backup_records + where expires_at is not null + and expires_at <= $1 + and status in ('completed', 'failed') + order by created_at, id + limit $2", + ) + .bind(now) + .bind(i64::from(limit)) + .fetch_all(&self.pool) + .await + .map_err(|_| store_unavailable("list expired backups"))?; + rows.iter() + .map(backup_record_from_row) + .collect::, _>>() + } + + async fn claim_next_queued( + &self, + now: DateTime, + ) -> AdminStoreResult> { + sqlx::query( + "with claimed as ( + select backup_records.id from backup_records + where backup_records.status = 'queued' + order by backup_records.created_at, backup_records.id + limit 1 + for update skip locked + ) + update backup_records br + set status = 'dumping', + started_at = $1, + completed_at = null, + attempt_count = br.attempt_count + 1, + updated_at = $1 + from claimed + where br.id = claimed.id + returning br.id, br.trigger_kind, br.status, br.scheduled_at, br.object_key, + br.size_bytes, br.sha256, br.attempt_count, br.error_code, + br.error_message, br.started_at, br.completed_at, br.expires_at, + br.created_at, br.updated_at", + ) + .bind(now) + .fetch_optional(&self.pool) + .await + .map_err(|_| store_unavailable("claim queued backup task"))? + .map(|row| backup_record_from_row(&row)) + .transpose() + } + + async fn transition_status( + &self, + id: &str, + from: BackupStatus, + to: BackupStatus, + update: StatusTransitionUpdate, + now: DateTime, + ) -> AdminStoreResult> { + let size_bytes = update + .size_bytes + .map(i64::try_from) + .transpose() + .map_err(|_| store_invalid("size bytes"))?; + sqlx::query( + "update backup_records + set status = $1, + size_bytes = coalesce($2, size_bytes), + sha256 = coalesce($3, sha256), + error_code = coalesce($4, error_code), + error_message = coalesce($5, error_message), + completed_at = coalesce( + $6, + case when $1 in ('completed', 'failed') then $7 else completed_at end + ), + updated_at = $7 + where id = $8 and status = $9 + returning id, trigger_kind, status, scheduled_at, object_key, size_bytes, sha256, + attempt_count, error_code, error_message, started_at, completed_at, expires_at, + created_at, updated_at", + ) + .bind(to.as_str()) + .bind(size_bytes) + .bind(update.sha256.as_deref()) + .bind(update.error_code.as_deref()) + .bind(update.error_message.as_deref()) + .bind(update.completed_at) + .bind(now) + .bind(id) + .bind(from.as_str()) + .fetch_optional(&self.pool) + .await + .map_err(|_| store_unavailable("transition backup task"))? + .map(|row| backup_record_from_row(&row)) + .transpose() + } + + async fn transition_to_deleting( + &self, + id: &str, + now: DateTime, + ) -> AdminStoreResult> { + sqlx::query( + "update backup_records + set status = 'deleting', updated_at = $1 + where id = $2 and status in ('completed', 'failed') + returning id, trigger_kind, status, scheduled_at, object_key, size_bytes, sha256, + attempt_count, error_code, error_message, started_at, completed_at, expires_at, + created_at, updated_at", + ) + .bind(now) + .bind(id) + .fetch_optional(&self.pool) + .await + .map_err(|_| store_unavailable("transition backup to deleting"))? + .map(|row| backup_record_from_row(&row)) + .transpose() + } + + async fn delete_record(&self, id: &str) -> AdminStoreResult<()> { + sqlx::query("delete from backup_records where id = $1") + .bind(id) + .execute(&self.pool) + .await + .map(|_| ()) + .map_err(|_| store_unavailable("delete backup record")) + } + + async fn advance_schedule_cursor( + &self, + next_run_at: DateTime, + expected_cron: &str, + expected_timezone: &str, + ) -> AdminStoreResult { + let result = sqlx::query( + "update backup_settings + set next_run_at = $1 + where id = 1 + and schedule_enabled + and cron_expression = $2 + and schedule_timezone = $3", + ) + .bind(next_run_at) + .bind(expected_cron) + .bind(expected_timezone) + .execute(&self.pool) + .await + .map_err(|_| store_unavailable("advance backup schedule cursor"))?; + Ok(result.rows_affected() == 1) + } + + async fn list_scheduled_completed_desc( + &self, + limit: u32, + ) -> AdminStoreResult> { + let limit = limit.clamp(1, 10000); + let rows = sqlx::query( + "select id, trigger_kind, status, scheduled_at, object_key, size_bytes, sha256, + attempt_count, error_code, error_message, started_at, completed_at, expires_at, + created_at, updated_at + from backup_records + where trigger_kind = 'scheduled' and status = 'completed' + order by completed_at desc, id desc + limit $1", + ) + .bind(i64::from(limit)) + .fetch_all(&self.pool) + .await + .map_err(|_| store_unavailable("list scheduled completed backups"))?; + rows.iter() + .map(backup_record_from_row) + .collect::, _>>() + } +} + +/// Store 内部 Revision → Admin Revision 转换。 +fn admin_revision(revision: crate::Revision) -> AdminStoreResult { + AdminRevision::new(revision.get()).map_err(|_| { + AdminStoreError::new(AdminStoreErrorKind::Invalid, "revision", "zero revision") + }) +} + +/// 锁定并读取单例配置行。 +async fn lock_settings_in_transaction( + transaction: &mut Transaction<'_, Postgres>, +) -> AdminStoreResult { + sqlx::query( + "select storage_revision, endpoint, region, bucket, access_key_id, secret_access_key, + prefix, force_path_style, schedule_enabled, cron_expression, schedule_timezone, + retention_days, retention_count, next_run_at, last_verified_at, updated_at + from backup_settings where id = 1 for update", + ) + .fetch_optional(&mut **transaction) + .await + .map_err(|_| store_unavailable("lock backup settings"))? + .map(|row| backup_settings_from_row(&row)) + .transpose()? + .ok_or_else(|| store_not_found("backup settings")) +} + +async fn load_settings_in_transaction( + transaction: &mut Transaction<'_, Postgres>, +) -> AdminStoreResult { + sqlx::query( + "select storage_revision, endpoint, region, bucket, access_key_id, secret_access_key, + prefix, force_path_style, schedule_enabled, cron_expression, schedule_timezone, + retention_days, retention_count, next_run_at, last_verified_at, updated_at + from backup_settings where id = 1", + ) + .fetch_optional(&mut **transaction) + .await + .map_err(|_| store_unavailable("load backup settings in transaction"))? + .map(|row| backup_settings_from_row(&row)) + .transpose()? + .ok_or_else(|| store_not_found("backup settings")) +} + +async fn load_backup_settings(pool: &PgPool) -> AdminStoreResult { + sqlx::query( + "select storage_revision, endpoint, region, bucket, access_key_id, secret_access_key, + prefix, force_path_style, schedule_enabled, cron_expression, schedule_timezone, + retention_days, retention_count, next_run_at, last_verified_at, updated_at + from backup_settings where id = 1", + ) + .fetch_optional(pool) + .await + .map_err(|_| store_unavailable("load backup settings"))? + .map(|row| backup_settings_from_row(&row)) + .transpose()? + .ok_or_else(|| store_not_found("backup settings")) +} + +/// 插入 queued 记录；返回记录或唯一约束冲突。 +async fn insert_queued_in_transaction( + transaction: &mut Transaction<'_, Postgres>, + seed: &BackupRecordSeed, +) -> AdminStoreResult { + let row = sqlx::query( + "insert into backup_records ( + id, trigger_kind, status, scheduled_at, object_key, expires_at, created_at, updated_at + ) values ($1, $2, 'queued', $3, $4, $5, now(), now()) + returning id, trigger_kind, status, scheduled_at, object_key, size_bytes, sha256, + attempt_count, error_code, error_message, started_at, completed_at, expires_at, + created_at, updated_at", + ) + .bind(&seed.id) + .bind(seed.trigger_kind.as_str()) + .bind(seed.scheduled_at) + .bind(&seed.object_key) + .bind(seed.expires_at) + .fetch_one(&mut **transaction) + .await + .map_err(map_insert_error)?; + backup_record_from_row(&row) +} + +/// 把唯一约束冲突映射为活跃任务/计划冲突。 +fn map_insert_error(error: sqlx::Error) -> AdminStoreError { + match &error { + sqlx::Error::Database(database_error) + if database_error.constraint() == Some("backup_records_active_uq") => + { + AdminStoreError::new( + AdminStoreErrorKind::Conflict, + "backup record", + "an active backup task already exists", + ) + } + sqlx::Error::Database(database_error) + if database_error.constraint() == Some("backup_records_scheduled_uq") => + { + AdminStoreError::new( + AdminStoreErrorKind::Conflict, + "backup record", + "a scheduled backup already exists at this time", + ) + } + _ => AdminStoreError::new( + AdminStoreErrorKind::Unavailable, + "backup record", + "backup record insert failed", + ), + } +} + +fn is_active_or_scheduled_conflict(error: &AdminStoreError) -> bool { + error.kind() == AdminStoreErrorKind::Conflict +} + +/// 已有记录时锁定存储身份：endpoint/region/bucket/path-style 不允许变化。 +async fn ensure_storage_identity_stable( + current: &BackupSettings, + command: &UpdateBackupStorageCommand, + transaction: &mut Transaction<'_, Postgres>, +) -> AdminStoreResult<()> { + let identity_changed = current.endpoint.as_deref() != Some(command.endpoint.as_str()) + || current.region.as_deref() != Some(command.region.as_str()) + || current.bucket.as_deref() != Some(command.bucket.as_str()) + || current.force_path_style != command.force_path_style; + if !identity_changed { + return Ok(()); + } + let records_exist = + sqlx::query_scalar::<_, bool>("select exists(select 1 from backup_records)") + .fetch_one(&mut **transaction) + .await + .map_err(|_| store_unavailable("check backup records existence"))?; + if records_exist { + return Err(AdminStoreError::new( + AdminStoreErrorKind::Conflict, + "backup storage", + "storage identity cannot change while backup records exist", + )); + } + Ok(()) +} + +fn storage_changed_fields( + current: &BackupSettings, + command: &UpdateBackupStorageCommand, +) -> Vec { + let mut fields = Vec::new(); + if current.endpoint.as_deref() != Some(command.endpoint.as_str()) { + fields.push("endpoint".to_owned()); + } + if current.region.as_deref() != Some(command.region.as_str()) { + fields.push("region".to_owned()); + } + if current.bucket.as_deref() != Some(command.bucket.as_str()) { + fields.push("bucket".to_owned()); + } + if current.access_key_id.as_deref() != Some(command.access_key_id.as_str()) { + fields.push("access_key_id".to_owned()); + } + if command.secret_access_key.is_some() { + fields.push("secret_access_key".to_owned()); + } + if current.prefix.as_deref() != Some(command.prefix.as_str()) { + fields.push("prefix".to_owned()); + } + if current.force_path_style != command.force_path_style { + fields.push("force_path_style".to_owned()); + } + fields +} + +/// 在同事务追加审计事件；`revision` 为空时不写 config_revision。 +async fn append_audit_in_transaction( + transaction: &mut Transaction<'_, Postgres>, + mut event: AdminAuditEvent, + revision: Option, +) -> StoreResult<()> { + event.config_revision = revision + .map(|revision| { + i64::try_from(revision.get()).map_err(|_| StoreError::InvalidData { + entity: "backup", + message: "revision overflow".to_owned(), + }) + }) + .transpose()?; + event.validate()?; + sqlx::query( + "insert into admin_audit_events ( + id, actor_kind, actor_admin_user_id, actor_ref, admin_request_id, + action, entity_kind, entity_ref, config_revision, changed_fields, created_at + ) values ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11)", + ) + .bind(event.id) + .bind(event.actor_kind.as_str()) + .bind(event.actor_admin_user_id) + .bind(event.actor_ref) + .bind(event.admin_request_id) + .bind(event.action) + .bind(event.entity_kind) + .bind(event.entity_ref) + .bind(event.config_revision) + .bind(event.changed_fields) + .bind(event.created_at) + .execute(&mut **transaction) + .await + .map_err(|_| crate::postgres_unavailable("append backup audit event"))?; + Ok(()) +} + +fn backup_record_from_row(row: &sqlx::postgres::PgRow) -> AdminStoreResult { + let size_bytes = decode(row.try_get::, _>("size_bytes"))? + .map(u64::try_from) + .transpose() + .map_err(|_| store_invalid("size_bytes"))?; + Ok(BackupRecord { + id: decode(row.try_get("id"))?, + trigger_kind: BackupTriggerKind::parse(decode(row.try_get::<&str, _>("trigger_kind"))?) + .ok_or_else(|| store_invalid("trigger_kind"))?, + status: BackupStatus::parse(decode(row.try_get::<&str, _>("status"))?) + .ok_or_else(|| store_invalid("status"))?, + scheduled_at: decode(row.try_get("scheduled_at"))?, + object_key: decode(row.try_get("object_key"))?, + size_bytes, + sha256: decode(row.try_get("sha256"))?, + attempt_count: u32::try_from(decode(row.try_get::("attempt_count"))?) + .map_err(|_| store_invalid("attempt_count"))?, + error_code: decode(row.try_get("error_code"))?, + error_message: decode(row.try_get("error_message"))?, + started_at: decode(row.try_get("started_at"))?, + completed_at: decode(row.try_get("completed_at"))?, + expires_at: decode(row.try_get("expires_at"))?, + created_at: decode(row.try_get("created_at"))?, + updated_at: decode(row.try_get("updated_at"))?, + }) +} + +fn backup_settings_from_row(row: &sqlx::postgres::PgRow) -> AdminStoreResult { + let storage_revision = u64::try_from(decode(row.try_get::("storage_revision"))?) + .map_err(|_| store_invalid("storage_revision"))?; + let secret = decode(row.try_get::, _>("secret_access_key"))?; + Ok(BackupSettings { + storage_revision, + endpoint: decode(row.try_get("endpoint"))?, + region: decode(row.try_get("region"))?, + bucket: decode(row.try_get("bucket"))?, + access_key_id: decode(row.try_get("access_key_id"))?, + secret_access_key: secret.map(SecretString::from), + prefix: decode(row.try_get("prefix"))?, + force_path_style: decode(row.try_get("force_path_style"))?, + schedule_enabled: decode(row.try_get("schedule_enabled"))?, + cron_expression: decode(row.try_get("cron_expression"))?, + schedule_timezone: decode(row.try_get("schedule_timezone"))?, + retention_days: u32::try_from(decode(row.try_get::("retention_days"))?) + .map_err(|_| store_invalid("retention_days"))?, + retention_count: u32::try_from(decode(row.try_get::("retention_count"))?) + .map_err(|_| store_invalid("retention_count"))?, + next_run_at: decode(row.try_get("next_run_at"))?, + last_verified_at: decode(row.try_get("last_verified_at"))?, + updated_at: decode(row.try_get("updated_at"))?, + }) +} + +/// sqlx 解码错误统一映射为备份 InvalidData。 +fn decode(result: Result) -> AdminStoreResult { + result.map_err(|_| { + AdminStoreError::new( + AdminStoreErrorKind::Invalid, + "backup", + "backup record decode failed", + ) + }) +} + +/// StoreError → AdminStoreError 的显式映射。 +fn map_admin_error(error: StoreError) -> AdminStoreError { + match error { + StoreError::Unavailable { .. } => AdminStoreError::new( + AdminStoreErrorKind::Unavailable, + "backup", + "backup store unavailable", + ), + StoreError::NotFound { entity, .. } => { + AdminStoreError::new(AdminStoreErrorKind::NotFound, entity, "not found") + } + StoreError::Conflict { entity, .. } => { + AdminStoreError::new(AdminStoreErrorKind::Conflict, entity, "conflict") + } + StoreError::InvalidData { entity, message } => { + AdminStoreError::new(AdminStoreErrorKind::Invalid, entity, message) + } + } +} + +fn store_unavailable(operation: &'static str) -> AdminStoreError { + AdminStoreError::new( + AdminStoreErrorKind::Unavailable, + "backup", + format!("{operation}: store unavailable"), + ) +} + +fn store_not_found(entity: &'static str) -> AdminStoreError { + AdminStoreError::new(AdminStoreErrorKind::NotFound, entity, "not found") +} + +fn store_invalid(message: &'static str) -> AdminStoreError { + AdminStoreError::new(AdminStoreErrorKind::Invalid, "backup", message) +} diff --git a/backend/crates/gateway-store/src/postgres/mod.rs b/backend/crates/gateway-store/src/postgres/mod.rs index 4ce275b0f..5506a6e47 100644 --- a/backend/crates/gateway-store/src/postgres/mod.rs +++ b/backend/crates/gateway-store/src/postgres/mod.rs @@ -9,6 +9,7 @@ use crate::{ mod admin_security_audit; mod admission_recovery; +mod backup; mod client_keys; mod execution; mod execution_buffer; @@ -21,6 +22,7 @@ mod snapshot; pub use admin_security_audit::*; pub use admission_recovery::*; +pub use backup::*; pub use client_keys::*; pub use execution::*; pub use execution_buffer::*; diff --git a/backend/crates/gateway-store/tests/postgres/backup.rs b/backend/crates/gateway-store/tests/postgres/backup.rs new file mode 100644 index 000000000..9ae65bd96 --- /dev/null +++ b/backend/crates/gateway-store/tests/postgres/backup.rs @@ -0,0 +1,308 @@ +//! `PgBackupRepository` 的真实 PostgreSQL 集成测试。 + +use chrono::Utc; +use secrecy::SecretString; + +use gateway_admin::model::backup::{ + BackupRecordSeed, BackupStatus, BackupTriggerKind, UpdateBackupScheduleCommand, + UpdateBackupStorageCommand, +}; +use gateway_admin::model::{MutationActor, MutationContext}; +use gateway_admin::ports::backup::{BackupRepository, StatusTransitionUpdate}; +use gateway_admin::ports::store::AdminStoreErrorKind; +use gateway_store::postgres::PgBackupRepository; + +use super::TestDatabase; + +fn seed( + hex_suffix: &str, + trigger: BackupTriggerKind, + scheduled_at: Option>, +) -> BackupRecordSeed { + let id = format!("backup_{hex_suffix}"); + BackupRecordSeed { + id: id.clone(), + trigger_kind: trigger, + scheduled_at, + object_key: format!("codex/2026/08/01/{id}.dump"), + expires_at: None, + } +} + +fn context() -> MutationContext { + MutationContext { + actor: MutationActor::System, + request_id: "store-test".to_owned(), + } +} + +fn storage_command(endpoint: &str) -> UpdateBackupStorageCommand { + UpdateBackupStorageCommand { + endpoint: endpoint.to_owned(), + region: "auto".to_owned(), + bucket: "backup-bucket".to_owned(), + access_key_id: "ak".to_owned(), + secret_access_key: Some(SecretString::from("sk-value")), + prefix: "codex".to_owned(), + force_path_style: false, + } +} + +#[tokio::test] +async fn backup_lifecycle_transitions_through_deleting_and_hard_delete() { + let Some(db) = TestDatabase::create("backup_lifecycle").await else { + return; + }; + let repository = PgBackupRepository::new(db.pool.clone()); + + let record = repository + .insert_backup_record(seed(&"a".repeat(32), BackupTriggerKind::Manual, None)) + .await + .expect("insert queued"); + assert_eq!(record.status, BackupStatus::Queued); + assert_eq!(record.attempt_count, 0); + + let claimed = repository + .claim_next_queued(Utc::now()) + .await + .expect("claim") + .expect("claimed one task"); + assert_eq!(claimed.status, BackupStatus::Dumping); + assert_eq!(claimed.attempt_count, 1); + assert!(claimed.started_at.is_some()); + + let update = StatusTransitionUpdate { + size_bytes: Some(10), + sha256: Some("a".repeat(64)), + ..Default::default() + }; + let uploading = repository + .transition_status( + &record.id, + BackupStatus::Dumping, + BackupStatus::Uploading, + update, + Utc::now(), + ) + .await + .expect("dump complete") + .expect("transition matched"); + assert_eq!(uploading.status, BackupStatus::Uploading); + assert_eq!(uploading.size_bytes, Some(10)); + + let completed = repository + .transition_status( + &record.id, + BackupStatus::Uploading, + BackupStatus::Completed, + StatusTransitionUpdate::default(), + Utc::now(), + ) + .await + .expect("upload verified") + .expect("transition matched"); + assert_eq!(completed.status, BackupStatus::Completed); + assert!(completed.completed_at.is_some()); + + let deleting = repository + .transition_to_deleting(&record.id, Utc::now()) + .await + .expect("to deleting") + .expect("transition matched"); + assert_eq!(deleting.status, BackupStatus::Deleting); + + repository + .delete_record(&record.id) + .await + .expect("hard delete"); + assert!( + repository + .load_backup_record(&record.id) + .await + .expect("load after delete") + .is_none() + ); + + db.close().await; +} + +#[tokio::test] +async fn active_task_unique_index_blocks_second_queued() { + let Some(db) = TestDatabase::create("backup_active_conflict").await else { + return; + }; + let repository = PgBackupRepository::new(db.pool.clone()); + + repository + .insert_backup_record(seed(&"a".repeat(32), BackupTriggerKind::Manual, None)) + .await + .expect("first queued"); + let error = repository + .insert_backup_record(seed(&"b".repeat(32), BackupTriggerKind::Manual, None)) + .await + .expect_err("second queued must conflict"); + assert_eq!(error.kind(), AdminStoreErrorKind::Conflict); + + // 第一个任务完成释放名额后，第二个可以插入。 + let claimed = repository + .claim_next_queued(Utc::now()) + .await + .expect("claim") + .expect("claimed"); + repository + .transition_status( + &claimed.id, + BackupStatus::Dumping, + BackupStatus::Failed, + StatusTransitionUpdate { + error_code: Some("backup.pg_dump_failed".to_owned()), + error_message: Some("boom".to_owned()), + completed_at: Some(Utc::now()), + ..Default::default() + }, + Utc::now(), + ) + .await + .expect("fail task"); + repository + .insert_backup_record(seed(&"c".repeat(32), BackupTriggerKind::Manual, None)) + .await + .expect("third queued after terminal"); + + db.close().await; +} + +#[tokio::test] +async fn scheduled_insert_dedupes_by_logical_time() { + let Some(db) = TestDatabase::create("backup_scheduled_dedup").await else { + return; + }; + let repository = PgBackupRepository::new(db.pool.clone()); + + let at = Utc::now(); + let first = repository + .insert_scheduled_record(seed( + &"a".repeat(32), + BackupTriggerKind::Scheduled, + Some(at), + )) + .await + .expect("first scheduled"); + assert!(first); + let second = repository + .insert_scheduled_record(seed( + &"b".repeat(32), + BackupTriggerKind::Scheduled, + Some(at), + )) + .await + .expect("dedup conflict returns false"); + assert!(!second); + + db.close().await; +} + +#[tokio::test] +async fn storage_identity_is_locked_once_records_exist() { + let Some(db) = TestDatabase::create("backup_storage_identity").await else { + return; + }; + let repository = PgBackupRepository::new(db.pool.clone()); + + repository + .update_storage_settings(storage_command("https://one.example.com"), &context()) + .await + .expect("configure storage with no records"); + + let changed_endpoint = { + let mut command = storage_command("https://two.example.com"); + command.endpoint = "https://two.example.com".to_owned(); + command + }; + // 尚无记录时允许切换 endpoint。 + repository + .update_storage_settings(changed_endpoint, &context()) + .await + .expect("endpoint change allowed with no records"); + + repository + .insert_backup_record(seed(&"a".repeat(32), BackupTriggerKind::Manual, None)) + .await + .expect("insert queued"); + + let locked = repository + .update_storage_settings(storage_command("https://three.example.com"), &context()) + .await + .expect_err("endpoint change must be rejected with records"); + assert_eq!(locked.kind(), AdminStoreErrorKind::Conflict); + + // 只轮换凭据与修改 prefix 仍然允许。 + let rotation = { + let mut command = storage_command("https://two.example.com"); + command.access_key_id = "new-ak".to_owned(); + command.secret_access_key = Some(SecretString::from("new-sk")); + command.prefix = "codex/production".to_owned(); + command + }; + repository + .update_storage_settings(rotation, &context()) + .await + .expect("credential rotation and prefix change allowed"); + + db.close().await; +} + +#[tokio::test] +async fn schedule_update_persists_cursor_and_clears_on_disable() { + let Some(db) = TestDatabase::create("backup_schedule_update").await else { + return; + }; + let repository = PgBackupRepository::new(db.pool.clone()); + + // 启用计划前必须先配置存储并通过连接测试（DB 约束要求）。 + repository + .update_storage_settings(storage_command("https://one.example.com"), &context()) + .await + .expect("configure storage"); + let settings = repository.load_settings().await.expect("load settings"); + repository + .record_verification(settings.storage_revision, Utc::now()) + .await + .expect("verify storage"); + + let enabled = repository + .update_schedule_settings( + UpdateBackupScheduleCommand { + schedule_enabled: true, + cron_expression: "0 2 * * *".to_owned(), + schedule_timezone: "Asia/Shanghai".to_owned(), + retention_days: 7, + retention_count: 5, + }, + Some(Utc::now() + chrono::Duration::hours(1)), + &context(), + ) + .await + .expect("enable schedule"); + assert!(enabled.schedule_enabled); + assert!(enabled.next_run_at.is_some()); + + let disabled = repository + .update_schedule_settings( + UpdateBackupScheduleCommand { + schedule_enabled: false, + cron_expression: "0 2 * * *".to_owned(), + schedule_timezone: "Asia/Shanghai".to_owned(), + retention_days: 7, + retention_count: 5, + }, + None, + &context(), + ) + .await + .expect("disable schedule"); + assert!(!disabled.schedule_enabled); + assert!(disabled.next_run_at.is_none()); + + db.close().await; +} diff --git a/backend/crates/gateway-store/tests/postgres/mod.rs b/backend/crates/gateway-store/tests/postgres/mod.rs index 44af622b5..68e45a490 100644 --- a/backend/crates/gateway-store/tests/postgres/mod.rs +++ b/backend/crates/gateway-store/tests/postgres/mod.rs @@ -9,6 +9,7 @@ use uuid::Uuid; mod admin_security_audit; mod admission_recovery; +mod backup; mod client_keys; mod execution; mod observability; @@ -65,6 +66,10 @@ impl TestDatabase { .execute(&mut *migration) .await .expect("apply snapshot migration"); + sqlx::raw_sql(include_str!("../../../../migrations/0003_s3_backup.sql")) + .execute(&mut *migration) + .await + .expect("apply backup migration"); migration.commit().await.expect("commit terminal migration"); Some(Self { admin, @@ -160,7 +165,7 @@ async fn connect_and_migrate_should_apply_all_migrations_once_and_reopen_cleanly .expect("drop migration test database"); admin.close().await; - assert_eq!((first_table_count, migration_count), (8, 2)); + assert_eq!((first_table_count, migration_count), (10, 3)); assert_eq!(response_id_types, ["bytea", "bytea"]); assert!(!raw_response_id_index_exists); } @@ -170,6 +175,7 @@ fn migrations_should_leave_transaction_ownership_to_sqlx() { let transaction_statements = [ include_str!("../../../../migrations/0001_initial.sql"), include_str!("../../../../migrations/0002_snapshot_provider_account_identity.sql"), + include_str!("../../../../migrations/0003_s3_backup.sql"), ] .into_iter() .flat_map(str::lines) diff --git a/backend/migrations/.frozen-sha256 b/backend/migrations/.frozen-sha256 index 24a98c708..0ad009135 100644 --- a/backend/migrations/.frozen-sha256 +++ b/backend/migrations/.frozen-sha256 @@ -1,2 +1,3 @@ a4681a5c040e80c5234e879293fc34972080e5251dfbc7c40e1374f8e4170a4c 0001_initial.sql 7dc3b626e7a958281f82c0e646b8284f8b373e0c7d3dbdaf750811ce0c4a7f83 0002_snapshot_provider_account_identity.sql +ada40b11728ba67cfe44d8492bc77a4754b36640e575b25daa5dfe90e92e43aa 0003_s3_backup.sql diff --git a/backend/migrations/0003_s3_backup.sql b/backend/migrations/0003_s3_backup.sql new file mode 100644 index 000000000..b297d53ba --- /dev/null +++ b/backend/migrations/0003_s3_backup.sql @@ -0,0 +1,189 @@ +-- S3 兼容对象存储（含 Cloudflare R2）与 PostgreSQL 备份。 +-- 按 docs/s3-backup-design-audit.md 最终状态实现。 +-- +-- backup_settings 是单例配置行（id = 1）；backup_records 记录每次备份任务。 +-- 当前部署边界是单副本：任务执行由单个可取消 Daemon 承担，记录表不需要 +-- fencing token、heartbeat 或墓碑状态。skipped 只是日志/指标，删除成功后记录 +-- 硬删除，操作历史进入 admin_audit_events。 +-- 手动过期 TTL：每份备份独立的 expires_at，创建时确定，到期自动清理 +-- （审计 A-06 曾移除该字段，按参考实现 sub2api 的语义加回）。 + +create table backup_settings ( + id bigint primary key, + storage_revision bigint not null default 1, + endpoint text, + region text, + bucket text, + access_key_id text, + secret_access_key text, + prefix text, + force_path_style boolean not null default false, + schedule_enabled boolean not null default false, + cron_expression text, + schedule_timezone text default 'Asia/Shanghai', + retention_days bigint not null default 0, + retention_count bigint not null default 0, + next_run_at timestamptz, + last_verified_at timestamptz, + updated_at timestamptz not null, + constraint backup_settings_singleton_ck check (id = 1), + constraint backup_settings_revision_ck check (storage_revision > 0), + constraint backup_settings_endpoint_ck check ( + endpoint is null + or ( + endpoint ~ '^https?://' + and endpoint !~ '[[:space:][:cntrl:]]' + and endpoint !~ '://[^/]*@' + ) + ), + constraint backup_settings_lengths_ck check ( + (bucket is null or (octet_length(bucket) between 1 and 255 and bucket !~ '[[:cntrl:]]')) + and (region is null or (octet_length(region) between 1 and 64 and region !~ '[[:cntrl:]]')) + and (prefix is null or (octet_length(prefix) <= 1024 and prefix !~ '[[:cntrl:]]')) + and ( + access_key_id is null + or (octet_length(access_key_id) between 1 and 255 and access_key_id !~ '[[:cntrl:]]') + ) + and ( + secret_access_key is null + or ( + octet_length(secret_access_key) between 1 and 512 + and secret_access_key !~ '[[:cntrl:]]' + ) + ) + ), + constraint backup_settings_schedule_ck check ( + (cron_expression is null or cron_expression ~ '^[^[:space:]]+([[:space:]]+[^[:space:]]+){4}$') + and ( + schedule_timezone is null + or (octet_length(schedule_timezone) between 1 and 64 and schedule_timezone !~ '[[:cntrl:]]') + ) + and retention_days >= 0 + and retention_count >= 0 + and ( + not schedule_enabled + or ( + endpoint is not null + and bucket is not null + and region is not null + and access_key_id is not null + and secret_access_key is not null + and prefix is not null + and cron_expression is not null + and schedule_timezone is not null + and last_verified_at is not null + ) + ) + ), + -- 计划启停与游标一致性：启用必有 next_run_at，关闭必清空。 + constraint backup_settings_cursor_ck check ( + (not schedule_enabled and next_run_at is null) + or (schedule_enabled and next_run_at is not null) + ) +); + +insert into backup_settings ( + id, + storage_revision, + schedule_enabled, + retention_days, + retention_count, + force_path_style, + schedule_timezone, + updated_at +) +values (1, 1, false, 0, 0, false, 'Asia/Shanghai', now()); + +create table backup_records ( + id text primary key, + trigger_kind text not null, + status text not null, + scheduled_at timestamptz, + object_key text not null, + size_bytes bigint, + sha256 text, + expires_at timestamptz, + attempt_count integer not null default 0, + error_code text, + error_message text, + started_at timestamptz, + completed_at timestamptz, + created_at timestamptz not null, + updated_at timestamptz not null, + constraint backup_records_id_ck check (id ~ '^backup_[0-9a-f]{32}$'), + constraint backup_records_trigger_kind_ck check ( + trigger_kind in ('manual', 'scheduled') + and ( + (trigger_kind = 'manual' and scheduled_at is null) + or (trigger_kind = 'scheduled' and scheduled_at is not null) + ) + ), + constraint backup_records_status_ck check ( + status in ('queued', 'dumping', 'uploading', 'completed', 'failed', 'deleting') + ), + constraint backup_records_attempt_ck check (attempt_count >= 0), + constraint backup_records_lifecycle_ck check ( + created_at <= updated_at + and (started_at is null or started_at >= created_at) + and (completed_at is null or completed_at >= created_at) + and ( + ( + status = 'queued' + and started_at is null + and completed_at is null + ) + or ( + status in ('dumping', 'uploading') + and started_at is not null + and completed_at is null + ) + or ( + status in ('completed', 'failed', 'deleting') + and started_at is not null + and completed_at is not null + ) + ) + ), + constraint backup_records_artifact_ck check ( + ( + status not in ('uploading', 'completed') + or (size_bytes is not null and sha256 is not null) + ) + and (size_bytes is null or size_bytes >= 0) + and (sha256 is null or sha256 ~ '^[0-9a-f]{64}$') + ), + constraint backup_records_error_ck check ( + (error_code is null) = (error_message is null) + and (status <> 'failed' or (error_code is not null and error_message is not null)) + and ( + status not in ('queued', 'dumping', 'uploading', 'completed') + or error_code is null + ) + ), + -- expires_at 为空，或不早于创建时间（创建时 now + 天数 恒晚于 created_at）。 + constraint backup_records_expiry_ck check ( + expires_at is null or expires_at >= created_at + ) +); + +create index backup_records_created_idx + on backup_records (created_at desc, id desc); +create index backup_records_status_idx + on backup_records (status, created_at, id); +create unique index backup_records_scheduled_uq + on backup_records (scheduled_at) + where trigger_kind = 'scheduled' and scheduled_at is not null; +create unique index backup_records_active_uq + on backup_records ((1)) + where status in ('queued', 'dumping', 'uploading'); +create unique index backup_records_object_key_uq + on backup_records (object_key) + where object_key is not null; +-- 同时服务 retentionDays 与 retentionCount。 +create index backup_records_scheduled_completed_idx + on backup_records (completed_at desc, id desc) + where trigger_kind = 'scheduled' and status = 'completed'; +-- 可清理记录（expires_at 已到期且非活跃）的扫描索引。 +create index backup_records_retention_idx + on backup_records (expires_at) + where expires_at is not null and status in ('completed', 'failed'); diff --git a/deploy/Dockerfile b/deploy/Dockerfile index 4ad45eab0..3966ebd77 100644 --- a/deploy/Dockerfile +++ b/deploy/Dockerfile @@ -1,5 +1,9 @@ # syntax=docker/dockerfile:1.7@sha256:a57df69d0ea827fb7266491f2813635de6f17269be881f696fbfdf2d83dda33e +# PostgreSQL 服务端镜像：备份客户端（pg_dump/pg_restore）从这里取，与 compose 同源、 +# 版本严格一致。升级数据库版本时只需同步修改这里与 deploy/compose.yaml。 +ARG POSTGRES_IMAGE=postgres:18-bookworm@sha256:d9c83446333daec3f0588cc709adb80c26090b7f9f0f7ec8d43c243385d79818 + FROM node:26-bookworm-slim@sha256:2d49d876e96237d76de412761cf05dbfe5aee325cc4406a4d41d5824c5bb8beb AS frontend-builder WORKDIR /app/frontend @@ -33,23 +37,35 @@ RUN --mount=type=cache,target=/usr/local/cargo/registry \ cargo build --release --locked --bin codex-proxy-rs \ && cp "${CARGO_TARGET_DIR}/release/codex-proxy-rs" /tmp/codex-proxy-rs +# 备份客户端来源：与 compose 服务端同一固定镜像，版本一致。 +FROM ${POSTGRES_IMAGE} AS pg-client + FROM debian:bookworm-slim@sha256:7b140f374b289a7c2befc338f42ebe6441b7ea838a042bbd5acbfca6ec875818 AS runtime-base ENV TZ=Asia/Shanghai +# 运行时基础只从 Debian 官方源补 libpq（pg_dump 的动态依赖；ABI 兼容，无需引入第三方 +# APT 源）。pg_dump/pg_restore 二进制由 pg-client stage COPY，构建期不依赖运行时安装。 RUN apt-get update \ - && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends ca-certificates curl tzdata \ + && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends ca-certificates curl tzdata libpq5 \ + && install -d /usr/lib/postgresql/18/bin \ && ln -snf "/usr/share/zoneinfo/${TZ}" /etc/localtime \ && printf '%s\n' "${TZ}" > /etc/timezone \ && groupadd --system --gid 10001 cpr \ && useradd --system --uid 10001 --gid cpr --home-dir /app --shell /usr/sbin/nologin cpr \ && install -d -o cpr -g cpr \ /app/bin /app/deploy /app/.runtime/data /app/.runtime/logs /app/web \ + && install -d -o cpr -g cpr -m 0700 /app/.runtime/data/backup-staging \ && rm -rf /var/lib/apt/lists/* +COPY --from=pg-client /usr/lib/postgresql/18/bin/pg_dump /usr/lib/postgresql/18/bin/pg_dump +COPY --from=pg-client /usr/lib/postgresql/18/bin/pg_restore /usr/lib/postgresql/18/bin/pg_restore +RUN /usr/lib/postgresql/18/bin/pg_dump --version \ + && /usr/lib/postgresql/18/bin/pg_restore --version + WORKDIR /app ENV CPR_DEPLOYMENT_MODE=docker -ENV PATH="/app/bin:${PATH}" +ENV PATH="/app/bin:/usr/lib/postgresql/18/bin:${PATH}" EXPOSE 8080 diff --git a/deploy/README.md b/deploy/README.md index 8931bc036..6c309ea5b 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -191,3 +191,35 @@ Release 必须提供当前 OS/架构的 `codex-proxy-rs___.ta .runtime/data/update.lock .runtime/data/update-tmp/ ``` + +## 备份与恢复 + +### 运行时事实 + +- 运行时镜像的 `pg_dump`/`pg_restore` 在构建期从与 compose 服务端同一固定 + `postgres:18-bookworm` 镜像（`Dockerfile` 的 `POSTGRES_IMAGE`）COPY 到 + `/usr/lib/postgresql/18/bin` 并加入容器 PATH，版本与服务端严格一致；运行时只从 + Debian 官方源补 `libpq5` 依赖，不引入第三方 APT 源，也不依赖运行时动态安装。 +- 备份暂存目录为 `/app/.runtime/data/backup-staging`，由 `.runtime/data` 卷持久化，权限 + `0700`（仅 `cpr` 用户可读写）。部署卷至少预留一个最大数据库归档的空间。 +- S3/R2 存储、Cron 计划、保留策略与备份记录都保存在 PostgreSQL（`backup_settings` / + `backup_records`），备份记录行在删除成功后硬删除，操作历史进入 `admin_audit_events`。 + +### 人工恢复数据库 + +备份归档由 `pg_dump --format=custom --no-owner --no-privileges` 生成，可通过标准 PostgreSQL +工具离线恢复： + +```bash +pg_restore --no-owner --no-privileges --dbname= .dump +``` + +恢复期间与恢复后的边界（见 `docs/s3-backup-design-audit.md` §8）： + +1. 恢复期间以维护模式启动，不运行备份 Worker（避免把快照中的中间状态当作当前任务继续处理）。 +2. 恢复完成后清理备份控制记录的非终态（`queued/dumping/uploading/deleting`），并重新计算 + `next_run_at`，避免按旧游标立即触发计划。 +3. 重新执行当前 S3 配置的连接探针，再允许计划任务运行。 +4. 不自动删除恢复前已经存在的远端对象。 + +首版不提供在线恢复 API；恢复属于人工运维操作。 diff --git a/frontend/src/api/index.ts b/frontend/src/api/index.ts index 9c7bb227b..f5499d3da 100644 --- a/frontend/src/api/index.ts +++ b/frontend/src/api/index.ts @@ -7,6 +7,9 @@ export * from './modules/api-keys' // 认证相关 export * from './modules/auth' +// 备份管理 +export * from './modules/backups' + // Dashboard 聚合 export * from './modules/dashboard' diff --git a/frontend/src/api/modules/backups.ts b/frontend/src/api/modules/backups.ts new file mode 100644 index 000000000..1eb2858d2 --- /dev/null +++ b/frontend/src/api/modules/backups.ts @@ -0,0 +1,157 @@ +import request from '../request' + +export interface BackupSettingsView { + storageRevision: number + endpoint: string | null + region: string | null + bucket: string | null + accessKeyId: string | null + secretAccessKey: string | null + prefix: string | null + forcePathStyle: boolean + verified: boolean + scheduleEnabled: boolean + cronExpression: string | null + scheduleTimezone: string | null + retentionDays: number + retentionCount: number + nextRunAt: string | null + lastVerifiedAt: string | null + updatedAt: string +} + +export interface UpdateBackupStoragePayload { + endpoint: string + region: string + bucket: string + accessKeyId: string + secretAccessKey?: string + prefix: string + forcePathStyle: boolean +} + +export interface UpdateBackupSchedulePayload { + scheduleEnabled: boolean + cronExpression: string + scheduleTimezone: string + retentionDays: number + retentionCount: number +} + +export interface ConnectionTestResult { + ok: boolean + stage: string + code: string | null + message: string +} + +export type BackupTriggerKind = 'manual' | 'scheduled' +export type BackupStatus + = | 'queued' + | 'dumping' + | 'uploading' + | 'completed' + | 'failed' + | 'deleting' + +export interface BackupRecord { + id: string + triggerKind: BackupTriggerKind + status: BackupStatus + scheduledAt: string | null + objectKey: string + sizeBytes: number | null + sha256: string | null + attemptCount: number + errorCode: string | null + errorMessage: string | null + startedAt: string | null + completedAt: string | null + expiresAt: string | null + createdAt: string + updatedAt: string +} + +export interface BackupRecordPage { + items: BackupRecord[] + page: { + page: number + pageSize: number + total: number + totalPages: number + } +} + +export interface BackupRecordsParams { + page: number + pageSize: number + status?: BackupStatus + trigger?: BackupTriggerKind +} + +export interface DownloadUrlResult { + url: string + fileName: string + expiresInSeconds: number +} + +export function getBackupSettings() { + return request({ + url: '/api/admin/settings/backups', + method: 'GET', + }) +} + +export function updateBackupStorage(data: UpdateBackupStoragePayload) { + return request({ + url: '/api/admin/settings/backups/storage/update', + method: 'POST', + data, + }) +} + +export function testBackupStorage() { + return request({ + url: '/api/admin/settings/backups/storage/test', + method: 'POST', + }) +} + +export function updateBackupSchedule(data: UpdateBackupSchedulePayload) { + return request({ + url: '/api/admin/settings/backups/schedule/update', + method: 'POST', + data, + }) +} + +export function getBackupRecords(params: BackupRecordsParams) { + return request({ + url: '/api/admin/settings/backups/records', + method: 'GET', + params, + }) +} + +export function createBackup() { + return request({ + url: '/api/admin/settings/backups/create', + method: 'POST', + }) +} + +export function getBackupDownloadUrl(backupId: string) { + return request({ + url: '/api/admin/settings/backups/download-url', + method: 'POST', + data: { backupId }, + }) +} + +export function deleteBackup(backupId: string) { + return request({ + url: '/api/admin/settings/backups/delete', + method: 'POST', + data: { backupId }, + }) +} diff --git a/frontend/src/components/base/BaseCheckbox.vue b/frontend/src/components/base/BaseCheckbox.vue index 322a702ce..18f2fc046 100644 --- a/frontend/src/components/base/BaseCheckbox.vue +++ b/frontend/src/components/base/BaseCheckbox.vue @@ -7,11 +7,13 @@ const props = withDefaults( indeterminate?: boolean disabled?: boolean label?: string + showLabel?: boolean }>(), { indeterminate: false, disabled: false, label: undefined, + showLabel: false, }, ) @@ -58,5 +60,11 @@ function toggle() { :class="[!indeterminate && model ? 'opacity-100' : 'opacity-0']" /> + + {{ label }} + diff --git a/frontend/src/router/routes.ts b/frontend/src/router/routes.ts index 4b84a9a21..7ce69a508 100644 --- a/frontend/src/router/routes.ts +++ b/frontend/src/router/routes.ts @@ -31,7 +31,7 @@ export const routes: RouteRecordRaw[] = [ component: () => import('@/views/usage/index.vue'), }, { - path: 'settings', + path: 'settings/:section?', name: 'settings', component: () => import('@/views/settings/index.vue'), }, diff --git a/frontend/src/views/settings/components/ModelAliasesCard.vue b/frontend/src/views/settings/components/ModelAliasesCard.vue index 2233f2ff0..d252089e7 100644 --- a/frontend/src/views/settings/components/ModelAliasesCard.vue +++ b/frontend/src/views/settings/components/ModelAliasesCard.vue @@ -1,12 +1,11 @@