mirror of
https://github.com/neondatabase/neon.git
synced 2026-01-07 21:42:56 +00:00
3f77f26aa2
Add support for backing up partial segments to remote storage. Disabled
by default, can be enabled with `--partial-backup-enabled`.
Safekeeper timeline has a background task which is subscribed to
`commit_lsn` and `flush_lsn` updates. After the partial segment was
updated (`flush_lsn` was changed), the segment will be uploaded to S3 in
about 15 minutes.
The filename format for partial segments is
`Segment_Term_Flush_Commit_skNN.partial`, where:
- `Segment` – the segment name, like `000000010000000000000001`
- `Term` – current term
- `Flush` – flush_lsn in hex format `{:016X}`, e.g. `00000000346BC568`
- `Commit` – commit_lsn in the same hex format
- `NN` – safekeeper_id, like `1`
The full object name example:
`000000010000000000000002_2_0000000002534868_0000000002534410_sk1.partial`
Each safekeeper will keep info about remote partial segments in its
control file. Code updates state in the control file before doing any S3
operations. This way control file stores information about all
potentially existing remote partial segments and can clean them up after
uploading a newer version.
Closes #6336
72 lines
1.9 KiB
TOML
72 lines
1.9 KiB
TOML
[package]
|
|
name = "safekeeper"
|
|
version = "0.1.0"
|
|
edition.workspace = true
|
|
license.workspace = true
|
|
|
|
[features]
|
|
default = []
|
|
# Enables test-only APIs, incuding failpoints. In particular, enables the `fail_point!` macro,
|
|
# which adds some runtime cost to run tests on outage conditions
|
|
testing = ["fail/failpoints"]
|
|
|
|
[dependencies]
|
|
async-stream.workspace = true
|
|
anyhow.workspace = true
|
|
async-trait.workspace = true
|
|
byteorder.workspace = true
|
|
bytes.workspace = true
|
|
camino.workspace = true
|
|
camino-tempfile.workspace = true
|
|
chrono.workspace = true
|
|
clap = { workspace = true, features = ["derive"] }
|
|
const_format.workspace = true
|
|
crc32c.workspace = true
|
|
fail.workspace = true
|
|
fs2.workspace = true
|
|
git-version.workspace = true
|
|
hex.workspace = true
|
|
humantime.workspace = true
|
|
hyper.workspace = true
|
|
futures.workspace = true
|
|
once_cell.workspace = true
|
|
parking_lot.workspace = true
|
|
postgres.workspace = true
|
|
postgres-protocol.workspace = true
|
|
rand.workspace = true
|
|
regex.workspace = true
|
|
scopeguard.workspace = true
|
|
reqwest = { workspace = true, features = ["json"] }
|
|
serde.workspace = true
|
|
serde_json.workspace = true
|
|
serde_with.workspace = true
|
|
signal-hook.workspace = true
|
|
thiserror.workspace = true
|
|
tokio = { workspace = true, features = ["fs"] }
|
|
tokio-util = { workspace = true }
|
|
tokio-io-timeout.workspace = true
|
|
tokio-postgres.workspace = true
|
|
toml_edit.workspace = true
|
|
tracing.workspace = true
|
|
url.workspace = true
|
|
metrics.workspace = true
|
|
postgres_backend.workspace = true
|
|
postgres_ffi.workspace = true
|
|
pq_proto.workspace = true
|
|
remote_storage.workspace = true
|
|
safekeeper_api.workspace = true
|
|
sha2.workspace = true
|
|
sd-notify.workspace = true
|
|
storage_broker.workspace = true
|
|
tokio-stream.workspace = true
|
|
utils.workspace = true
|
|
|
|
workspace_hack.workspace = true
|
|
|
|
[dev-dependencies]
|
|
walproposer.workspace = true
|
|
rand.workspace = true
|
|
desim.workspace = true
|
|
tracing.workspace = true
|
|
tracing-subscriber = { workspace = true, features = ["json"] }
|