mirror of
https://github.com/neondatabase/neon.git
synced 2026-01-16 01:42:55 +00:00
c4e0ef507f
Dependency (commits inline): https://github.com/neondatabase/neon/pull/5842 ## Problem Secondary mode tenants need a manifest of what to download. Ultimately this will be some kind of heat-scored set of layers, but as a robust first step we will simply use the set of resident layers: secondary tenant locations will aim to match the on-disk content of the attached location. ## Summary of changes - Add heatmap types representing the remote structure - Add hooks to Tenant/Timeline for generating these heatmaps - Create a new `HeatmapUploader` type that is external to `Tenant`, and responsible for walking the list of attached tenants and scheduling heatmap uploads. Notes to reviewers: - Putting the logic for uploads (and later, secondary mode downloads) outside of `Tenant` is an opinionated choice, motivated by: - Enable future smarter scheduling of operations, e.g. uploading the stalest tenant first, rather than having all tenants compete for a fair semaphore on a first-come-first-served basis. Similarly for downloads, we may wish to schedule the tenants with the hottest un-downloaded layers first. - Enable accessing upload-related state without synchronization (it belongs to HeatmapUploader, rather than being some Mutex<>'d part of Tenant) - Avoid further expanding the scope of Tenant/Timeline types, which are already among the largest in the codebase - You might reasonably wonder how much of the uploader code could be a generic job manager thing. Probably some of it: but let's defer pulling that out until we have at least two users (perhaps secondary downloads will be the second one) to highlight which bits are really generic. Compromises: - Later, instead of using digests of heatmaps to decide whether anything changed, I would prefer to avoid walking the layers in tenants that don't have changes: tracking that will be a bit invasive, as it needs input from both remote_timeline_client and Layer.
100 lines
2.8 KiB
TOML
100 lines
2.8 KiB
TOML
[package]
|
|
name = "pageserver"
|
|
version = "0.1.0"
|
|
edition.workspace = true
|
|
license.workspace = true
|
|
|
|
[features]
|
|
default = []
|
|
# Enables test-only APIs, incuding failpoints. In particular, enables the `fail_point!` macro,
|
|
# which adds some runtime cost to run tests on outage conditions
|
|
testing = ["fail/failpoints"]
|
|
|
|
[dependencies]
|
|
anyhow.workspace = true
|
|
async-compression.workspace = true
|
|
async-stream.workspace = true
|
|
async-trait.workspace = true
|
|
byteorder.workspace = true
|
|
bytes.workspace = true
|
|
camino.workspace = true
|
|
camino-tempfile.workspace = true
|
|
chrono = { workspace = true, features = ["serde"] }
|
|
clap = { workspace = true, features = ["string"] }
|
|
close_fds.workspace = true
|
|
const_format.workspace = true
|
|
consumption_metrics.workspace = true
|
|
crc32c.workspace = true
|
|
crossbeam-utils.workspace = true
|
|
either.workspace = true
|
|
flate2.workspace = true
|
|
fail.workspace = true
|
|
futures.workspace = true
|
|
git-version.workspace = true
|
|
hex.workspace = true
|
|
humantime.workspace = true
|
|
humantime-serde.workspace = true
|
|
hyper.workspace = true
|
|
itertools.workspace = true
|
|
md5.workspace = true
|
|
nix.workspace = true
|
|
# hack to get the number of worker threads tokio uses
|
|
num_cpus = { version = "1.15" }
|
|
num-traits.workspace = true
|
|
once_cell.workspace = true
|
|
pin-project-lite.workspace = true
|
|
postgres.workspace = true
|
|
postgres_backend.workspace = true
|
|
postgres-protocol.workspace = true
|
|
postgres-types.workspace = true
|
|
rand.workspace = true
|
|
regex.workspace = true
|
|
scopeguard.workspace = true
|
|
serde.workspace = true
|
|
serde_json = { workspace = true, features = ["raw_value"] }
|
|
serde_path_to_error.workspace = true
|
|
serde_with.workspace = true
|
|
signal-hook.workspace = true
|
|
smallvec = { workspace = true, features = ["write"] }
|
|
svg_fmt.workspace = true
|
|
sync_wrapper.workspace = true
|
|
tokio-tar.workspace = true
|
|
thiserror.workspace = true
|
|
tokio = { workspace = true, features = ["process", "sync", "fs", "rt", "io-util", "time"] }
|
|
tokio-io-timeout.workspace = true
|
|
tokio-postgres.workspace = true
|
|
tokio-util.workspace = true
|
|
toml_edit = { workspace = true, features = [ "serde" ] }
|
|
tracing.workspace = true
|
|
url.workspace = true
|
|
walkdir.workspace = true
|
|
metrics.workspace = true
|
|
pageserver_api.workspace = true
|
|
postgres_connection.workspace = true
|
|
postgres_ffi.workspace = true
|
|
pq_proto.workspace = true
|
|
remote_storage.workspace = true
|
|
storage_broker.workspace = true
|
|
tenant_size_model.workspace = true
|
|
utils.workspace = true
|
|
workspace_hack.workspace = true
|
|
reqwest.workspace = true
|
|
rpds.workspace = true
|
|
enum-map.workspace = true
|
|
enumset.workspace = true
|
|
strum.workspace = true
|
|
strum_macros.workspace = true
|
|
|
|
[dev-dependencies]
|
|
criterion.workspace = true
|
|
hex-literal.workspace = true
|
|
tokio = { workspace = true, features = ["process", "sync", "fs", "rt", "io-util", "time", "test-util"] }
|
|
|
|
[[bench]]
|
|
name = "bench_layer_map"
|
|
harness = false
|
|
|
|
[[bench]]
|
|
name = "bench_walredo"
|
|
harness = false
|