mirror of
https://github.com/neondatabase/neon.git
synced 2026-01-08 14:02:55 +00:00
2621ce2daf
## Problem Timeline imports do not have progress checkpointing. Any time that the tenant is shut-down, all progress is lost and the import restarts from the beginning when the tenant is re-attached. ## Summary of changes This PR adds progress checkpointing. ### Preliminaries The **unit of work** is a `ChunkProcessingJob`. Each `ChunkProcessingJob` deals with the import for a set of key ranges. The job split is done by using an estimation of how many pages each job will produce. The planning stage must be **pure**: given a fixed set of contents in the import bucket, it will always yield the same plan. This property is enforced by checking that the hash of the plan is identical when resuming from a checkpoint. The storage controller tracks the progress of each shard in the import in the database in the form of the **latest job** that has has completed. ### Flow This is the high level flow for the happy path: 1. On the first run of the import task, the import task queries storcon for the progress and sees that none is recorded. 2. Execute the preparatory stage of the import 3. Import jobs start running concurrently in a `FuturesOrdered`. Every time the checkpointing threshold of jobs has been reached, notify the storage controller. 4. Tenant is detached and re-attached 5. Import task starts up again and gets the latest progress checkpoint from the storage controller in the form of a job index. 6. The plan is computed again and we check that the hash matches with the original plan. 7. Jobs are spawned from where the previous import task left off. Note that we will not report progress after the completion of each job, so some jobs might run twice. Closes https://github.com/neondatabase/neon/issues/11568 Closes https://github.com/neondatabase/neon/issues/11664
135 lines
3.7 KiB
TOML
135 lines
3.7 KiB
TOML
[package]
|
|
name = "pageserver"
|
|
version = "0.1.0"
|
|
edition = "2024"
|
|
license.workspace = true
|
|
|
|
[features]
|
|
default = []
|
|
# Enables test-only APIs, incuding failpoints. In particular, enables the `fail_point!` macro,
|
|
# which adds some runtime cost to run tests on outage conditions
|
|
testing = ["fail/failpoints", "pageserver_api/testing", "wal_decoder/testing", "pageserver_client/testing"]
|
|
|
|
fuzz-read-path = ["testing"]
|
|
|
|
[dependencies]
|
|
anyhow.workspace = true
|
|
arc-swap.workspace = true
|
|
async-compression.workspace = true
|
|
async-stream.workspace = true
|
|
bit_field.workspace = true
|
|
bincode.workspace = true
|
|
byteorder.workspace = true
|
|
bytes.workspace = true
|
|
camino.workspace = true
|
|
camino-tempfile.workspace = true
|
|
chrono = { workspace = true, features = ["serde"] }
|
|
clap = { workspace = true, features = ["string"] }
|
|
consumption_metrics.workspace = true
|
|
crc32c.workspace = true
|
|
either.workspace = true
|
|
fail.workspace = true
|
|
futures.workspace = true
|
|
hex.workspace = true
|
|
humantime.workspace = true
|
|
humantime-serde.workspace = true
|
|
hyper0.workspace = true
|
|
itertools.workspace = true
|
|
jsonwebtoken.workspace = true
|
|
md5.workspace = true
|
|
nix.workspace = true
|
|
# hack to get the number of worker threads tokio uses
|
|
num_cpus.workspace = true
|
|
num-traits.workspace = true
|
|
once_cell.workspace = true
|
|
pin-project-lite.workspace = true
|
|
postgres_backend.workspace = true
|
|
postgres-protocol.workspace = true
|
|
postgres-types.workspace = true
|
|
postgres_initdb.workspace = true
|
|
pprof.workspace = true
|
|
rand.workspace = true
|
|
range-set-blaze = { version = "0.1.16", features = ["alloc"] }
|
|
regex.workspace = true
|
|
rustls.workspace = true
|
|
scopeguard.workspace = true
|
|
send-future.workspace = true
|
|
serde.workspace = true
|
|
serde_json = { workspace = true, features = ["raw_value"] }
|
|
serde_path_to_error.workspace = true
|
|
serde_with.workspace = true
|
|
sysinfo.workspace = true
|
|
tokio-tar.workspace = true
|
|
thiserror.workspace = true
|
|
tikv-jemallocator.workspace = true
|
|
tokio = { workspace = true, features = ["process", "sync", "fs", "rt", "io-util", "time"] }
|
|
tokio-epoll-uring.workspace = true
|
|
tokio-io-timeout.workspace = true
|
|
tokio-postgres.workspace = true
|
|
tokio-rustls.workspace = true
|
|
tokio-stream.workspace = true
|
|
tokio-util.workspace = true
|
|
toml_edit = { workspace = true, features = [ "serde" ] }
|
|
tracing.workspace = true
|
|
tracing-utils.workspace = true
|
|
url.workspace = true
|
|
walkdir.workspace = true
|
|
metrics.workspace = true
|
|
pageserver_api.workspace = true
|
|
pageserver_client.workspace = true # for ResponseErrorMessageExt TOOD refactor that
|
|
pageserver_compaction.workspace = true
|
|
pem.workspace = true
|
|
postgres_connection.workspace = true
|
|
postgres_ffi.workspace = true
|
|
pq_proto.workspace = true
|
|
remote_storage.workspace = true
|
|
storage_broker.workspace = true
|
|
tenant_size_model.workspace = true
|
|
http-utils.workspace = true
|
|
utils.workspace = true
|
|
workspace_hack.workspace = true
|
|
reqwest.workspace = true
|
|
rpds.workspace = true
|
|
enum-map.workspace = true
|
|
enumset = { workspace = true, features = ["serde"]}
|
|
strum.workspace = true
|
|
strum_macros.workspace = true
|
|
wal_decoder.workspace = true
|
|
smallvec.workspace = true
|
|
twox-hash.workspace = true
|
|
|
|
[target.'cfg(target_os = "linux")'.dependencies]
|
|
procfs.workspace = true
|
|
|
|
[dev-dependencies]
|
|
criterion.workspace = true
|
|
hex-literal.workspace = true
|
|
tokio = { workspace = true, features = ["process", "sync", "fs", "rt", "io-util", "time", "test-util"] }
|
|
indoc.workspace = true
|
|
uuid.workspace = true
|
|
rstest.workspace = true
|
|
|
|
[[bench]]
|
|
name = "bench_layer_map"
|
|
harness = false
|
|
|
|
[[bench]]
|
|
name = "bench_walredo"
|
|
harness = false
|
|
|
|
[[bench]]
|
|
name = "bench_ingest"
|
|
harness = false
|
|
|
|
[[bench]]
|
|
name = "upload_queue"
|
|
harness = false
|
|
|
|
[[bench]]
|
|
name = "bench_metrics"
|
|
harness = false
|
|
|
|
[[bin]]
|
|
name = "test_helper_slow_client_reads"
|
|
required-features = [ "testing" ]
|