mirror of
https://github.com/neondatabase/neon.git
synced 2026-01-08 14:02:55 +00:00
450be26bbb
Co-authored-by: Heikki Linnakangas <heikki@neon.tech> Co-authored-by: Stas Kelvic <stas@neon.tech> # Context This PR contains PoC-level changes for a product feature that allows onboarding large databases into Neon without going through the regular data path. # Changes This internal RFC provides all the context * https://github.com/neondatabase/cloud/pull/19799 In the language of the RFC, this PR covers * the Importer code (`fast_import`) * all the Pageserver changes (mgmt API changes, flow implementation, etc) * a basic test for the Pageserver changes # Reviewing As acknowledged in the RFC, the code added in this PR is not ready for general availability. Also, the **architecture is not to be discussed in this PR**, but in the RFC and associated Slack channel instead. Reviewers of this PR should take that into consideration. The quality bar to apply during review depends on what area of the code is being reviewed: * Importer code (`fast_import`): practically anything goes * Core flow (`flow.rs`): * Malicious input data must be expected and the existing threat models apply. * The code must not be safe to execute on *dedicated* Pageserver instances: * This means in particular that tenants *on other* Pageserver instances must not be affected negatively wrt data confidentiality, integrity or availability. * Other code: the usual quality bar * Pay special attention to correct use of gate guards, timeline cancellation in all places during shutdown & migration, etc. * Consider the broader system impact; if you find potentially problematic interactions with Storage features that were not covered in the RFC, bring that up during the review. I recommend submitting three separate reviews, for the three high-level areas with different quality bars. # References (Internal-only) * refs https://github.com/neondatabase/cloud/issues/17507 * refs https://github.com/neondatabase/company_projects/issues/293 * refs https://github.com/neondatabase/company_projects/issues/309 * refs https://github.com/neondatabase/cloud/issues/20646 --------- Co-authored-by: Stas Kelvich <stas.kelvich@gmail.com> Co-authored-by: Heikki Linnakangas <heikki@neon.tech> Co-authored-by: John Spray <john@neon.tech>
111 lines
3.2 KiB
TOML
111 lines
3.2 KiB
TOML
[package]
|
|
name = "pageserver"
|
|
version = "0.1.0"
|
|
edition.workspace = true
|
|
license.workspace = true
|
|
|
|
[features]
|
|
default = []
|
|
# Enables test-only APIs, incuding failpoints. In particular, enables the `fail_point!` macro,
|
|
# which adds some runtime cost to run tests on outage conditions
|
|
testing = ["fail/failpoints", "pageserver_api/testing", "wal_decoder/testing"]
|
|
|
|
[dependencies]
|
|
anyhow.workspace = true
|
|
arc-swap.workspace = true
|
|
async-compression.workspace = true
|
|
async-stream.workspace = true
|
|
bit_field.workspace = true
|
|
byteorder.workspace = true
|
|
bytes.workspace = true
|
|
camino.workspace = true
|
|
camino-tempfile.workspace = true
|
|
chrono = { workspace = true, features = ["serde"] }
|
|
clap = { workspace = true, features = ["string"] }
|
|
consumption_metrics.workspace = true
|
|
crc32c.workspace = true
|
|
either.workspace = true
|
|
fail.workspace = true
|
|
futures.workspace = true
|
|
hex.workspace = true
|
|
humantime.workspace = true
|
|
humantime-serde.workspace = true
|
|
hyper0.workspace = true
|
|
itertools.workspace = true
|
|
md5.workspace = true
|
|
nix.workspace = true
|
|
# hack to get the number of worker threads tokio uses
|
|
num_cpus = { version = "1.15" }
|
|
num-traits.workspace = true
|
|
once_cell.workspace = true
|
|
pin-project-lite.workspace = true
|
|
postgres.workspace = true
|
|
postgres_backend.workspace = true
|
|
postgres-protocol.workspace = true
|
|
postgres-types.workspace = true
|
|
postgres_initdb.workspace = true
|
|
rand.workspace = true
|
|
range-set-blaze = { version = "0.1.16", features = ["alloc"] }
|
|
regex.workspace = true
|
|
scopeguard.workspace = true
|
|
send-future.workspace = true
|
|
serde.workspace = true
|
|
serde_json = { workspace = true, features = ["raw_value"] }
|
|
serde_path_to_error.workspace = true
|
|
serde_with.workspace = true
|
|
sysinfo.workspace = true
|
|
tokio-tar.workspace = true
|
|
thiserror.workspace = true
|
|
tikv-jemallocator.workspace = true
|
|
tokio = { workspace = true, features = ["process", "sync", "fs", "rt", "io-util", "time"] }
|
|
tokio-epoll-uring.workspace = true
|
|
tokio-io-timeout.workspace = true
|
|
tokio-postgres.workspace = true
|
|
tokio-stream.workspace = true
|
|
tokio-util.workspace = true
|
|
toml_edit = { workspace = true, features = [ "serde" ] }
|
|
tracing.workspace = true
|
|
url.workspace = true
|
|
walkdir.workspace = true
|
|
metrics.workspace = true
|
|
pageserver_api.workspace = true
|
|
pageserver_client.workspace = true # for ResponseErrorMessageExt TOOD refactor that
|
|
pageserver_compaction.workspace = true
|
|
postgres_connection.workspace = true
|
|
postgres_ffi.workspace = true
|
|
pq_proto.workspace = true
|
|
remote_storage.workspace = true
|
|
storage_broker.workspace = true
|
|
tenant_size_model.workspace = true
|
|
utils.workspace = true
|
|
workspace_hack.workspace = true
|
|
reqwest.workspace = true
|
|
rpds.workspace = true
|
|
enum-map.workspace = true
|
|
enumset = { workspace = true, features = ["serde"]}
|
|
strum.workspace = true
|
|
strum_macros.workspace = true
|
|
wal_decoder.workspace = true
|
|
smallvec.workspace = true
|
|
|
|
[target.'cfg(target_os = "linux")'.dependencies]
|
|
procfs.workspace = true
|
|
|
|
[dev-dependencies]
|
|
criterion.workspace = true
|
|
hex-literal.workspace = true
|
|
tokio = { workspace = true, features = ["process", "sync", "fs", "rt", "io-util", "time", "test-util"] }
|
|
indoc.workspace = true
|
|
|
|
[[bench]]
|
|
name = "bench_layer_map"
|
|
harness = false
|
|
|
|
[[bench]]
|
|
name = "bench_walredo"
|
|
harness = false
|
|
|
|
[[bench]]
|
|
name = "bench_ingest"
|
|
harness = false
|