mirror of
https://github.com/neondatabase/neon.git
synced 2026-01-08 14:02:55 +00:00
450be26bbb
Co-authored-by: Heikki Linnakangas <heikki@neon.tech> Co-authored-by: Stas Kelvic <stas@neon.tech> # Context This PR contains PoC-level changes for a product feature that allows onboarding large databases into Neon without going through the regular data path. # Changes This internal RFC provides all the context * https://github.com/neondatabase/cloud/pull/19799 In the language of the RFC, this PR covers * the Importer code (`fast_import`) * all the Pageserver changes (mgmt API changes, flow implementation, etc) * a basic test for the Pageserver changes # Reviewing As acknowledged in the RFC, the code added in this PR is not ready for general availability. Also, the **architecture is not to be discussed in this PR**, but in the RFC and associated Slack channel instead. Reviewers of this PR should take that into consideration. The quality bar to apply during review depends on what area of the code is being reviewed: * Importer code (`fast_import`): practically anything goes * Core flow (`flow.rs`): * Malicious input data must be expected and the existing threat models apply. * The code must not be safe to execute on *dedicated* Pageserver instances: * This means in particular that tenants *on other* Pageserver instances must not be affected negatively wrt data confidentiality, integrity or availability. * Other code: the usual quality bar * Pay special attention to correct use of gate guards, timeline cancellation in all places during shutdown & migration, etc. * Consider the broader system impact; if you find potentially problematic interactions with Storage features that were not covered in the RFC, bring that up during the review. I recommend submitting three separate reviews, for the three high-level areas with different quality bars. # References (Internal-only) * refs https://github.com/neondatabase/cloud/issues/17507 * refs https://github.com/neondatabase/company_projects/issues/293 * refs https://github.com/neondatabase/company_projects/issues/309 * refs https://github.com/neondatabase/cloud/issues/20646 --------- Co-authored-by: Stas Kelvich <stas.kelvich@gmail.com> Co-authored-by: Heikki Linnakangas <heikki@neon.tech> Co-authored-by: John Spray <john@neon.tech>
138 lines
8.3 KiB
TOML
138 lines
8.3 KiB
TOML
# This file is generated by `cargo hakari`.
|
|
# To regenerate, run:
|
|
# cargo hakari generate
|
|
|
|
[package]
|
|
name = "workspace_hack"
|
|
version = "0.1.0"
|
|
description = "workspace-hack package, managed by hakari"
|
|
# You can choose to publish this crate: see https://docs.rs/cargo-hakari/latest/cargo_hakari/publishing.
|
|
publish = false
|
|
edition.workspace = true
|
|
license.workspace = true
|
|
# The parts of the file between the BEGIN HAKARI SECTION and END HAKARI SECTION comments
|
|
# are managed by hakari.
|
|
|
|
### BEGIN HAKARI SECTION
|
|
[dependencies]
|
|
ahash = { version = "0.8" }
|
|
anyhow = { version = "1", features = ["backtrace"] }
|
|
axum = { version = "0.7", features = ["ws"] }
|
|
axum-core = { version = "0.4", default-features = false, features = ["tracing"] }
|
|
base64-594e8ee84c453af0 = { package = "base64", version = "0.13", features = ["alloc"] }
|
|
base64-647d43efb71741da = { package = "base64", version = "0.21", features = ["alloc"] }
|
|
base64ct = { version = "1", default-features = false, features = ["std"] }
|
|
bytes = { version = "1", features = ["serde"] }
|
|
camino = { version = "1", default-features = false, features = ["serde1"] }
|
|
chrono = { version = "0.4", default-features = false, features = ["clock", "serde", "wasmbind"] }
|
|
clap = { version = "4", features = ["derive", "env", "string"] }
|
|
clap_builder = { version = "4", default-features = false, features = ["color", "env", "help", "std", "string", "suggestions", "usage"] }
|
|
crypto-bigint = { version = "0.5", features = ["generic-array", "zeroize"] }
|
|
der = { version = "0.7", default-features = false, features = ["oid", "pem", "std"] }
|
|
deranged = { version = "0.3", default-features = false, features = ["powerfmt", "serde", "std"] }
|
|
digest = { version = "0.10", features = ["mac", "oid", "std"] }
|
|
either = { version = "1" }
|
|
fail = { version = "0.5", default-features = false, features = ["failpoints"] }
|
|
futures-channel = { version = "0.3", features = ["sink"] }
|
|
futures-executor = { version = "0.3" }
|
|
futures-io = { version = "0.3" }
|
|
futures-util = { version = "0.3", features = ["channel", "io", "sink"] }
|
|
generic-array = { version = "0.14", default-features = false, features = ["more_lengths", "zeroize"] }
|
|
getrandom = { version = "0.2", default-features = false, features = ["std"] }
|
|
half = { version = "2", default-features = false, features = ["num-traits"] }
|
|
hashbrown = { version = "0.14", features = ["raw"] }
|
|
hex = { version = "0.4", features = ["serde"] }
|
|
hmac = { version = "0.12", default-features = false, features = ["reset"] }
|
|
hyper-582f2526e08bb6a0 = { package = "hyper", version = "0.14", features = ["full"] }
|
|
hyper-dff4ba8e3ae991db = { package = "hyper", version = "1", features = ["full"] }
|
|
hyper-util = { version = "0.1", features = ["client-legacy", "server-auto", "service"] }
|
|
indexmap-dff4ba8e3ae991db = { package = "indexmap", version = "1", default-features = false, features = ["std"] }
|
|
indexmap-f595c2ba2a3f28df = { package = "indexmap", version = "2", features = ["serde"] }
|
|
itertools = { version = "0.12" }
|
|
lazy_static = { version = "1", default-features = false, features = ["spin_no_std"] }
|
|
libc = { version = "0.2", features = ["extra_traits", "use_std"] }
|
|
log = { version = "0.4", default-features = false, features = ["std"] }
|
|
memchr = { version = "2" }
|
|
nix = { version = "0.26" }
|
|
nom = { version = "7" }
|
|
num-bigint = { version = "0.4" }
|
|
num-integer = { version = "0.1", features = ["i128"] }
|
|
num-traits = { version = "0.2", features = ["i128", "libm"] }
|
|
once_cell = { version = "1" }
|
|
parquet = { version = "53", default-features = false, features = ["zstd"] }
|
|
postgres-types = { git = "https://github.com/neondatabase/rust-postgres.git", branch = "neon", default-features = false, features = ["with-serde_json-1"] }
|
|
prost = { version = "0.13", features = ["prost-derive"] }
|
|
rand = { version = "0.8", features = ["small_rng"] }
|
|
regex = { version = "1" }
|
|
regex-automata = { version = "0.4", default-features = false, features = ["dfa-onepass", "hybrid", "meta", "nfa-backtrack", "perf-inline", "perf-literal", "unicode"] }
|
|
regex-syntax = { version = "0.8" }
|
|
reqwest = { version = "0.12", default-features = false, features = ["blocking", "json", "rustls-tls", "rustls-tls-native-roots", "stream"] }
|
|
rustls = { version = "0.23", default-features = false, features = ["logging", "ring", "std", "tls12"] }
|
|
scopeguard = { version = "1" }
|
|
serde = { version = "1", features = ["alloc", "derive"] }
|
|
serde_json = { version = "1", features = ["alloc", "raw_value"] }
|
|
sha2 = { version = "0.10", features = ["asm", "oid"] }
|
|
signature = { version = "2", default-features = false, features = ["digest", "rand_core", "std"] }
|
|
smallvec = { version = "1", default-features = false, features = ["const_new", "write"] }
|
|
spki = { version = "0.7", default-features = false, features = ["pem", "std"] }
|
|
subtle = { version = "2" }
|
|
sync_wrapper = { version = "0.1", default-features = false, features = ["futures"] }
|
|
tikv-jemalloc-sys = { version = "0.6", features = ["stats"] }
|
|
time = { version = "0.3", features = ["macros", "serde-well-known"] }
|
|
tokio = { version = "1", features = ["fs", "io-std", "io-util", "macros", "net", "process", "rt-multi-thread", "signal", "test-util"] }
|
|
tokio-postgres = { git = "https://github.com/neondatabase/rust-postgres.git", branch = "neon", features = ["with-serde_json-1"] }
|
|
tokio-rustls = { version = "0.26", default-features = false, features = ["logging", "ring", "tls12"] }
|
|
tokio-stream = { version = "0.1", features = ["net"] }
|
|
tokio-util = { version = "0.7", features = ["codec", "compat", "io", "rt"] }
|
|
toml_edit = { version = "0.22", features = ["serde"] }
|
|
tonic = { version = "0.12", features = ["tls-roots"] }
|
|
tower = { version = "0.4", default-features = false, features = ["balance", "buffer", "limit", "log", "util"] }
|
|
tracing = { version = "0.1", features = ["log"] }
|
|
tracing-core = { version = "0.1" }
|
|
url = { version = "2", features = ["serde"] }
|
|
zerocopy = { version = "0.7", features = ["derive", "simd"] }
|
|
zeroize = { version = "1", features = ["derive", "serde"] }
|
|
zstd = { version = "0.13" }
|
|
zstd-safe = { version = "7", default-features = false, features = ["arrays", "legacy", "std", "zdict_builder"] }
|
|
zstd-sys = { version = "2", default-features = false, features = ["legacy", "std", "zdict_builder"] }
|
|
|
|
[build-dependencies]
|
|
ahash = { version = "0.8" }
|
|
anyhow = { version = "1", features = ["backtrace"] }
|
|
bytes = { version = "1", features = ["serde"] }
|
|
cc = { version = "1", default-features = false, features = ["parallel"] }
|
|
chrono = { version = "0.4", default-features = false, features = ["clock", "serde", "wasmbind"] }
|
|
either = { version = "1" }
|
|
getrandom = { version = "0.2", default-features = false, features = ["std"] }
|
|
half = { version = "2", default-features = false, features = ["num-traits"] }
|
|
hashbrown = { version = "0.14", features = ["raw"] }
|
|
indexmap-dff4ba8e3ae991db = { package = "indexmap", version = "1", default-features = false, features = ["std"] }
|
|
indexmap-f595c2ba2a3f28df = { package = "indexmap", version = "2", features = ["serde"] }
|
|
itertools = { version = "0.12" }
|
|
libc = { version = "0.2", features = ["extra_traits", "use_std"] }
|
|
log = { version = "0.4", default-features = false, features = ["std"] }
|
|
memchr = { version = "2" }
|
|
nom = { version = "7" }
|
|
num-bigint = { version = "0.4" }
|
|
num-integer = { version = "0.1", features = ["i128"] }
|
|
num-traits = { version = "0.2", features = ["i128", "libm"] }
|
|
once_cell = { version = "1" }
|
|
parquet = { version = "53", default-features = false, features = ["zstd"] }
|
|
prettyplease = { version = "0.2", default-features = false, features = ["verbatim"] }
|
|
proc-macro2 = { version = "1" }
|
|
prost = { version = "0.13", features = ["prost-derive"] }
|
|
quote = { version = "1" }
|
|
regex = { version = "1" }
|
|
regex-automata = { version = "0.4", default-features = false, features = ["dfa-onepass", "hybrid", "meta", "nfa-backtrack", "perf-inline", "perf-literal", "unicode"] }
|
|
regex-syntax = { version = "0.8" }
|
|
serde = { version = "1", features = ["alloc", "derive"] }
|
|
syn = { version = "2", features = ["extra-traits", "fold", "full", "visit", "visit-mut"] }
|
|
time-macros = { version = "0.2", default-features = false, features = ["formatting", "parsing", "serde"] }
|
|
toml_edit = { version = "0.22", features = ["serde"] }
|
|
zerocopy = { version = "0.7", features = ["derive", "simd"] }
|
|
zstd = { version = "0.13" }
|
|
zstd-safe = { version = "7", default-features = false, features = ["arrays", "legacy", "std", "zdict_builder"] }
|
|
zstd-sys = { version = "2", default-features = false, features = ["legacy", "std", "zdict_builder"] }
|
|
|
|
### END HAKARI SECTION
|