From c4ee62d427622d2d576db0e3b2ac3035ea37cd04 Mon Sep 17 00:00:00 2001 From: Kirill Bulatov Date: Mon, 17 Oct 2022 12:58:40 +0300 Subject: [PATCH] Bump clap and other minor dependencies (#2623) --- Cargo.lock | 172 ++++-- compute_tools/Cargo.toml | 2 +- compute_tools/src/bin/compute_ctl.rs | 95 +-- control_plane/Cargo.toml | 2 +- control_plane/src/bin/neon_local.rs | 580 +++++++++--------- libs/postgres_ffi/Cargo.toml | 4 +- libs/postgres_ffi/wal_craft/Cargo.toml | 2 +- .../wal_craft/src/bin/wal_craft.rs | 163 ++--- pageserver/Cargo.toml | 2 +- pageserver/src/bin/pageserver.rs | 121 ++-- pageserver/src/bin/pageserver_binutils.rs | 88 +-- proxy/Cargo.toml | 20 +- proxy/src/main.rs | 160 ++--- safekeeper/Cargo.toml | 2 +- safekeeper/src/bin/safekeeper.rs | 242 ++++---- workspace_hack/Cargo.toml | 7 +- 16 files changed, 893 insertions(+), 769 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 7659be6c92..131d9a8aa2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -75,9 +75,9 @@ checksum = "8da52d66c7071e2e3fa2a1e5c6d088fec47b593032b254f5e980de8ea54454d6" [[package]] name = "asn1-rs" -version = "0.3.1" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30ff05a702273012438132f449575dbc804e27b2f3cbe3069aa237d26c98fa33" +checksum = "cf6690c370453db30743b373a60ba498fc0d6d83b11f4abfd87a84a075db5dd4" dependencies = [ "asn1-rs-derive", "asn1-rs-impl", @@ -91,9 +91,9 @@ dependencies = [ [[package]] name = "asn1-rs-derive" -version = "0.1.0" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db8b7511298d5b7784b40b092d9e9dcd3a627a5707e4b5e507931ab0d44eeebf" +checksum = "726535892e8eae7e70657b4c8ea93d26b8553afb1ce617caee529ef96d7dee6c" dependencies = [ "proc-macro2", "quote", @@ -262,15 +262,13 @@ dependencies = [ [[package]] name = "bindgen" -version = "0.60.1" +version = "0.61.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "062dddbc1ba4aca46de6338e2bf87771414c335f7b2f2036e8f3e9befebf88e6" +checksum = "8a022e58a142a46fea340d68012b9201c094e93ec3d033a944a24f8fd4a4f09a" dependencies = [ "bitflags", "cexpr", "clang-sys", - "clap", - "env_logger", "lazy_static", "lazycell", "log", @@ -280,6 +278,7 @@ dependencies = [ "regex", "rustc-hash", "shlex", + "syn", "which", ] @@ -327,13 +326,14 @@ checksum = "5988cb1d626264ac94100be357308f29ff7cbdd3b36bda27f450a4ee3f713426" [[package]] name = "bstr" -version = "0.2.17" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba3569f383e8f1598449f1a423e72e99569137b47740b1da11ef19af3d5c3223" +checksum = "fca0852af221f458706eb0725c03e4ed6c46af9ac98e6a689d5e634215d594dd" dependencies = [ - "lazy_static", "memchr", + "once_cell", "regex-automata", + "serde", ] [[package]] @@ -449,14 +449,24 @@ name = "clap" version = "3.2.22" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "86447ad904c7fb335a790c9d7fe3d0d971dc523b8ccd1561a520de9a85302750" +dependencies = [ + "bitflags", + "clap_lex 0.2.4", + "indexmap", + "textwrap", +] + +[[package]] +name = "clap" +version = "4.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6bf8832993da70a4c6d13c581f4463c2bdda27b9bf1c5498dc4365543abe6d6f" dependencies = [ "atty", "bitflags", - "clap_lex", - "indexmap", + "clap_lex 0.3.0", "strsim", "termcolor", - "textwrap", ] [[package]] @@ -468,6 +478,15 @@ dependencies = [ "os_str_bytes", ] +[[package]] +name = "clap_lex" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d4198f73e42b4936b35b5bb248d81d2b595ecb170da0bac7655c54eedfa8da8" +dependencies = [ + "os_str_bytes", +] + [[package]] name = "close_fds" version = "0.3.2" @@ -525,7 +544,7 @@ version = "0.1.0" dependencies = [ "anyhow", "chrono", - "clap", + "clap 4.0.15", "env_logger", "futures", "hyper", @@ -567,7 +586,7 @@ name = "control_plane" version = "0.1.0" dependencies = [ "anyhow", - "clap", + "clap 4.0.15", "comfy-table", "git-version", "nix 0.25.0", @@ -660,7 +679,7 @@ dependencies = [ "atty", "cast", "ciborium", - "clap", + "clap 3.2.22", "criterion-plot", "itertools", "lazy_static", @@ -728,7 +747,7 @@ dependencies = [ "autocfg", "cfg-if", "crossbeam-utils", - "memoffset", + "memoffset 0.6.5", "scopeguard", ] @@ -789,9 +808,9 @@ dependencies = [ [[package]] name = "cxx" -version = "1.0.78" +version = "1.0.79" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19f39818dcfc97d45b03953c1292efc4e80954e1583c4aa770bac1383e2310a4" +checksum = "3f83d0ebf42c6eafb8d7c52f7e5f2d3003b89c7aa4fd2b79229209459a849af8" dependencies = [ "cc", "cxxbridge-flags", @@ -801,9 +820,9 @@ dependencies = [ [[package]] name = "cxx-build" -version = "1.0.78" +version = "1.0.79" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e580d70777c116df50c390d1211993f62d40302881e54d4b79727acb83d0199" +checksum = "07d050484b55975889284352b0ffc2ecbda25c0c55978017c132b29ba0818a86" dependencies = [ "cc", "codespan-reporting", @@ -816,15 +835,15 @@ dependencies = [ [[package]] name = "cxxbridge-flags" -version = "1.0.78" +version = "1.0.79" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56a46460b88d1cec95112c8c363f0e2c39afdb237f60583b0b36343bf627ea9c" +checksum = "99d2199b00553eda8012dfec8d3b1c75fce747cf27c169a270b3b99e3448ab78" [[package]] name = "cxxbridge-macro" -version = "1.0.78" +version = "1.0.79" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "747b608fecf06b0d72d440f27acc99288207324b793be2c17991839f3d4995ea" +checksum = "dcb67a6de1f602736dd7eaead0080cf3435df806c61b24b13328db128c58868f" dependencies = [ "proc-macro2", "quote", @@ -888,14 +907,14 @@ version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d6ee87af31d84ef885378aebca32be3d682b0e0dc119d5b4860a2c5bb5046730" dependencies = [ - "uuid", + "uuid 0.8.2", ] [[package]] name = "der-parser" -version = "7.0.0" +version = "8.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fe398ac75057914d7d07307bf67dc7f3f574a26783b4fc7805a20ffa9f506e82" +checksum = "42d4bc9b0db0a0df9ae64634ac5bdefb7afcb534e182275ca0beadbe486701c1" dependencies = [ "asn1-rs", "displaydoc", @@ -1204,6 +1223,12 @@ version = "0.3.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a6508c467c73851293f390476d4491cf4d227dbabcd4170f3bb6044959b294f1" +[[package]] +name = "futures-timer" +version = "3.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e64b03909df88034c26dc1547e8970b91f98bdb65165d6a4e9110d94263dbb2c" + [[package]] name = "futures-util" version = "0.3.24" @@ -1516,9 +1541,9 @@ dependencies = [ [[package]] name = "iana-time-zone-haiku" -version = "0.1.0" +version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fde6edd6cef363e9359ed3c98ba64590ba9eecba2293eb5a723ab32aee8926aa" +checksum = "0703ae284fc167426161c2e3f1da3ea71d94b21bedbcc9494e92b28e334e3dca" dependencies = [ "cxx", "cxx-build", @@ -1644,9 +1669,9 @@ dependencies = [ [[package]] name = "kqueue" -version = "1.0.6" +version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d6112e8f37b59803ac47a42d14f1f3a59bbf72fc6857ffc5be455e28a691f8e" +checksum = "2c8fc60ba15bf51257aa9807a48a61013db043fcf3a78cb0d916e8e396dcad98" dependencies = [ "kqueue-sys", "libc", @@ -1790,6 +1815,15 @@ dependencies = [ "autocfg", ] +[[package]] +name = "memoffset" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5de893c32cde5f383baa4c04c5d6dbdd735cfd4a794b0debdb2bb1b421da5ff4" +dependencies = [ + "autocfg", +] + [[package]] name = "metrics" version = "0.1.0" @@ -1882,7 +1916,7 @@ dependencies = [ "cc", "cfg-if", "libc", - "memoffset", + "memoffset 0.6.5", ] [[package]] @@ -1895,7 +1929,7 @@ dependencies = [ "bitflags", "cfg-if", "libc", - "memoffset", + "memoffset 0.6.5", "pin-utils", ] @@ -2008,9 +2042,9 @@ dependencies = [ [[package]] name = "oid-registry" -version = "0.4.0" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38e20717fa0541f39bd146692035c37bedfa532b3e5071b35761082407546b2a" +checksum = "7d4bda43fd1b844cbc6e6e54b5444e2b1bc7838bce59ad205902cccbb26d6761" dependencies = [ "asn1-rs", ] @@ -2101,7 +2135,7 @@ dependencies = [ "byteorder", "bytes", "chrono", - "clap", + "clap 4.0.15", "close_fds", "const_format", "crc32c", @@ -2377,7 +2411,7 @@ dependencies = [ "env_logger", "hex", "log", - "memoffset", + "memoffset 0.7.1", "once_cell", "postgres", "rand", @@ -2416,9 +2450,9 @@ checksum = "eb9f9e6e233e5c4a35559a617bf40a4ec447db2e84c20b55a6f83167b7e57872" [[package]] name = "prettyplease" -version = "0.1.20" +version = "0.1.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83fead41e178796ef8274dc612a7d8ce4c7e10ca35cd2c5b5ad24cac63aeb6c0" +checksum = "c142c0e46b57171fe0c528bee8c5b7569e80f0c17e377cd0e30ea57dbc11bb51" dependencies = [ "proc-macro2", "syn", @@ -2432,9 +2466,9 @@ checksum = "dbf0c48bc1d91375ae5c3cd81e3722dff1abcf81a30960240640d223f59fe0e5" [[package]] name = "proc-macro2" -version = "1.0.46" +version = "1.0.47" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94e2ef8dbfc347b10c094890f778ee2e36ca9bb4262e86dc99cd217e35f3470b" +checksum = "5ea3d908b0e36316caf9e9e2c4625cdde190a7e6f440d794667ed17a1855e725" dependencies = [ "unicode-ident", ] @@ -2533,7 +2567,7 @@ dependencies = [ "base64", "bstr", "bytes", - "clap", + "clap 4.0.15", "futures", "git-version", "hashbrown", @@ -2567,7 +2601,7 @@ dependencies = [ "tracing-subscriber", "url", "utils", - "uuid", + "uuid 1.2.1", "workspace_hack", "x509-parser", ] @@ -2656,13 +2690,13 @@ dependencies = [ [[package]] name = "rcgen" -version = "0.8.14" +version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5911d1403f4143c9d56a702069d593e8d0f3fab880a85e103604d0893ea31ba7" +checksum = "ffbe84efe2f38dea12e9bfc1f65377fdf03e53a18cb3b995faedf7934c7e785b" dependencies = [ - "chrono", "pem", "ring", + "time 0.3.15", "yasna", ] @@ -2852,9 +2886,21 @@ dependencies = [ [[package]] name = "rstest" -version = "0.12.0" +version = "0.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d912f35156a3f99a66ee3e11ac2e0b3f34ac85a07e05263d05a7e2c8810d616f" +checksum = "e9c9dc66cc29792b663ffb5269be669f1613664e69ad56441fdb895c2347b930" +dependencies = [ + "futures", + "futures-timer", + "rstest_macros", + "rustc_version 0.4.0", +] + +[[package]] +name = "rstest_macros" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5015e68a0685a95ade3eee617ff7101ab6a3fc689203101ca16ebc16f2b89c66" dependencies = [ "cfg-if", "proc-macro2", @@ -3034,7 +3080,7 @@ dependencies = [ "async-trait", "byteorder", "bytes", - "clap", + "clap 4.0.15", "const_format", "crc32c", "daemonize", @@ -3424,7 +3470,7 @@ dependencies = [ "debugid", "memmap2", "stable_deref_trait", - "uuid", + "uuid 0.8.2", ] [[package]] @@ -4010,6 +4056,12 @@ name = "uuid" version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bc5cf98d8186244414c848017f0e2676b3fcb46807f6668a97dfe67359a3c4b7" + +[[package]] +name = "uuid" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "feb41e78f93363bb2df8b0e86a2ca30eed7806ea16ea0c790d757cf93f79be83" dependencies = [ "getrandom", "serde", @@ -4059,7 +4111,7 @@ name = "wal_craft" version = "0.1.0" dependencies = [ "anyhow", - "clap", + "clap 4.0.15", "env_logger", "log", "once_cell", @@ -4299,7 +4351,7 @@ dependencies = [ "anyhow", "bytes", "chrono", - "clap", + "clap 4.0.15", "crossbeam-utils", "either", "fail", @@ -4307,6 +4359,7 @@ dependencies = [ "indexmap", "libc", "log", + "memchr", "nom", "num-bigint", "num-integer", @@ -4324,14 +4377,13 @@ dependencies = [ "tokio-util", "tracing", "tracing-core", - "uuid", ] [[package]] name = "x509-parser" -version = "0.13.2" +version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9fb9bace5b5589ffead1afb76e43e34cff39cd0f3ce7e170ae0c29e53b88eb1c" +checksum = "e0ecbeb7b67ce215e40e3cc7f2ff902f94a223acf44995934763467e7b1febc8" dependencies = [ "asn1-rs", "base64", @@ -4362,11 +4414,11 @@ checksum = "d2d7d3948613f75c98fd9328cfdcc45acc4d360655289d0a7d4ec931392200a3" [[package]] name = "yasna" -version = "0.4.0" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e262a29d0e61ccf2b6190d7050d4b237535fc76ce4c1210d9caa316f71dffa75" +checksum = "346d34a236c9d3e5f3b9b74563f238f955bbd05fa0b8b4efa53c130c43982f4c" dependencies = [ - "chrono", + "time 0.3.15", ] [[package]] diff --git a/compute_tools/Cargo.toml b/compute_tools/Cargo.toml index 43cf7ae2dd..d6f8fae34c 100644 --- a/compute_tools/Cargo.toml +++ b/compute_tools/Cargo.toml @@ -6,7 +6,7 @@ edition = "2021" [dependencies] anyhow = "1.0" chrono = "0.4" -clap = "3.0" +clap = "4.0" env_logger = "0.9" futures = "0.3.13" hyper = { version = "0.14", features = ["full"] } diff --git a/compute_tools/src/bin/compute_ctl.rs b/compute_tools/src/bin/compute_ctl.rs index fc5bbc5fd2..7786d7af9c 100644 --- a/compute_tools/src/bin/compute_ctl.rs +++ b/compute_tools/src/bin/compute_ctl.rs @@ -51,53 +51,19 @@ fn main() -> Result<()> { // TODO: re-use `utils::logging` later init_logger(DEFAULT_LOG_LEVEL)?; - // Env variable is set by `cargo` - let version: Option<&str> = option_env!("CARGO_PKG_VERSION"); - let matches = clap::App::new("compute_ctl") - .version(version.unwrap_or("unknown")) - .arg( - Arg::new("connstr") - .short('C') - .long("connstr") - .value_name("DATABASE_URL") - .required(true), - ) - .arg( - Arg::new("pgdata") - .short('D') - .long("pgdata") - .value_name("DATADIR") - .required(true), - ) - .arg( - Arg::new("pgbin") - .short('b') - .long("pgbin") - .value_name("POSTGRES_PATH"), - ) - .arg( - Arg::new("spec") - .short('s') - .long("spec") - .value_name("SPEC_JSON"), - ) - .arg( - Arg::new("spec-path") - .short('S') - .long("spec-path") - .value_name("SPEC_PATH"), - ) - .get_matches(); + let matches = cli().get_matches(); - let pgdata = matches.value_of("pgdata").expect("PGDATA path is required"); + let pgdata = matches + .get_one::("pgdata") + .expect("PGDATA path is required"); let connstr = matches - .value_of("connstr") + .get_one::("connstr") .expect("Postgres connection string is required"); - let spec = matches.value_of("spec"); - let spec_path = matches.value_of("spec-path"); + let spec = matches.get_one::("spec"); + let spec_path = matches.get_one::("spec-path"); // Try to use just 'postgres' if no path is provided - let pgbin = matches.value_of("pgbin").unwrap_or("postgres"); + let pgbin = matches.get_one::("pgbin").unwrap(); let spec: ComputeSpec = match spec { // First, try to get cluster spec from the cli argument @@ -173,3 +139,48 @@ fn main() -> Result<()> { } } } + +fn cli() -> clap::Command { + // Env variable is set by `cargo` + let version = option_env!("CARGO_PKG_VERSION").unwrap_or("unknown"); + clap::Command::new("compute_ctl") + .version(version) + .arg( + Arg::new("connstr") + .short('C') + .long("connstr") + .value_name("DATABASE_URL") + .required(true), + ) + .arg( + Arg::new("pgdata") + .short('D') + .long("pgdata") + .value_name("DATADIR") + .required(true), + ) + .arg( + Arg::new("pgbin") + .short('b') + .long("pgbin") + .default_value("postgres") + .value_name("POSTGRES_PATH"), + ) + .arg( + Arg::new("spec") + .short('s') + .long("spec") + .value_name("SPEC_JSON"), + ) + .arg( + Arg::new("spec-path") + .short('S') + .long("spec-path") + .value_name("SPEC_PATH"), + ) +} + +#[test] +fn verify_cli() { + cli().debug_assert() +} diff --git a/control_plane/Cargo.toml b/control_plane/Cargo.toml index 690b63613a..287385c709 100644 --- a/control_plane/Cargo.toml +++ b/control_plane/Cargo.toml @@ -4,7 +4,7 @@ version = "0.1.0" edition = "2021" [dependencies] -clap = "3.0" +clap = "4.0" comfy-table = "6.1" git-version = "0.3.5" tar = "0.4.38" diff --git a/control_plane/src/bin/neon_local.rs b/control_plane/src/bin/neon_local.rs index 08797fe907..70a2c97a9e 100644 --- a/control_plane/src/bin/neon_local.rs +++ b/control_plane/src/bin/neon_local.rs @@ -6,7 +6,7 @@ //! rely on `neon_local` to set up the environment for each test. //! use anyhow::{anyhow, bail, Context, Result}; -use clap::{App, AppSettings, Arg, ArgMatches}; +use clap::{value_parser, Arg, ArgAction, ArgMatches, Command}; use control_plane::compute::ComputeControlPlane; use control_plane::local_env::{EtcdBroker, LocalEnv}; use control_plane::safekeeper::SafekeeperNode; @@ -85,212 +85,7 @@ struct TimelineTreeEl { // * Providing CLI api to the pageserver // * TODO: export/import to/from usual postgres fn main() -> Result<()> { - let branch_name_arg = Arg::new("branch-name") - .long("branch-name") - .takes_value(true) - .help("Name of the branch to be created or used as an alias for other services") - .required(false); - - let pg_node_arg = Arg::new("node").help("Postgres node name").required(false); - - let safekeeper_id_arg = Arg::new("id").help("safekeeper id").required(false); - - let tenant_id_arg = Arg::new("tenant-id") - .long("tenant-id") - .help("Tenant id. Represented as a hexadecimal string 32 symbols length") - .takes_value(true) - .required(false); - - let timeline_id_arg = Arg::new("timeline-id") - .long("timeline-id") - .help("Timeline id. Represented as a hexadecimal string 32 symbols length") - .takes_value(true) - .required(false); - - let pg_version_arg = Arg::new("pg-version") - .long("pg-version") - .help("Postgres version to use for the initial tenant") - .required(false) - .takes_value(true) - .default_value(DEFAULT_PG_VERSION); - - let port_arg = Arg::new("port") - .long("port") - .required(false) - .value_name("port"); - - let stop_mode_arg = Arg::new("stop-mode") - .short('m') - .takes_value(true) - .possible_values(&["fast", "immediate"]) - .help("If 'immediate', don't flush repository data at shutdown") - .required(false) - .value_name("stop-mode"); - - let pageserver_config_args = Arg::new("pageserver-config-override") - .long("pageserver-config-override") - .takes_value(true) - .number_of_values(1) - .multiple_occurrences(true) - .help("Additional pageserver's configuration options or overrides, refer to pageserver's 'config-override' CLI parameter docs for more") - .required(false); - - let lsn_arg = Arg::new("lsn") - .long("lsn") - .help("Specify Lsn on the timeline to start from. By default, end of the timeline would be used.") - .takes_value(true) - .required(false); - - let matches = App::new("Neon CLI") - .setting(AppSettings::ArgRequiredElseHelp) - .version(GIT_VERSION) - .subcommand( - App::new("init") - .about("Initialize a new Neon repository") - .arg(pageserver_config_args.clone()) - .arg(timeline_id_arg.clone().help("Use a specific timeline id when creating a tenant and its initial timeline")) - .arg( - Arg::new("config") - .long("config") - .required(false) - .value_name("config"), - ) - .arg(pg_version_arg.clone()) - ) - .subcommand( - App::new("timeline") - .about("Manage timelines") - .subcommand(App::new("list") - .about("List all timelines, available to this pageserver") - .arg(tenant_id_arg.clone())) - .subcommand(App::new("branch") - .about("Create a new timeline, using another timeline as a base, copying its data") - .arg(tenant_id_arg.clone()) - .arg(branch_name_arg.clone()) - .arg(Arg::new("ancestor-branch-name").long("ancestor-branch-name").takes_value(true) - .help("Use last Lsn of another timeline (and its data) as base when creating the new timeline. The timeline gets resolved by its branch name.").required(false)) - .arg(Arg::new("ancestor-start-lsn").long("ancestor-start-lsn").takes_value(true) - .help("When using another timeline as base, use a specific Lsn in it instead of the latest one").required(false))) - .subcommand(App::new("create") - .about("Create a new blank timeline") - .arg(tenant_id_arg.clone()) - .arg(branch_name_arg.clone()) - .arg(pg_version_arg.clone()) - ) - .subcommand(App::new("import") - .about("Import timeline from basebackup directory") - .arg(tenant_id_arg.clone()) - .arg(timeline_id_arg.clone()) - .arg(Arg::new("node-name").long("node-name").takes_value(true) - .help("Name to assign to the imported timeline")) - .arg(Arg::new("base-tarfile").long("base-tarfile").takes_value(true) - .help("Basebackup tarfile to import")) - .arg(Arg::new("base-lsn").long("base-lsn").takes_value(true) - .help("Lsn the basebackup starts at")) - .arg(Arg::new("wal-tarfile").long("wal-tarfile").takes_value(true) - .help("Wal to add after base")) - .arg(Arg::new("end-lsn").long("end-lsn").takes_value(true) - .help("Lsn the basebackup ends at")) - .arg(pg_version_arg.clone()) - ) - ).subcommand( - App::new("tenant") - .setting(AppSettings::ArgRequiredElseHelp) - .about("Manage tenants") - .subcommand(App::new("list")) - .subcommand(App::new("create") - .arg(tenant_id_arg.clone()) - .arg(timeline_id_arg.clone().help("Use a specific timeline id when creating a tenant and its initial timeline")) - .arg(Arg::new("config").short('c').takes_value(true).multiple_occurrences(true).required(false)) - .arg(pg_version_arg.clone()) - ) - .subcommand(App::new("config") - .arg(tenant_id_arg.clone()) - .arg(Arg::new("config").short('c').takes_value(true).multiple_occurrences(true).required(false)) - ) - ) - .subcommand( - App::new("pageserver") - .setting(AppSettings::ArgRequiredElseHelp) - .about("Manage pageserver") - .subcommand(App::new("status")) - .subcommand(App::new("start").about("Start local pageserver").arg(pageserver_config_args.clone())) - .subcommand(App::new("stop").about("Stop local pageserver") - .arg(stop_mode_arg.clone())) - .subcommand(App::new("restart").about("Restart local pageserver").arg(pageserver_config_args.clone())) - ) - .subcommand( - App::new("safekeeper") - .setting(AppSettings::ArgRequiredElseHelp) - .about("Manage safekeepers") - .subcommand(App::new("start") - .about("Start local safekeeper") - .arg(safekeeper_id_arg.clone()) - ) - .subcommand(App::new("stop") - .about("Stop local safekeeper") - .arg(safekeeper_id_arg.clone()) - .arg(stop_mode_arg.clone()) - ) - .subcommand(App::new("restart") - .about("Restart local safekeeper") - .arg(safekeeper_id_arg.clone()) - .arg(stop_mode_arg.clone()) - ) - ) - .subcommand( - App::new("pg") - .setting(AppSettings::ArgRequiredElseHelp) - .about("Manage postgres instances") - .subcommand(App::new("list").arg(tenant_id_arg.clone())) - .subcommand(App::new("create") - .about("Create a postgres compute node") - .arg(pg_node_arg.clone()) - .arg(branch_name_arg.clone()) - .arg(tenant_id_arg.clone()) - .arg(lsn_arg.clone()) - .arg(port_arg.clone()) - .arg( - Arg::new("config-only") - .help("Don't do basebackup, create compute node with only config files") - .long("config-only") - .required(false)) - .arg(pg_version_arg.clone()) - ) - .subcommand(App::new("start") - .about("Start a postgres compute node.\n This command actually creates new node from scratch, but preserves existing config files") - .arg(pg_node_arg.clone()) - .arg(tenant_id_arg.clone()) - .arg(branch_name_arg.clone()) - .arg(timeline_id_arg.clone()) - .arg(lsn_arg.clone()) - .arg(port_arg.clone()) - .arg(pg_version_arg.clone()) - ) - .subcommand( - App::new("stop") - .arg(pg_node_arg.clone()) - .arg(tenant_id_arg.clone()) - .arg( - Arg::new("destroy") - .help("Also delete data directory (now optional, should be default in future)") - .long("destroy") - .required(false) - ) - ) - - ) - .subcommand( - App::new("start") - .about("Start page server and safekeepers") - .arg(pageserver_config_args) - ) - .subcommand( - App::new("stop") - .about("Stop page server and safekeepers") - .arg(stop_mode_arg.clone()) - ) - .get_matches(); + let matches = cli().get_matches(); let (sub_name, sub_args) = match matches.subcommand() { Some(subcommand_data) => subcommand_data, @@ -475,16 +270,16 @@ fn get_tenant_id(sub_match: &ArgMatches, env: &local_env::LocalEnv) -> anyhow::R fn parse_tenant_id(sub_match: &ArgMatches) -> anyhow::Result> { sub_match - .value_of("tenant-id") - .map(TenantId::from_str) + .get_one::("tenant-id") + .map(|tenant_id| TenantId::from_str(tenant_id)) .transpose() .context("Failed to parse tenant id from the argument string") } fn parse_timeline_id(sub_match: &ArgMatches) -> anyhow::Result> { sub_match - .value_of("timeline-id") - .map(TimelineId::from_str) + .get_one::("timeline-id") + .map(|timeline_id| TimelineId::from_str(timeline_id)) .transpose() .context("Failed to parse timeline id from the argument string") } @@ -493,19 +288,22 @@ fn handle_init(init_match: &ArgMatches) -> anyhow::Result { let initial_timeline_id_arg = parse_timeline_id(init_match)?; // Create config file - let toml_file: String = if let Some(config_path) = init_match.value_of("config") { + let toml_file: String = if let Some(config_path) = init_match.get_one::("config") { // load and parse the file - std::fs::read_to_string(std::path::Path::new(config_path)) - .with_context(|| format!("Could not read configuration file '{config_path}'"))? + std::fs::read_to_string(config_path).with_context(|| { + format!( + "Could not read configuration file '{}'", + config_path.display() + ) + })? } else { // Built-in default config default_conf(&EtcdBroker::locate_etcd()?) }; let pg_version = init_match - .value_of("pg-version") - .unwrap() - .parse::() + .get_one::("pg-version") + .copied() .context("Failed to parse postgres version from the argument string")?; let mut env = @@ -541,9 +339,10 @@ fn handle_init(init_match: &ArgMatches) -> anyhow::Result { fn pageserver_config_overrides(init_match: &ArgMatches) -> Vec<&str> { init_match - .values_of("pageserver-config-override") + .get_many::("pageserver-config-override") .into_iter() .flatten() + .map(|s| s.as_str()) .collect() } @@ -558,7 +357,7 @@ fn handle_tenant(tenant_match: &ArgMatches, env: &mut local_env::LocalEnv) -> an Some(("create", create_match)) => { let initial_tenant_id = parse_tenant_id(create_match)?; let tenant_conf: HashMap<_, _> = create_match - .values_of("config") + .get_many::("config") .map(|vals| vals.flat_map(|c| c.split_once(':')).collect()) .unwrap_or_default(); let new_tenant_id = pageserver.tenant_create(initial_tenant_id, tenant_conf)?; @@ -567,9 +366,8 @@ fn handle_tenant(tenant_match: &ArgMatches, env: &mut local_env::LocalEnv) -> an // Create an initial timeline for the new tenant let new_timeline_id = parse_timeline_id(create_match)?; let pg_version = create_match - .value_of("pg-version") - .unwrap() - .parse::() + .get_one::("pg-version") + .copied() .context("Failed to parse postgres version from the argument string")?; let timeline_info = pageserver.timeline_create( @@ -595,7 +393,7 @@ fn handle_tenant(tenant_match: &ArgMatches, env: &mut local_env::LocalEnv) -> an Some(("config", create_match)) => { let tenant_id = get_tenant_id(create_match, env)?; let tenant_conf: HashMap<_, _> = create_match - .values_of("config") + .get_many::("config") .map(|vals| vals.flat_map(|c| c.split_once(':')).collect()) .unwrap_or_default(); @@ -622,13 +420,12 @@ fn handle_timeline(timeline_match: &ArgMatches, env: &mut local_env::LocalEnv) - Some(("create", create_match)) => { let tenant_id = get_tenant_id(create_match, env)?; let new_branch_name = create_match - .value_of("branch-name") + .get_one::("branch-name") .ok_or_else(|| anyhow!("No branch name provided"))?; let pg_version = create_match - .value_of("pg-version") - .unwrap() - .parse::() + .get_one::("pg-version") + .copied() .context("Failed to parse postgres version from the argument string")?; let timeline_info = @@ -647,35 +444,32 @@ fn handle_timeline(timeline_match: &ArgMatches, env: &mut local_env::LocalEnv) - let tenant_id = get_tenant_id(import_match, env)?; let timeline_id = parse_timeline_id(import_match)?.expect("No timeline id provided"); let name = import_match - .value_of("node-name") + .get_one::("node-name") .ok_or_else(|| anyhow!("No node name provided"))?; // Parse base inputs let base_tarfile = import_match - .value_of("base-tarfile") - .map(|s| PathBuf::from_str(s).unwrap()) - .ok_or_else(|| anyhow!("No base-tarfile provided"))?; + .get_one::("base-tarfile") + .ok_or_else(|| anyhow!("No base-tarfile provided"))? + .to_owned(); let base_lsn = Lsn::from_str( import_match - .value_of("base-lsn") + .get_one::("base-lsn") .ok_or_else(|| anyhow!("No base-lsn provided"))?, )?; let base = (base_lsn, base_tarfile); // Parse pg_wal inputs - let wal_tarfile = import_match - .value_of("wal-tarfile") - .map(|s| PathBuf::from_str(s).unwrap()); + let wal_tarfile = import_match.get_one::("wal-tarfile").cloned(); let end_lsn = import_match - .value_of("end-lsn") + .get_one::("end-lsn") .map(|s| Lsn::from_str(s).unwrap()); // TODO validate both or none are provided let pg_wal = end_lsn.zip(wal_tarfile); let pg_version = import_match - .value_of("pg-version") - .unwrap() - .parse::() + .get_one::("pg-version") + .copied() .context("Failed to parse postgres version from the argument string")?; let mut cplane = ComputeControlPlane::load(env.clone())?; @@ -690,10 +484,11 @@ fn handle_timeline(timeline_match: &ArgMatches, env: &mut local_env::LocalEnv) - Some(("branch", branch_match)) => { let tenant_id = get_tenant_id(branch_match, env)?; let new_branch_name = branch_match - .value_of("branch-name") + .get_one::("branch-name") .ok_or_else(|| anyhow!("No branch name provided"))?; let ancestor_branch_name = branch_match - .value_of("ancestor-branch-name") + .get_one::("ancestor-branch-name") + .map(|s| s.as_str()) .unwrap_or(DEFAULT_BRANCH_NAME); let ancestor_timeline_id = env .get_branch_timeline_id(ancestor_branch_name, tenant_id) @@ -702,8 +497,8 @@ fn handle_timeline(timeline_match: &ArgMatches, env: &mut local_env::LocalEnv) - })?; let start_lsn = branch_match - .value_of("ancestor-start-lsn") - .map(Lsn::from_str) + .get_one::("ancestor-start-lsn") + .map(|lsn_str| Lsn::from_str(lsn_str)) .transpose() .context("Failed to parse ancestor start Lsn from the request")?; let timeline_info = pageserver.timeline_create( @@ -804,45 +599,39 @@ fn handle_pg(pg_match: &ArgMatches, env: &local_env::LocalEnv) -> Result<()> { } "create" => { let branch_name = sub_args - .value_of("branch-name") + .get_one::("branch-name") + .map(|s| s.as_str()) .unwrap_or(DEFAULT_BRANCH_NAME); let node_name = sub_args - .value_of("node") - .map(ToString::to_string) - .unwrap_or_else(|| format!("{}_node", branch_name)); + .get_one::("node") + .map(|node_name| node_name.to_string()) + .unwrap_or_else(|| format!("{branch_name}_node")); let lsn = sub_args - .value_of("lsn") - .map(Lsn::from_str) + .get_one::("lsn") + .map(|lsn_str| Lsn::from_str(lsn_str)) .transpose() .context("Failed to parse Lsn from the request")?; let timeline_id = env .get_branch_timeline_id(branch_name, tenant_id) - .ok_or_else(|| anyhow!("Found no timeline id for branch name '{}'", branch_name))?; + .ok_or_else(|| anyhow!("Found no timeline id for branch name '{branch_name}'"))?; - let port: Option = match sub_args.value_of("port") { - Some(p) => Some(p.parse()?), - None => None, - }; + let port: Option = sub_args.get_one::("port").copied(); let pg_version = sub_args - .value_of("pg-version") - .unwrap() - .parse::() + .get_one::("pg-version") + .copied() .context("Failed to parse postgres version from the argument string")?; cplane.new_node(tenant_id, &node_name, timeline_id, lsn, port, pg_version)?; } "start" => { - let port: Option = match sub_args.value_of("port") { - Some(p) => Some(p.parse()?), - None => None, - }; + let port: Option = sub_args.get_one::("port").copied(); let node_name = sub_args - .value_of("node") + .get_one::("node") .ok_or_else(|| anyhow!("No node name was provided to start"))?; - let node = cplane.nodes.get(&(tenant_id, node_name.to_owned())); + let node = cplane.nodes.get(&(tenant_id, node_name.to_string())); let auth_token = if matches!(env.pageserver.auth_type, AuthType::NeonJWT) { let claims = Claims::new(Some(tenant_id), Scope::Tenant); @@ -853,36 +642,33 @@ fn handle_pg(pg_match: &ArgMatches, env: &local_env::LocalEnv) -> Result<()> { }; if let Some(node) = node { - println!("Starting existing postgres {}...", node_name); + println!("Starting existing postgres {node_name}..."); node.start(&auth_token)?; } else { let branch_name = sub_args - .value_of("branch-name") + .get_one::("branch-name") + .map(|s| s.as_str()) .unwrap_or(DEFAULT_BRANCH_NAME); let timeline_id = env .get_branch_timeline_id(branch_name, tenant_id) .ok_or_else(|| { - anyhow!("Found no timeline id for branch name '{}'", branch_name) + anyhow!("Found no timeline id for branch name '{branch_name}'") })?; let lsn = sub_args - .value_of("lsn") - .map(Lsn::from_str) + .get_one::("lsn") + .map(|lsn_str| Lsn::from_str(lsn_str)) .transpose() .context("Failed to parse Lsn from the request")?; let pg_version = sub_args - .value_of("pg-version") - .unwrap() - .parse::() - .context("Failed to parse postgres version from the argument string")?; + .get_one::("pg-version") + .copied() + .context("Failed to `pg-version` from the argument string")?; // when used with custom port this results in non obvious behaviour // port is remembered from first start command, i e // start --port X // stop // start <-- will also use port X even without explicit port argument - println!( - "Starting new postgres (v{}) {} on timeline {} ...", - pg_version, node_name, timeline_id - ); + println!("Starting new postgres (v{pg_version}) {node_name} on timeline {timeline_id} ..."); let node = cplane.new_node(tenant_id, node_name, timeline_id, lsn, port, pg_version)?; @@ -891,18 +677,18 @@ fn handle_pg(pg_match: &ArgMatches, env: &local_env::LocalEnv) -> Result<()> { } "stop" => { let node_name = sub_args - .value_of("node") + .get_one::("node") .ok_or_else(|| anyhow!("No node name was provided to stop"))?; - let destroy = sub_args.is_present("destroy"); + let destroy = sub_args.get_flag("destroy"); let node = cplane .nodes - .get(&(tenant_id, node_name.to_owned())) - .with_context(|| format!("postgres {} is not found", node_name))?; + .get(&(tenant_id, node_name.to_string())) + .with_context(|| format!("postgres {node_name} is not found"))?; node.stop(destroy)?; } - _ => bail!("Unexpected pg subcommand '{}'", sub_name), + _ => bail!("Unexpected pg subcommand '{sub_name}'"), } Ok(()) @@ -920,7 +706,10 @@ fn handle_pageserver(sub_match: &ArgMatches, env: &local_env::LocalEnv) -> Resul } Some(("stop", stop_match)) => { - let immediate = stop_match.value_of("stop-mode") == Some("immediate"); + let immediate = stop_match + .get_one::("stop-mode") + .map(|s| s.as_str()) + == Some("immediate"); if let Err(e) = pageserver.stop(immediate) { eprintln!("pageserver stop failed: {}", e); @@ -970,7 +759,7 @@ fn handle_safekeeper(sub_match: &ArgMatches, env: &local_env::LocalEnv) -> Resul }; // All the commands take an optional safekeeper name argument - let sk_id = if let Some(id_str) = sub_args.value_of("id") { + let sk_id = if let Some(id_str) = sub_args.get_one::("id") { NodeId(id_str.parse().context("while parsing safekeeper id")?) } else { DEFAULT_SAFEKEEPER_ID @@ -986,7 +775,8 @@ fn handle_safekeeper(sub_match: &ArgMatches, env: &local_env::LocalEnv) -> Resul } "stop" => { - let immediate = sub_args.value_of("stop-mode") == Some("immediate"); + let immediate = + sub_args.get_one::("stop-mode").map(|s| s.as_str()) == Some("immediate"); if let Err(e) = safekeeper.stop(immediate) { eprintln!("safekeeper stop failed: {}", e); @@ -995,7 +785,8 @@ fn handle_safekeeper(sub_match: &ArgMatches, env: &local_env::LocalEnv) -> Resul } "restart" => { - let immediate = sub_args.value_of("stop-mode") == Some("immediate"); + let immediate = + sub_args.get_one::("stop-mode").map(|s| s.as_str()) == Some("immediate"); if let Err(e) = safekeeper.stop(immediate) { eprintln!("safekeeper stop failed: {}", e); @@ -1039,7 +830,8 @@ fn handle_start_all(sub_match: &ArgMatches, env: &local_env::LocalEnv) -> anyhow } fn handle_stop_all(sub_match: &ArgMatches, env: &local_env::LocalEnv) -> Result<()> { - let immediate = sub_match.value_of("stop-mode") == Some("immediate"); + let immediate = + sub_match.get_one::("stop-mode").map(|s| s.as_str()) == Some("immediate"); let pageserver = PageServerNode::from_env(env); @@ -1072,3 +864,219 @@ fn try_stop_etcd_process(env: &local_env::LocalEnv) { eprintln!("etcd stop failed: {e}"); } } + +fn cli() -> Command { + let branch_name_arg = Arg::new("branch-name") + .long("branch-name") + .help("Name of the branch to be created or used as an alias for other services") + .required(false); + + let pg_node_arg = Arg::new("node").help("Postgres node name").required(false); + + let safekeeper_id_arg = Arg::new("id").help("safekeeper id").required(false); + + let tenant_id_arg = Arg::new("tenant-id") + .long("tenant-id") + .help("Tenant id. Represented as a hexadecimal string 32 symbols length") + .required(false); + + let timeline_id_arg = Arg::new("timeline-id") + .long("timeline-id") + .help("Timeline id. Represented as a hexadecimal string 32 symbols length") + .required(false); + + let pg_version_arg = Arg::new("pg-version") + .long("pg-version") + .help("Postgres version to use for the initial tenant") + .required(false) + .value_parser(value_parser!(u32)) + .default_value(DEFAULT_PG_VERSION); + + let port_arg = Arg::new("port") + .long("port") + .required(false) + .value_parser(value_parser!(u16)) + .value_name("port"); + + let stop_mode_arg = Arg::new("stop-mode") + .short('m') + .value_parser(["fast", "immediate"]) + .help("If 'immediate', don't flush repository data at shutdown") + .required(false) + .value_name("stop-mode"); + + let pageserver_config_args = Arg::new("pageserver-config-override") + .long("pageserver-config-override") + .num_args(1) + .action(ArgAction::Append) + .help("Additional pageserver's configuration options or overrides, refer to pageserver's 'config-override' CLI parameter docs for more") + .required(false); + + let lsn_arg = Arg::new("lsn") + .long("lsn") + .help("Specify Lsn on the timeline to start from. By default, end of the timeline would be used.") + .required(false); + + Command::new("Neon CLI") + .arg_required_else_help(true) + .version(GIT_VERSION) + .subcommand( + Command::new("init") + .about("Initialize a new Neon repository") + .arg(pageserver_config_args.clone()) + .arg(timeline_id_arg.clone().help("Use a specific timeline id when creating a tenant and its initial timeline")) + .arg( + Arg::new("config") + .long("config") + .required(false) + .value_parser(value_parser!(PathBuf)) + .value_name("config"), + ) + .arg(pg_version_arg.clone()) + ) + .subcommand( + Command::new("timeline") + .about("Manage timelines") + .subcommand(Command::new("list") + .about("List all timelines, available to this pageserver") + .arg(tenant_id_arg.clone())) + .subcommand(Command::new("branch") + .about("Create a new timeline, using another timeline as a base, copying its data") + .arg(tenant_id_arg.clone()) + .arg(branch_name_arg.clone()) + .arg(Arg::new("ancestor-branch-name").long("ancestor-branch-name") + .help("Use last Lsn of another timeline (and its data) as base when creating the new timeline. The timeline gets resolved by its branch name.").required(false)) + .arg(Arg::new("ancestor-start-lsn").long("ancestor-start-lsn") + .help("When using another timeline as base, use a specific Lsn in it instead of the latest one").required(false))) + .subcommand(Command::new("create") + .about("Create a new blank timeline") + .arg(tenant_id_arg.clone()) + .arg(branch_name_arg.clone()) + .arg(pg_version_arg.clone()) + ) + .subcommand(Command::new("import") + .about("Import timeline from basebackup directory") + .arg(tenant_id_arg.clone()) + .arg(timeline_id_arg.clone()) + .arg(Arg::new("node-name").long("node-name") + .help("Name to assign to the imported timeline")) + .arg(Arg::new("base-tarfile") + .long("base-tarfile") + .value_parser(value_parser!(PathBuf)) + .help("Basebackup tarfile to import") + ) + .arg(Arg::new("base-lsn").long("base-lsn") + .help("Lsn the basebackup starts at")) + .arg(Arg::new("wal-tarfile") + .long("wal-tarfile") + .value_parser(value_parser!(PathBuf)) + .help("Wal to add after base") + ) + .arg(Arg::new("end-lsn").long("end-lsn") + .help("Lsn the basebackup ends at")) + .arg(pg_version_arg.clone()) + ) + ).subcommand( + Command::new("tenant") + .arg_required_else_help(true) + .about("Manage tenants") + .subcommand(Command::new("list")) + .subcommand(Command::new("create") + .arg(tenant_id_arg.clone()) + .arg(timeline_id_arg.clone().help("Use a specific timeline id when creating a tenant and its initial timeline")) + .arg(Arg::new("config").short('c').num_args(1).action(ArgAction::Append).required(false)) + .arg(pg_version_arg.clone()) + ) + .subcommand(Command::new("config") + .arg(tenant_id_arg.clone()) + .arg(Arg::new("config").short('c').num_args(1).action(ArgAction::Append).required(false)) + ) + ) + .subcommand( + Command::new("pageserver") + .arg_required_else_help(true) + .about("Manage pageserver") + .subcommand(Command::new("status")) + .subcommand(Command::new("start").about("Start local pageserver").arg(pageserver_config_args.clone())) + .subcommand(Command::new("stop").about("Stop local pageserver") + .arg(stop_mode_arg.clone())) + .subcommand(Command::new("restart").about("Restart local pageserver").arg(pageserver_config_args.clone())) + ) + .subcommand( + Command::new("safekeeper") + .arg_required_else_help(true) + .about("Manage safekeepers") + .subcommand(Command::new("start") + .about("Start local safekeeper") + .arg(safekeeper_id_arg.clone()) + ) + .subcommand(Command::new("stop") + .about("Stop local safekeeper") + .arg(safekeeper_id_arg.clone()) + .arg(stop_mode_arg.clone()) + ) + .subcommand(Command::new("restart") + .about("Restart local safekeeper") + .arg(safekeeper_id_arg) + .arg(stop_mode_arg.clone()) + ) + ) + .subcommand( + Command::new("pg") + .arg_required_else_help(true) + .about("Manage postgres instances") + .subcommand(Command::new("list").arg(tenant_id_arg.clone())) + .subcommand(Command::new("create") + .about("Create a postgres compute node") + .arg(pg_node_arg.clone()) + .arg(branch_name_arg.clone()) + .arg(tenant_id_arg.clone()) + .arg(lsn_arg.clone()) + .arg(port_arg.clone()) + .arg( + Arg::new("config-only") + .help("Don't do basebackup, create compute node with only config files") + .long("config-only") + .required(false)) + .arg(pg_version_arg.clone()) + ) + .subcommand(Command::new("start") + .about("Start a postgres compute node.\n This command actually creates new node from scratch, but preserves existing config files") + .arg(pg_node_arg.clone()) + .arg(tenant_id_arg.clone()) + .arg(branch_name_arg) + .arg(timeline_id_arg) + .arg(lsn_arg) + .arg(port_arg) + .arg(pg_version_arg) + ) + .subcommand( + Command::new("stop") + .arg(pg_node_arg) + .arg(tenant_id_arg) + .arg( + Arg::new("destroy") + .help("Also delete data directory (now optional, should be default in future)") + .long("destroy") + .action(ArgAction::SetTrue) + .required(false) + ) + ) + + ) + .subcommand( + Command::new("start") + .about("Start page server and safekeepers") + .arg(pageserver_config_args) + ) + .subcommand( + Command::new("stop") + .about("Stop page server and safekeepers") + .arg(stop_mode_arg) + ) +} + +#[test] +fn verify_cli() { + cli().debug_assert(); +} diff --git a/libs/postgres_ffi/Cargo.toml b/libs/postgres_ffi/Cargo.toml index 60caca76b8..01ff6ab60e 100644 --- a/libs/postgres_ffi/Cargo.toml +++ b/libs/postgres_ffi/Cargo.toml @@ -13,7 +13,7 @@ crc32c = "0.6.0" hex = "0.4.3" once_cell = "1.13.0" log = "0.4.14" -memoffset = "0.6.2" +memoffset = "0.7" thiserror = "1.0" serde = { version = "1.0", features = ["derive"] } utils = { path = "../utils" } @@ -26,4 +26,4 @@ wal_craft = { path = "wal_craft" } [build-dependencies] anyhow = "1.0" -bindgen = "0.60.1" +bindgen = "0.61" diff --git a/libs/postgres_ffi/wal_craft/Cargo.toml b/libs/postgres_ffi/wal_craft/Cargo.toml index 88466737ed..4c35c5a650 100644 --- a/libs/postgres_ffi/wal_craft/Cargo.toml +++ b/libs/postgres_ffi/wal_craft/Cargo.toml @@ -7,7 +7,7 @@ edition = "2021" [dependencies] anyhow = "1.0" -clap = "3.0" +clap = "4.0" env_logger = "0.9" log = "0.4" once_cell = "1.13.0" diff --git a/libs/postgres_ffi/wal_craft/src/bin/wal_craft.rs b/libs/postgres_ffi/wal_craft/src/bin/wal_craft.rs index 9563298cd8..e87ca27e90 100644 --- a/libs/postgres_ffi/wal_craft/src/bin/wal_craft.rs +++ b/libs/postgres_ffi/wal_craft/src/bin/wal_craft.rs @@ -1,68 +1,19 @@ use anyhow::*; -use clap::{App, Arg, ArgMatches}; -use std::str::FromStr; +use clap::{value_parser, Arg, ArgMatches, Command}; +use std::{path::PathBuf, str::FromStr}; use wal_craft::*; fn main() -> Result<()> { env_logger::Builder::from_env(env_logger::Env::default().default_filter_or("wal_craft=info")) .init(); - let type_arg = &Arg::new("type") - .takes_value(true) - .help("Type of WAL to craft") - .possible_values([ - Simple::NAME, - LastWalRecordXlogSwitch::NAME, - LastWalRecordXlogSwitchEndsOnPageBoundary::NAME, - WalRecordCrossingSegmentFollowedBySmallOne::NAME, - LastWalRecordCrossingSegment::NAME, - ]) - .required(true); - let arg_matches = App::new("Postgres WAL crafter") - .about("Crafts Postgres databases with specific WAL properties") - .subcommand( - App::new("print-postgres-config") - .about("Print the configuration required for PostgreSQL server before running this script") - ) - .subcommand( - App::new("with-initdb") - .about("Craft WAL in a new data directory first initialized with initdb") - .arg(type_arg) - .arg( - Arg::new("datadir") - .takes_value(true) - .help("Data directory for the Postgres server") - .required(true) - ) - .arg( - Arg::new("pg-distrib-dir") - .long("pg-distrib-dir") - .takes_value(true) - .help("Directory with Postgres distributions (bin and lib directories, e.g. pg_install containing subpath `v14/bin/postgresql`)") - .default_value("/usr/local") - ) - .arg( - Arg::new("pg-version") - .long("pg-version") - .help("Postgres version to use for the initial tenant") - .required(true) - .takes_value(true) - ) - ) - .subcommand( - App::new("in-existing") - .about("Craft WAL at an existing recently created Postgres database. Note that server may append new WAL entries on shutdown.") - .arg(type_arg) - .arg( - Arg::new("connection") - .takes_value(true) - .help("Connection string to the Postgres database to populate") - .required(true) - ) - ) - .get_matches(); + let arg_matches = cli().get_matches(); let wal_craft = |arg_matches: &ArgMatches, client| { - let (intermediate_lsns, end_of_wal_lsn) = match arg_matches.value_of("type").unwrap() { + let (intermediate_lsns, end_of_wal_lsn) = match arg_matches + .get_one::("type") + .map(|s| s.as_str()) + .context("'type' is required")? + { Simple::NAME => Simple::craft(client)?, LastWalRecordXlogSwitch::NAME => LastWalRecordXlogSwitch::craft(client)?, LastWalRecordXlogSwitchEndsOnPageBoundary::NAME => { @@ -72,12 +23,12 @@ fn main() -> Result<()> { WalRecordCrossingSegmentFollowedBySmallOne::craft(client)? } LastWalRecordCrossingSegment::NAME => LastWalRecordCrossingSegment::craft(client)?, - a => panic!("Unknown --type argument: {}", a), + a => panic!("Unknown --type argument: {a}"), }; for lsn in intermediate_lsns { - println!("intermediate_lsn = {}", lsn); + println!("intermediate_lsn = {lsn}"); } - println!("end_of_wal = {}", end_of_wal_lsn); + println!("end_of_wal = {end_of_wal_lsn}"); Ok(()) }; @@ -85,20 +36,24 @@ fn main() -> Result<()> { None => panic!("No subcommand provided"), Some(("print-postgres-config", _)) => { for cfg in REQUIRED_POSTGRES_CONFIG.iter() { - println!("{}", cfg); + println!("{cfg}"); } Ok(()) } Some(("with-initdb", arg_matches)) => { let cfg = Conf { - pg_version: arg_matches - .value_of("pg-version") - .unwrap() - .parse::() - .context("Failed to parse postgres version from the argument string")?, - pg_distrib_dir: arg_matches.value_of("pg-distrib-dir").unwrap().into(), - datadir: arg_matches.value_of("datadir").unwrap().into(), + pg_version: *arg_matches + .get_one::("pg-version") + .context("'pg-version' is required")?, + pg_distrib_dir: arg_matches + .get_one::("pg-distrib-dir") + .context("'pg-distrib-dir' is required")? + .to_owned(), + datadir: arg_matches + .get_one::("datadir") + .context("'datadir' is required")? + .to_owned(), }; cfg.initdb()?; let srv = cfg.start_server()?; @@ -108,9 +63,77 @@ fn main() -> Result<()> { } Some(("in-existing", arg_matches)) => wal_craft( arg_matches, - &mut postgres::Config::from_str(arg_matches.value_of("connection").unwrap())? - .connect(postgres::NoTls)?, + &mut postgres::Config::from_str( + arg_matches + .get_one::("connection") + .context("'connection' is required")?, + ) + .context( + "'connection' argument value could not be parsed as a postgres connection string", + )? + .connect(postgres::NoTls)?, ), Some(_) => panic!("Unknown subcommand"), } } + +fn cli() -> Command { + let type_arg = &Arg::new("type") + .help("Type of WAL to craft") + .value_parser([ + Simple::NAME, + LastWalRecordXlogSwitch::NAME, + LastWalRecordXlogSwitchEndsOnPageBoundary::NAME, + WalRecordCrossingSegmentFollowedBySmallOne::NAME, + LastWalRecordCrossingSegment::NAME, + ]) + .required(true); + + Command::new("Postgres WAL crafter") + .about("Crafts Postgres databases with specific WAL properties") + .subcommand( + Command::new("print-postgres-config") + .about("Print the configuration required for PostgreSQL server before running this script") + ) + .subcommand( + Command::new("with-initdb") + .about("Craft WAL in a new data directory first initialized with initdb") + .arg(type_arg) + .arg( + Arg::new("datadir") + .help("Data directory for the Postgres server") + .value_parser(value_parser!(PathBuf)) + .required(true) + ) + .arg( + Arg::new("pg-distrib-dir") + .long("pg-distrib-dir") + .value_parser(value_parser!(PathBuf)) + .help("Directory with Postgres distributions (bin and lib directories, e.g. pg_install containing subpath `v14/bin/postgresql`)") + .default_value("/usr/local") + ) + .arg( + Arg::new("pg-version") + .long("pg-version") + .help("Postgres version to use for the initial tenant") + .value_parser(value_parser!(u32)) + .required(true) + + ) + ) + .subcommand( + Command::new("in-existing") + .about("Craft WAL at an existing recently created Postgres database. Note that server may append new WAL entries on shutdown.") + .arg(type_arg) + .arg( + Arg::new("connection") + .help("Connection string to the Postgres database to populate") + .required(true) + ) + ) +} + +#[test] +fn verify_cli() { + cli().debug_assert(); +} diff --git a/pageserver/Cargo.toml b/pageserver/Cargo.toml index ea0cf3f18a..ebdda5c6da 100644 --- a/pageserver/Cargo.toml +++ b/pageserver/Cargo.toml @@ -23,7 +23,7 @@ futures = "0.3.13" hex = "0.4.3" hyper = "0.14" itertools = "0.10.3" -clap = "3.0" +clap = { version = "4.0", features = ["string"] } daemonize = "0.4.1" tokio = { version = "1.17", features = ["process", "sync", "macros", "fs", "rt", "io-util", "time"] } tokio-util = { version = "0.7.3", features = ["io", "io-util"] } diff --git a/pageserver/src/bin/pageserver.rs b/pageserver/src/bin/pageserver.rs index 4cd82e37b1..12f594077e 100644 --- a/pageserver/src/bin/pageserver.rs +++ b/pageserver/src/bin/pageserver.rs @@ -6,7 +6,7 @@ use tracing::*; use anyhow::{anyhow, bail, Context, Result}; -use clap::{App, Arg}; +use clap::{Arg, ArgAction, Command}; use daemonize::Daemonize; use fail::FailScenario; @@ -51,57 +51,17 @@ fn version() -> String { } fn main() -> anyhow::Result<()> { - let arg_matches = App::new("Neon page server") - .about("Materializes WAL stream to pages and serves them to the postgres") - .version(&*version()) - .arg( + let arg_matches = cli().get_matches(); - Arg::new("daemonize") - .short('d') - .long("daemonize") - .takes_value(false) - .help("Run in the background"), - ) - .arg( - Arg::new("init") - .long("init") - .takes_value(false) - .help("Initialize pageserver with all given config overrides"), - ) - .arg( - Arg::new("workdir") - .short('D') - .long("workdir") - .takes_value(true) - .help("Working directory for the pageserver"), - ) - // See `settings.md` for more details on the extra configuration patameters pageserver can process - .arg( - Arg::new("config-override") - .short('c') - .takes_value(true) - .number_of_values(1) - .multiple_occurrences(true) - .help("Additional configuration overrides of the ones from the toml config file (or new ones to add there). - Any option has to be a valid toml document, example: `-c=\"foo='hey'\"` `-c=\"foo={value=1}\"`"), - ) - .arg(Arg::new("update-config").long("update-config").takes_value(false).help( - "Update the config file when started", - )) - .arg( - Arg::new("enabled-features") - .long("enabled-features") - .takes_value(false) - .help("Show enabled compile time features"), - ) - .get_matches(); - - if arg_matches.is_present("enabled-features") { + if arg_matches.get_flag("enabled-features") { println!("{{\"features\": {FEATURES:?} }}"); return Ok(()); } - let workdir = Path::new(arg_matches.value_of("workdir").unwrap_or(".neon")); + let workdir = arg_matches + .get_one::("workdir") + .map(Path::new) + .unwrap_or_else(|| Path::new(".neon")); let workdir = workdir .canonicalize() .with_context(|| format!("Error opening workdir '{}'", workdir.display()))?; @@ -115,7 +75,7 @@ fn main() -> anyhow::Result<()> { ) })?; - let daemonize = arg_matches.is_present("daemonize"); + let daemonize = arg_matches.get_flag("daemonize"); let conf = match initialize_config(&cfg_file_path, arg_matches, &workdir)? { ControlFlow::Continue(conf) => conf, @@ -153,8 +113,8 @@ fn initialize_config( arg_matches: clap::ArgMatches, workdir: &Path, ) -> anyhow::Result> { - let init = arg_matches.is_present("init"); - let update_config = init || arg_matches.is_present("update-config"); + let init = arg_matches.get_flag("init"); + let update_config = init || arg_matches.get_flag("update-config"); let (mut toml, config_file_exists) = if cfg_file_path.is_file() { if init { @@ -196,13 +156,10 @@ fn initialize_config( ) }; - if let Some(values) = arg_matches.values_of("config-override") { + if let Some(values) = arg_matches.get_many::("config-override") { for option_line in values { let doc = toml_edit::Document::from_str(option_line).with_context(|| { - format!( - "Option '{}' could not be parsed as a toml document", - option_line - ) + format!("Option '{option_line}' could not be parsed as a toml document") })?; for (key, item) in doc.iter() { @@ -244,7 +201,7 @@ fn start_pageserver(conf: &'static PageServerConf, daemonize: bool) -> Result<() // Initialize logger let log_file = logging::init(LOG_FILE_NAME, daemonize)?; - info!("version: {GIT_VERSION}"); + info!("version: {}", version()); // TODO: Check that it looks like a valid repository before going further @@ -385,3 +342,55 @@ fn start_pageserver(conf: &'static PageServerConf, daemonize: bool) -> Result<() } }) } + +fn cli() -> Command { + Command::new("Neon page server") + .about("Materializes WAL stream to pages and serves them to the postgres") + .version(version()) + .arg( + + Arg::new("daemonize") + .short('d') + .long("daemonize") + .action(ArgAction::SetTrue) + .help("Run in the background"), + ) + .arg( + Arg::new("init") + .long("init") + .action(ArgAction::SetTrue) + .help("Initialize pageserver with all given config overrides"), + ) + .arg( + Arg::new("workdir") + .short('D') + .long("workdir") + .help("Working directory for the pageserver"), + ) + // See `settings.md` for more details on the extra configuration patameters pageserver can process + .arg( + Arg::new("config-override") + .short('c') + .num_args(1) + .action(ArgAction::Append) + .help("Additional configuration overrides of the ones from the toml config file (or new ones to add there). \ + Any option has to be a valid toml document, example: `-c=\"foo='hey'\"` `-c=\"foo={value=1}\"`"), + ) + .arg( + Arg::new("update-config") + .long("update-config") + .action(ArgAction::SetTrue) + .help("Update the config file when started"), + ) + .arg( + Arg::new("enabled-features") + .long("enabled-features") + .action(ArgAction::SetTrue) + .help("Show enabled compile time features"), + ) +} + +#[test] +fn verify_cli() { + cli().debug_assert(); +} diff --git a/pageserver/src/bin/pageserver_binutils.rs b/pageserver/src/bin/pageserver_binutils.rs index ec7699f194..b1484ac45a 100644 --- a/pageserver/src/bin/pageserver_binutils.rs +++ b/pageserver/src/bin/pageserver_binutils.rs @@ -9,7 +9,7 @@ use std::{ }; use anyhow::Context; -use clap::{App, Arg}; +use clap::{value_parser, Arg, Command}; use pageserver::{ page_cache, @@ -24,40 +24,14 @@ project_git_version!(GIT_VERSION); const METADATA_SUBCOMMAND: &str = "metadata"; fn main() -> anyhow::Result<()> { - let arg_matches = App::new("Neon Pageserver binutils") - .about("Reads pageserver (and related) binary files management utility") - .version(GIT_VERSION) - .arg(Arg::new("path").help("Input file path").required(false)) - .subcommand( - App::new(METADATA_SUBCOMMAND) - .about("Read and update pageserver metadata file") - .arg( - Arg::new("metadata_path") - .help("Input metadata file path") - .required(false), - ) - .arg( - Arg::new("disk_consistent_lsn") - .long("disk_consistent_lsn") - .takes_value(true) - .help("Replace disk consistent Lsn"), - ) - .arg( - Arg::new("prev_record_lsn") - .long("prev_record_lsn") - .takes_value(true) - .help("Replace previous record Lsn"), - ), - ) - .get_matches(); + let arg_matches = cli().get_matches(); match arg_matches.subcommand() { Some((subcommand_name, subcommand_matches)) => { - let path = PathBuf::from( - subcommand_matches - .value_of("metadata_path") - .context("'metadata_path' argument is missing")?, - ); + let path = subcommand_matches + .get_one::("metadata_path") + .context("'metadata_path' argument is missing")? + .to_path_buf(); anyhow::ensure!( subcommand_name == METADATA_SUBCOMMAND, "Unknown subcommand {subcommand_name}" @@ -65,11 +39,10 @@ fn main() -> anyhow::Result<()> { handle_metadata(&path, subcommand_matches)?; } None => { - let path = PathBuf::from( - arg_matches - .value_of("path") - .context("'path' argument is missing")?, - ); + let path = arg_matches + .get_one::("path") + .context("'path' argument is missing")? + .to_path_buf(); println!( "No subcommand specified, attempting to guess the format for file {}", path.display() @@ -110,7 +83,7 @@ fn handle_metadata(path: &Path, arg_matches: &clap::ArgMatches) -> Result<(), an let mut meta = TimelineMetadata::from_bytes(&metadata_bytes)?; println!("Current metadata:\n{meta:?}"); let mut update_meta = false; - if let Some(disk_consistent_lsn) = arg_matches.value_of("disk_consistent_lsn") { + if let Some(disk_consistent_lsn) = arg_matches.get_one::("disk_consistent_lsn") { meta = TimelineMetadata::new( Lsn::from_str(disk_consistent_lsn)?, meta.prev_record_lsn(), @@ -122,7 +95,7 @@ fn handle_metadata(path: &Path, arg_matches: &clap::ArgMatches) -> Result<(), an ); update_meta = true; } - if let Some(prev_record_lsn) = arg_matches.value_of("prev_record_lsn") { + if let Some(prev_record_lsn) = arg_matches.get_one::("prev_record_lsn") { meta = TimelineMetadata::new( meta.disk_consistent_lsn(), Some(Lsn::from_str(prev_record_lsn)?), @@ -142,3 +115,40 @@ fn handle_metadata(path: &Path, arg_matches: &clap::ArgMatches) -> Result<(), an Ok(()) } + +fn cli() -> Command { + Command::new("Neon Pageserver binutils") + .about("Reads pageserver (and related) binary files management utility") + .version(GIT_VERSION) + .arg( + Arg::new("path") + .help("Input file path") + .value_parser(value_parser!(PathBuf)) + .required(false), + ) + .subcommand( + Command::new(METADATA_SUBCOMMAND) + .about("Read and update pageserver metadata file") + .arg( + Arg::new("metadata_path") + .help("Input metadata file path") + .value_parser(value_parser!(PathBuf)) + .required(false), + ) + .arg( + Arg::new("disk_consistent_lsn") + .long("disk_consistent_lsn") + .help("Replace disk consistent Lsn"), + ) + .arg( + Arg::new("prev_record_lsn") + .long("prev_record_lsn") + .help("Replace previous record Lsn"), + ), + ) +} + +#[test] +fn verify_cli() { + cli().debug_assert(); +} diff --git a/proxy/Cargo.toml b/proxy/Cargo.toml index 8049737989..395c22b8bc 100644 --- a/proxy/Cargo.toml +++ b/proxy/Cargo.toml @@ -7,9 +7,9 @@ edition = "2021" anyhow = "1.0" atty = "0.2.14" base64 = "0.13.0" -bstr = "0.2.17" +bstr = "1.0" bytes = { version = "1.0.1", features = ['serde'] } -clap = "3.0" +clap = "4.0" futures = "0.3.13" git-version = "0.3.5" hashbrown = "0.12" @@ -22,7 +22,11 @@ once_cell = "1.13.0" parking_lot = "0.12" pin-project-lite = "0.2.7" rand = "0.8.3" -reqwest = { version = "0.11", default-features = false, features = ["blocking", "json", "rustls-tls"] } +reqwest = { version = "0.11", default-features = false, features = [ + "blocking", + "json", + "rustls-tls", +] } routerify = "3" rustls = "0.20.0" rustls-pemfile = "1" @@ -33,13 +37,13 @@ sha2 = "0.10.2" socket2 = "0.4.4" thiserror = "1.0.30" tokio = { version = "1.17", features = ["macros"] } -tokio-postgres = { git = "https://github.com/neondatabase/rust-postgres.git", rev="d052ee8b86fff9897c77b0fe89ea9daba0e1fa38" } +tokio-postgres = { git = "https://github.com/neondatabase/rust-postgres.git", rev = "d052ee8b86fff9897c77b0fe89ea9daba0e1fa38" } tokio-rustls = "0.23.0" tracing = "0.1.36" tracing-subscriber = { version = "0.3", features = ["env-filter"] } url = "2.2.2" -uuid = { version = "0.8.2", features = ["v4", "serde"]} -x509-parser = "0.13.2" +uuid = { version = "1.2", features = ["v4", "serde"] } +x509-parser = "0.14" utils = { path = "../libs/utils" } metrics = { path = "../libs/metrics" } @@ -47,6 +51,6 @@ workspace_hack = { version = "0.1", path = "../workspace_hack" } [dev-dependencies] async-trait = "0.1" -rcgen = "0.8.14" -rstest = "0.12" +rcgen = "0.10" +rstest = "0.15" tokio-postgres-rustls = "0.9.0" diff --git a/proxy/src/main.rs b/proxy/src/main.rs index 91ef26a37f..2055616a6e 100644 --- a/proxy/src/main.rs +++ b/proxy/src/main.rs @@ -45,98 +45,43 @@ async fn main() -> anyhow::Result<()> { .with_target(false) .init(); - let arg_matches = clap::App::new("Neon proxy/router") - .version(GIT_VERSION) - .arg( - Arg::new("proxy") - .short('p') - .long("proxy") - .takes_value(true) - .help("listen for incoming client connections on ip:port") - .default_value("127.0.0.1:4432"), - ) - .arg( - Arg::new("auth-backend") - .long("auth-backend") - .takes_value(true) - .possible_values(["console", "postgres", "link"]) - .default_value("link"), - ) - .arg( - Arg::new("mgmt") - .short('m') - .long("mgmt") - .takes_value(true) - .help("listen for management callback connection on ip:port") - .default_value("127.0.0.1:7000"), - ) - .arg( - Arg::new("http") - .short('h') - .long("http") - .takes_value(true) - .help("listen for incoming http connections (metrics, etc) on ip:port") - .default_value("127.0.0.1:7001"), - ) - .arg( - Arg::new("uri") - .short('u') - .long("uri") - .takes_value(true) - .help("redirect unauthenticated users to the given uri in case of link auth") - .default_value("http://localhost:3000/psql_session/"), - ) - .arg( - Arg::new("auth-endpoint") - .short('a') - .long("auth-endpoint") - .takes_value(true) - .help("cloud API endpoint for authenticating users") - .default_value("http://localhost:3000/authenticate_proxy_request/"), - ) - .arg( - Arg::new("tls-key") - .short('k') - .long("tls-key") - .alias("ssl-key") // backwards compatibility - .takes_value(true) - .help("path to TLS key for client postgres connections"), - ) - .arg( - Arg::new("tls-cert") - .short('c') - .long("tls-cert") - .alias("ssl-cert") // backwards compatibility - .takes_value(true) - .help("path to TLS cert for client postgres connections"), - ) - .get_matches(); + let arg_matches = cli().get_matches(); let tls_config = match ( - arg_matches.value_of("tls-key"), - arg_matches.value_of("tls-cert"), + arg_matches.get_one::("tls-key"), + arg_matches.get_one::("tls-cert"), ) { (Some(key_path), Some(cert_path)) => Some(config::configure_tls(key_path, cert_path)?), (None, None) => None, _ => bail!("either both or neither tls-key and tls-cert must be specified"), }; - let proxy_address: SocketAddr = arg_matches.value_of("proxy").unwrap().parse()?; - let mgmt_address: SocketAddr = arg_matches.value_of("mgmt").unwrap().parse()?; - let http_address: SocketAddr = arg_matches.value_of("http").unwrap().parse()?; + let proxy_address: SocketAddr = arg_matches.get_one::("proxy").unwrap().parse()?; + let mgmt_address: SocketAddr = arg_matches.get_one::("mgmt").unwrap().parse()?; + let http_address: SocketAddr = arg_matches.get_one::("http").unwrap().parse()?; - let auth_backend = match arg_matches.value_of("auth-backend").unwrap() { + let auth_backend = match arg_matches + .get_one::("auth-backend") + .unwrap() + .as_str() + { "console" => { - let url = arg_matches.value_of("auth-endpoint").unwrap().parse()?; + let url = arg_matches + .get_one::("auth-endpoint") + .unwrap() + .parse()?; let endpoint = http::Endpoint::new(url, reqwest::Client::new()); auth::BackendType::Console(Cow::Owned(endpoint), ()) } "postgres" => { - let url = arg_matches.value_of("auth-endpoint").unwrap().parse()?; + let url = arg_matches + .get_one::("auth-endpoint") + .unwrap() + .parse()?; auth::BackendType::Postgres(Cow::Owned(url), ()) } "link" => { - let url = arg_matches.value_of("uri").unwrap().parse()?; + let url = arg_matches.get_one::("uri").unwrap().parse()?; auth::BackendType::Link(Cow::Owned(url)) } other => bail!("unsupported auth backend: {other}"), @@ -174,3 +119,68 @@ async fn main() -> anyhow::Result<()> { Ok(()) } + +fn cli() -> clap::Command { + clap::Command::new("Neon proxy/router") + .disable_help_flag(true) + .version(GIT_VERSION) + .arg( + Arg::new("proxy") + .short('p') + .long("proxy") + .help("listen for incoming client connections on ip:port") + .default_value("127.0.0.1:4432"), + ) + .arg( + Arg::new("auth-backend") + .long("auth-backend") + .value_parser(["console", "postgres", "link"]) + .default_value("link"), + ) + .arg( + Arg::new("mgmt") + .short('m') + .long("mgmt") + .help("listen for management callback connection on ip:port") + .default_value("127.0.0.1:7000"), + ) + .arg( + Arg::new("http") + .long("http") + .help("listen for incoming http connections (metrics, etc) on ip:port") + .default_value("127.0.0.1:7001"), + ) + .arg( + Arg::new("uri") + .short('u') + .long("uri") + .help("redirect unauthenticated users to the given uri in case of link auth") + .default_value("http://localhost:3000/psql_session/"), + ) + .arg( + Arg::new("auth-endpoint") + .short('a') + .long("auth-endpoint") + .help("cloud API endpoint for authenticating users") + .default_value("http://localhost:3000/authenticate_proxy_request/"), + ) + .arg( + Arg::new("tls-key") + .short('k') + .long("tls-key") + .alias("ssl-key") // backwards compatibility + .help("path to TLS key for client postgres connections"), + ) + .arg( + Arg::new("tls-cert") + .short('c') + .long("tls-cert") + .alias("ssl-cert") // backwards compatibility + .help("path to TLS cert for client postgres connections"), + ) +} + +#[test] +fn verify_cli() { + cli().debug_assert(); +} diff --git a/safekeeper/Cargo.toml b/safekeeper/Cargo.toml index ddc3956d74..64c541ddef 100644 --- a/safekeeper/Cargo.toml +++ b/safekeeper/Cargo.toml @@ -11,7 +11,7 @@ hyper = "0.14" fs2 = "0.4.3" serde_json = "1" tracing = "0.1.27" -clap = "3.0" +clap = "4.0" daemonize = "0.4.1" tokio = { version = "1.17", features = ["macros", "fs"] } postgres-protocol = { git = "https://github.com/neondatabase/rust-postgres.git", rev="d052ee8b86fff9897c77b0fe89ea9daba0e1fa38" } diff --git a/safekeeper/src/bin/safekeeper.rs b/safekeeper/src/bin/safekeeper.rs index 3f55d823cc..9422b55d60 100644 --- a/safekeeper/src/bin/safekeeper.rs +++ b/safekeeper/src/bin/safekeeper.rs @@ -2,7 +2,7 @@ // Main entry point for the safekeeper executable // use anyhow::{bail, Context, Result}; -use clap::{App, Arg}; +use clap::{value_parser, Arg, ArgAction, Command}; use const_format::formatcp; use daemonize::Daemonize; use fs2::FileExt; @@ -40,145 +40,44 @@ const ID_FILE_NAME: &str = "safekeeper.id"; project_git_version!(GIT_VERSION); fn main() -> anyhow::Result<()> { - let arg_matches = App::new("Neon safekeeper") - .about("Store WAL stream to local file system and push it to WAL receivers") - .version(GIT_VERSION) - .arg( - Arg::new("datadir") - .short('D') - .long("dir") - .takes_value(true) - .help("Path to the safekeeper data directory"), - ) - .arg( - Arg::new("init") - .long("init") - .takes_value(false) - .help("Initialize safekeeper with ID"), - ) - .arg( - Arg::new("listen-pg") - .short('l') - .long("listen-pg") - .alias("listen") // for compatibility - .takes_value(true) - .help(formatcp!("listen for incoming WAL data connections on ip:port (default: {DEFAULT_PG_LISTEN_ADDR})")), - ) - .arg( - Arg::new("listen-http") - .long("listen-http") - .takes_value(true) - .help(formatcp!("http endpoint address for metrics on ip:port (default: {DEFAULT_HTTP_LISTEN_ADDR})")), - ) - // FIXME this argument is no longer needed since pageserver address is forwarded from compute. - // However because this argument is in use by console's e2e tests let's keep it for now and remove separately. - // So currently it is a noop. - .arg( - Arg::new("pageserver") - .short('p') - .long("pageserver") - .takes_value(true), - ) - .arg( - Arg::new("recall") - .long("recall") - .takes_value(true) - .help("Period for requestion pageserver to call for replication"), - ) - .arg( - Arg::new("daemonize") - .short('d') - .long("daemonize") - .takes_value(false) - .help("Run in the background"), - ) - .arg( - Arg::new("no-sync") - .short('n') - .long("no-sync") - .takes_value(false) - .help("Do not wait for changes to be written safely to disk"), - ) - .arg( - Arg::new("dump-control-file") - .long("dump-control-file") - .takes_value(true) - .help("Dump control file at path specified by this argument and exit"), - ) - .arg( - Arg::new("id").long("id").takes_value(true).help("safekeeper node id: integer") - ).arg( - Arg::new("broker-endpoints") - .long("broker-endpoints") - .takes_value(true) - .help("a comma separated broker (etcd) endpoints for storage nodes coordination, e.g. 'http://127.0.0.1:2379'"), - ) - .arg( - Arg::new("broker-etcd-prefix") - .long("broker-etcd-prefix") - .takes_value(true) - .help("a prefix to always use when polling/pusing data in etcd from this safekeeper"), - ) - .arg( - Arg::new("wal-backup-threads").long("backup-threads").takes_value(true).help(formatcp!("number of threads for wal backup (default {DEFAULT_WAL_BACKUP_RUNTIME_THREADS}")), - ).arg( - Arg::new("remote-storage") - .long("remote-storage") - .takes_value(true) - .help("Remote storage configuration for WAL backup (offloading to s3) as TOML inline table, e.g. {\"max_concurrent_syncs\" = 17, \"max_sync_errors\": 13, \"bucket_name\": \"\", \"bucket_region\":\"\", \"concurrency_limit\": 119}.\nSafekeeper offloads WAL to [prefix_in_bucket/]//, mirroring structure on the file system.") - ) - .arg( - Arg::new("enable-wal-backup") - .long("enable-wal-backup") - .takes_value(true) - .default_value("true") - .default_missing_value("true") - .help("Enable/disable WAL backup to s3. When disabled, safekeeper removes WAL ignoring WAL backup horizon."), - ) - .arg( - Arg::new("auth-validation-public-key-path") - .long("auth-validation-public-key-path") - .takes_value(true) - .help("Path to an RSA .pem public key which is used to check JWT tokens") - ) - .get_matches(); + let arg_matches = cli().get_matches(); - if let Some(addr) = arg_matches.value_of("dump-control-file") { + if let Some(addr) = arg_matches.get_one::("dump-control-file") { let state = control_file::FileStorage::load_control_file(Path::new(addr))?; let json = serde_json::to_string(&state)?; - print!("{}", json); + print!("{json}"); return Ok(()); } let mut conf = SafeKeeperConf::default(); - if let Some(dir) = arg_matches.value_of("datadir") { + if let Some(dir) = arg_matches.get_one::("datadir") { // change into the data directory. - std::env::set_current_dir(PathBuf::from(dir))?; + std::env::set_current_dir(dir)?; } - if arg_matches.is_present("no-sync") { + if arg_matches.get_flag("no-sync") { conf.no_sync = true; } - if arg_matches.is_present("daemonize") { + if arg_matches.get_flag("daemonize") { conf.daemonize = true; } - if let Some(addr) = arg_matches.value_of("listen-pg") { - conf.listen_pg_addr = addr.to_owned(); + if let Some(addr) = arg_matches.get_one::("listen-pg") { + conf.listen_pg_addr = addr.to_string(); } - if let Some(addr) = arg_matches.value_of("listen-http") { - conf.listen_http_addr = addr.to_owned(); + if let Some(addr) = arg_matches.get_one::("listen-http") { + conf.listen_http_addr = addr.to_string(); } - if let Some(recall) = arg_matches.value_of("recall") { + if let Some(recall) = arg_matches.get_one::("recall") { conf.recall_period = humantime::parse_duration(recall)?; } let mut given_id = None; - if let Some(given_id_str) = arg_matches.value_of("id") { + if let Some(given_id_str) = arg_matches.get_one::("id") { given_id = Some(NodeId( given_id_str .parse() @@ -186,20 +85,20 @@ fn main() -> anyhow::Result<()> { )); } - if let Some(addr) = arg_matches.value_of("broker-endpoints") { + if let Some(addr) = arg_matches.get_one::("broker-endpoints") { let collected_ep: Result, ParseError> = addr.split(',').map(Url::parse).collect(); conf.broker_endpoints = collected_ep.context("Failed to parse broker endpoint urls")?; } - if let Some(prefix) = arg_matches.value_of("broker-etcd-prefix") { + if let Some(prefix) = arg_matches.get_one::("broker-etcd-prefix") { conf.broker_etcd_prefix = prefix.to_string(); } - if let Some(backup_threads) = arg_matches.value_of("wal-backup-threads") { + if let Some(backup_threads) = arg_matches.get_one::("wal-backup-threads") { conf.backup_runtime_threads = backup_threads .parse() .with_context(|| format!("Failed to parse backup threads {}", backup_threads))?; } - if let Some(storage_conf) = arg_matches.value_of("remote-storage") { + if let Some(storage_conf) = arg_matches.get_one::("remote-storage") { // funny toml doesn't consider plain inline table as valid document, so wrap in a key to parse let storage_conf_toml = format!("remote_storage = {}", storage_conf); let parsed_toml = storage_conf_toml.parse::()?; // parse @@ -208,16 +107,16 @@ fn main() -> anyhow::Result<()> { } // Seems like there is no better way to accept bool values explicitly in clap. conf.wal_backup_enabled = arg_matches - .value_of("enable-wal-backup") + .get_one::("enable-wal-backup") .unwrap() .parse() .context("failed to parse bool enable-s3-offload bool")?; conf.auth_validation_public_key_path = arg_matches - .value_of("auth-validation-public-key-path") + .get_one::("auth-validation-public-key-path") .map(PathBuf::from); - start_safekeeper(conf, given_id, arg_matches.is_present("init")) + start_safekeeper(conf, given_id, arg_matches.get_flag("init")) } fn start_safekeeper(mut conf: SafeKeeperConf, given_id: Option, init: bool) -> Result<()> { @@ -424,3 +323,102 @@ fn set_id(conf: &mut SafeKeeperConf, given_id: Option) -> Result<()> { conf.my_id = my_id; Ok(()) } + +fn cli() -> Command { + Command::new("Neon safekeeper") + .about("Store WAL stream to local file system and push it to WAL receivers") + .version(GIT_VERSION) + .arg( + Arg::new("datadir") + .short('D') + .long("dir") + .value_parser(value_parser!(PathBuf)) + .help("Path to the safekeeper data directory"), + ) + .arg( + Arg::new("init") + .long("init") + .action(ArgAction::SetTrue) + .help("Initialize safekeeper with ID"), + ) + .arg( + Arg::new("listen-pg") + .short('l') + .long("listen-pg") + .alias("listen") // for compatibility + .help(formatcp!("listen for incoming WAL data connections on ip:port (default: {DEFAULT_PG_LISTEN_ADDR})")), + ) + .arg( + Arg::new("listen-http") + .long("listen-http") + .help(formatcp!("http endpoint address for metrics on ip:port (default: {DEFAULT_HTTP_LISTEN_ADDR})")), + ) + // FIXME this argument is no longer needed since pageserver address is forwarded from compute. + // However because this argument is in use by console's e2e tests let's keep it for now and remove separately. + // So currently it is a noop. + .arg( + Arg::new("pageserver") + .short('p') + .long("pageserver"), + ) + .arg( + Arg::new("recall") + .long("recall") + .help("Period for requestion pageserver to call for replication"), + ) + .arg( + Arg::new("daemonize") + .short('d') + .long("daemonize") + .action(ArgAction::SetTrue) + .help("Run in the background"), + ) + .arg( + Arg::new("no-sync") + .short('n') + .long("no-sync") + .action(ArgAction::SetTrue) + .help("Do not wait for changes to be written safely to disk"), + ) + .arg( + Arg::new("dump-control-file") + .long("dump-control-file") + .help("Dump control file at path specified by this argument and exit"), + ) + .arg( + Arg::new("id").long("id").help("safekeeper node id: integer") + ).arg( + Arg::new("broker-endpoints") + .long("broker-endpoints") + .help("a comma separated broker (etcd) endpoints for storage nodes coordination, e.g. 'http://127.0.0.1:2379'"), + ) + .arg( + Arg::new("broker-etcd-prefix") + .long("broker-etcd-prefix") + .help("a prefix to always use when polling/pusing data in etcd from this safekeeper"), + ) + .arg( + Arg::new("wal-backup-threads").long("backup-threads").help(formatcp!("number of threads for wal backup (default {DEFAULT_WAL_BACKUP_RUNTIME_THREADS}")), + ).arg( + Arg::new("remote-storage") + .long("remote-storage") + .help("Remote storage configuration for WAL backup (offloading to s3) as TOML inline table, e.g. {\"max_concurrent_syncs\" = 17, \"max_sync_errors\": 13, \"bucket_name\": \"\", \"bucket_region\":\"\", \"concurrency_limit\": 119}.\nSafekeeper offloads WAL to [prefix_in_bucket/]//, mirroring structure on the file system.") + ) + .arg( + Arg::new("enable-wal-backup") + .long("enable-wal-backup") + .default_value("true") + .default_missing_value("true") + .help("Enable/disable WAL backup to s3. When disabled, safekeeper removes WAL ignoring WAL backup horizon."), + ) + .arg( + Arg::new("auth-validation-public-key-path") + .long("auth-validation-public-key-path") + .help("Path to an RSA .pem public key which is used to check JWT tokens") + ) +} + +#[test] +fn verify_cli() { + cli().debug_assert(); +} diff --git a/workspace_hack/Cargo.toml b/workspace_hack/Cargo.toml index af055ed9a4..f4468d85f0 100644 --- a/workspace_hack/Cargo.toml +++ b/workspace_hack/Cargo.toml @@ -8,7 +8,6 @@ version = "0.1.0" description = "workspace-hack package, managed by hakari" # You can choose to publish this crate: see https://docs.rs/cargo-hakari/latest/cargo_hakari/publishing. publish = false - # The parts of the file between the BEGIN HAKARI SECTION and END HAKARI SECTION comments # are managed by hakari. @@ -18,7 +17,7 @@ ahash = { version = "0.7", features = ["std"] } anyhow = { version = "1", features = ["backtrace", "std"] } bytes = { version = "1", features = ["serde", "std"] } chrono = { version = "0.4", features = ["clock", "iana-time-zone", "js-sys", "oldtime", "serde", "std", "time", "wasm-bindgen", "wasmbind", "winapi"] } -clap = { version = "3", features = ["atty", "color", "std", "strsim", "suggestions", "termcolor"] } +clap = { version = "4", features = ["color", "error-context", "help", "std", "string", "suggestions", "usage"] } crossbeam-utils = { version = "0.8", features = ["once_cell", "std"] } either = { version = "1", features = ["use_std"] } fail = { version = "0.5", default-features = false, features = ["failpoints"] } @@ -26,6 +25,7 @@ hashbrown = { version = "0.12", features = ["ahash", "inline-more", "raw"] } indexmap = { version = "1", default-features = false, features = ["std"] } libc = { version = "0.2", features = ["extra_traits", "std"] } log = { version = "0.4", default-features = false, features = ["serde", "std"] } +memchr = { version = "2", features = ["std"] } nom = { version = "7", features = ["alloc", "std"] } num-bigint = { version = "0.4", features = ["std"] } num-integer = { version = "0.1", default-features = false, features = ["i128", "std"] } @@ -42,18 +42,17 @@ tokio = { version = "1", features = ["bytes", "fs", "io-std", "io-util", "libc", tokio-util = { version = "0.7", features = ["codec", "io", "io-util", "tracing"] } tracing = { version = "0.1", features = ["attributes", "log", "std", "tracing-attributes"] } tracing-core = { version = "0.1", features = ["once_cell", "std"] } -uuid = { version = "0.8", features = ["getrandom", "serde", "std", "v4"] } [build-dependencies] ahash = { version = "0.7", features = ["std"] } anyhow = { version = "1", features = ["backtrace", "std"] } bytes = { version = "1", features = ["serde", "std"] } -clap = { version = "3", features = ["atty", "color", "std", "strsim", "suggestions", "termcolor"] } either = { version = "1", features = ["use_std"] } hashbrown = { version = "0.12", features = ["ahash", "inline-more", "raw"] } indexmap = { version = "1", default-features = false, features = ["std"] } libc = { version = "0.2", features = ["extra_traits", "std"] } log = { version = "0.4", default-features = false, features = ["serde", "std"] } +memchr = { version = "2", features = ["std"] } nom = { version = "7", features = ["alloc", "std"] } prost = { version = "0.10", features = ["prost-derive", "std"] } regex = { version = "1", features = ["aho-corasick", "memchr", "perf", "perf-cache", "perf-dfa", "perf-inline", "perf-literal", "std", "unicode", "unicode-age", "unicode-bool", "unicode-case", "unicode-gencat", "unicode-perl", "unicode-script", "unicode-segment"] }