diff --git a/.github/ansible/staging.hosts.yaml b/.github/ansible/staging.hosts.yaml index 44d971455d..ae55f9223c 100644 --- a/.github/ansible/staging.hosts.yaml +++ b/.github/ansible/staging.hosts.yaml @@ -3,7 +3,7 @@ storage: bucket_name: zenith-staging-storage-us-east-1 bucket_region: us-east-1 console_mgmt_base_url: http://console-staging.local - etcd_endpoints: zenith-us-stage-etcd.local:2379 + etcd_endpoints: etcd-0.us-east-2.aws.neon.build:2379 pageserver_config_stub: pg_distrib_dir: /usr/local remote_storage: diff --git a/CODEOWNERS b/CODEOWNERS index 4c8c8924d6..6b1273520d 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -8,3 +8,4 @@ /pgxn/ @neondatabase/compute /proxy/ @neondatabase/control-plane /safekeeper/ @neondatabase/safekeepers +/vendor/ @neondatabase/compute diff --git a/Cargo.lock b/Cargo.lock index a34a5b44f9..c112c05188 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -169,333 +169,6 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" -[[package]] -name = "aws-config" -version = "0.51.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56a636c44c77fa18bdba56126a34d30cfe5538fe88f7d34988fa731fee143ddd" -dependencies = [ - "aws-http", - "aws-sdk-sso", - "aws-sdk-sts", - "aws-smithy-async", - "aws-smithy-client", - "aws-smithy-http", - "aws-smithy-http-tower", - "aws-smithy-json", - "aws-smithy-types", - "aws-types", - "bytes", - "hex", - "http", - "hyper", - "ring", - "time 0.3.15", - "tokio", - "tower", - "tracing", - "zeroize", -] - -[[package]] -name = "aws-endpoint" -version = "0.51.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ca8f374874f6459aaa88dc861d7f5d834ca1ff97668eae190e97266b5f6c3fb" -dependencies = [ - "aws-smithy-http", - "aws-smithy-types", - "aws-types", - "http", - "regex", - "tracing", -] - -[[package]] -name = "aws-http" -version = "0.51.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78d41e19e779b73463f5f0c21b3aacc995f4ba783ab13a7ae9f5dfb159a551b4" -dependencies = [ - "aws-smithy-http", - "aws-smithy-types", - "aws-types", - "bytes", - "http", - "http-body", - "lazy_static", - "percent-encoding", - "pin-project-lite", - "tracing", -] - -[[package]] -name = "aws-sdk-s3" -version = "0.21.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9f08665c8e03aca8cb092ef01e617436ebfa977fddc1240e1b062488ab5d48a" -dependencies = [ - "aws-endpoint", - "aws-http", - "aws-sig-auth", - "aws-sigv4", - "aws-smithy-async", - "aws-smithy-checksums", - "aws-smithy-client", - "aws-smithy-eventstream", - "aws-smithy-http", - "aws-smithy-http-tower", - "aws-smithy-types", - "aws-smithy-xml", - "aws-types", - "bytes", - "bytes-utils", - "http", - "http-body", - "tokio-stream", - "tower", - "tracing", -] - -[[package]] -name = "aws-sdk-sso" -version = "0.21.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86dcb1cb71aa8763b327542ead410424515cff0cde5b753eedd2917e09c63734" -dependencies = [ - "aws-endpoint", - "aws-http", - "aws-sig-auth", - "aws-smithy-async", - "aws-smithy-client", - "aws-smithy-http", - "aws-smithy-http-tower", - "aws-smithy-json", - "aws-smithy-types", - "aws-types", - "bytes", - "http", - "tokio-stream", - "tower", -] - -[[package]] -name = "aws-sdk-sts" -version = "0.21.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fdfcf584297c666f6b472d5368a78de3bc714b6e0a53d7fbf76c3e347c292ab1" -dependencies = [ - "aws-endpoint", - "aws-http", - "aws-sig-auth", - "aws-smithy-async", - "aws-smithy-client", - "aws-smithy-http", - "aws-smithy-http-tower", - "aws-smithy-query", - "aws-smithy-types", - "aws-smithy-xml", - "aws-types", - "bytes", - "http", - "tower", -] - -[[package]] -name = "aws-sig-auth" -version = "0.51.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12cbe7b2be9e185c1fbce27fc9c41c66b195b32d89aa099f98768d9544221308" -dependencies = [ - "aws-sigv4", - "aws-smithy-eventstream", - "aws-smithy-http", - "aws-types", - "http", - "tracing", -] - -[[package]] -name = "aws-sigv4" -version = "0.51.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03ff4cff8c4a101962d593ba94e72cd83891aecd423f0c6e3146bff6fb92c9e3" -dependencies = [ - "aws-smithy-eventstream", - "aws-smithy-http", - "bytes", - "form_urlencoded", - "hex", - "http", - "once_cell", - "percent-encoding", - "regex", - "ring", - "time 0.3.15", - "tracing", -] - -[[package]] -name = "aws-smithy-async" -version = "0.51.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b3442b4c5d3fc39891a2e5e625735fba6b24694887d49c6518460fde98247a9" -dependencies = [ - "futures-util", - "pin-project-lite", - "tokio", - "tokio-stream", -] - -[[package]] -name = "aws-smithy-checksums" -version = "0.51.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc227e36e346f45298288359f37123e1a92628d1cec6b11b5eb335553278bd9e" -dependencies = [ - "aws-smithy-http", - "aws-smithy-types", - "bytes", - "crc32c", - "crc32fast", - "hex", - "http", - "http-body", - "md-5", - "pin-project-lite", - "sha1", - "sha2", - "tracing", -] - -[[package]] -name = "aws-smithy-client" -version = "0.51.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff28d553714f8f54cd921227934fc13a536a1c03f106e56b362fd57e16d450ad" -dependencies = [ - "aws-smithy-async", - "aws-smithy-http", - "aws-smithy-http-tower", - "aws-smithy-types", - "bytes", - "fastrand", - "http", - "http-body", - "hyper", - "hyper-rustls", - "lazy_static", - "pin-project-lite", - "tokio", - "tower", - "tracing", -] - -[[package]] -name = "aws-smithy-eventstream" -version = "0.51.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7ea0df7161ce65b5c8ca6eb709a1a907376fa18226976e41c748ce02ccccf24" -dependencies = [ - "aws-smithy-types", - "bytes", - "crc32fast", -] - -[[package]] -name = "aws-smithy-http" -version = "0.51.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf58ed4fefa61dbf038e5421a521cbc2c448ef69deff0ab1d915d8a10eda5664" -dependencies = [ - "aws-smithy-eventstream", - "aws-smithy-types", - "bytes", - "bytes-utils", - "futures-core", - "http", - "http-body", - "hyper", - "once_cell", - "percent-encoding", - "pin-project-lite", - "pin-utils", - "tokio", - "tokio-util", - "tracing", -] - -[[package]] -name = "aws-smithy-http-tower" -version = "0.51.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "20c96d7bd35e7cf96aca1134b2f81b1b59ffe493f7c6539c051791cbbf7a42d3" -dependencies = [ - "aws-smithy-http", - "bytes", - "http", - "http-body", - "pin-project-lite", - "tower", - "tracing", -] - -[[package]] -name = "aws-smithy-json" -version = "0.51.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d8324ba98c8a94187723cc16c37aefa09504646ee65c3d2c3af495bab5ea701b" -dependencies = [ - "aws-smithy-types", -] - -[[package]] -name = "aws-smithy-query" -version = "0.51.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83834ed2ff69ea6f6657baf205267dc2c0abe940703503a3e5d60ce23be3d306" -dependencies = [ - "aws-smithy-types", - "urlencoding", -] - -[[package]] -name = "aws-smithy-types" -version = "0.51.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b02e06ea63498c43bc0217ea4d16605d4e58d85c12fc23f6572ff6d0a840c61" -dependencies = [ - "itoa", - "num-integer", - "ryu", - "time 0.3.15", -] - -[[package]] -name = "aws-smithy-xml" -version = "0.51.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "246e9f83dd1fdf5d347fa30ae4ad30a9d1d42ce4cd74a93d94afa874646f94cd" -dependencies = [ - "xmlparser", -] - -[[package]] -name = "aws-types" -version = "0.51.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05701d32da168b44f7ee63147781aed8723e792cc131cb9b18363b5393f17f70" -dependencies = [ - "aws-smithy-async", - "aws-smithy-client", - "aws-smithy-http", - "aws-smithy-types", - "http", - "rustc_version 0.4.0", - "tracing", - "zeroize", -] - [[package]] name = "axum" version = "0.5.16" @@ -626,6 +299,15 @@ version = "1.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" +[[package]] +name = "block-buffer" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4152116fd6e9dadb291ae18fc1ec3575ed6d84c29642d97890f4b4a3417297e4" +dependencies = [ + "generic-array", +] + [[package]] name = "block-buffer" version = "0.10.3" @@ -674,16 +356,6 @@ dependencies = [ "serde", ] -[[package]] -name = "bytes-utils" -version = "0.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e47d3a8076e283f3acd27400535992edb3ba4b5bb72f8891ad8fbe7932a7d4b9" -dependencies = [ - "bytes", - "either", -] - [[package]] name = "cast" version = "0.3.0" @@ -1118,6 +790,16 @@ dependencies = [ "typenum", ] +[[package]] +name = "crypto-mac" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1d1a86f49236c215f271d40892d5fc950490551400b02ef360692c29815c714" +dependencies = [ + "generic-array", + "subtle", +] + [[package]] name = "cxx" version = "1.0.79" @@ -1226,17 +908,47 @@ dependencies = [ "rusticata-macros", ] +[[package]] +name = "digest" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3dd60d1080a57a05ab032377049e0591415d2b31afd7028356dbf3cc6dcb066" +dependencies = [ + "generic-array", +] + [[package]] name = "digest" version = "0.10.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "adfbc57365a37acbd2ebf2b64d7e69bb766e2fea813521ed536f5d0520dcf86c" dependencies = [ - "block-buffer", + "block-buffer 0.10.3", "crypto-common", "subtle", ] +[[package]] +name = "dirs-next" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b98cf8ebf19c3d1b223e151f99a4f9f0690dca41414773390fc824184ac833e1" +dependencies = [ + "cfg-if", + "dirs-sys-next", +] + +[[package]] +name = "dirs-sys-next" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ebda144c4fe02d1f7ea1a7d9641b6fc6b580adcfa024ae48797ecdeb6825b4d" +dependencies = [ + "libc", + "redox_users", + "winapi", +] + [[package]] name = "displaydoc" version = "0.2.3" @@ -1381,6 +1093,21 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" +[[package]] +name = "foreign-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" +dependencies = [ + "foreign-types-shared", +] + +[[package]] +name = "foreign-types-shared" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" + [[package]] name = "form_urlencoded" version = "1.1.0" @@ -1645,13 +1372,23 @@ version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7ebdb29d2ea9ed0083cd8cece49bbd968021bd99b0849edb4a9a7ee0fdf6a4e0" +[[package]] +name = "hmac" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a2a2320eb7ec0ebe8da8f744d7812d9fc4cb4d09344ac01898dbcb6a20ae69b" +dependencies = [ + "crypto-mac", + "digest 0.9.0", +] + [[package]] name = "hmac" version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" dependencies = [ - "digest", + "digest 0.10.5", ] [[package]] @@ -1742,9 +1479,7 @@ checksum = "d87c48c02e0dc5e3b849a2041db3029fd066650f8f717c07bf8ed78ccb895cac" dependencies = [ "http", "hyper", - "log", "rustls", - "rustls-native-certs", "tokio", "tokio-rustls", ] @@ -1761,6 +1496,19 @@ dependencies = [ "tokio-io-timeout", ] +[[package]] +name = "hyper-tls" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905" +dependencies = [ + "bytes", + "hyper", + "native-tls", + "tokio", + "tokio-native-tls", +] + [[package]] name = "iana-time-zone" version = "0.1.51" @@ -2001,13 +1749,24 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "73cbba799671b762df5a175adf59ce145165747bb891505c43d09aefbbf38beb" +[[package]] +name = "md-5" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b5a279bb9607f9f53c22d496eade00d138d1bdcccd07d74650387cf94942a15" +dependencies = [ + "block-buffer 0.9.0", + "digest 0.9.0", + "opaque-debug", +] + [[package]] name = "md-5" version = "0.10.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6365506850d44bff6e2fbcb5176cf63650e48bd45ef2fe2665ae1570e0f4b9ca" dependencies = [ - "digest", + "digest 0.10.5", ] [[package]] @@ -2098,6 +1857,24 @@ version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e5ce46fe64a9d73be07dcbe690a38ce1b293be448fd8ce1e6c1b8062c9f72c6a" +[[package]] +name = "native-tls" +version = "0.2.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd7e2f3618557f980e0b17e8856252eee3c97fa12c54dff0ca290fb6266ca4a9" +dependencies = [ + "lazy_static", + "libc", + "log", + "openssl", + "openssl-probe", + "openssl-sys", + "schannel", + "security-framework", + "security-framework-sys", + "tempfile", +] + [[package]] name = "nb" version = "0.1.3" @@ -2268,12 +2045,57 @@ version = "11.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ab1bc2a289d34bd04a330323ac98a1b4bc82c9d9fcb1e66b63caa84da26b575" +[[package]] +name = "opaque-debug" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5" + +[[package]] +name = "openssl" +version = "0.10.42" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12fc0523e3bd51a692c8850d075d74dc062ccf251c0110668cbd921917118a13" +dependencies = [ + "bitflags", + "cfg-if", + "foreign-types", + "libc", + "once_cell", + "openssl-macros", + "openssl-sys", +] + +[[package]] +name = "openssl-macros" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b501e44f11665960c7e7fcf062c7d96a14ade4aa98116c004b2e37b5be7d736c" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "openssl-probe" version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" +[[package]] +name = "openssl-sys" +version = "0.9.76" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5230151e44c0f05157effb743e8d517472843121cf9243e8b81393edb5acd9ce" +dependencies = [ + "autocfg", + "cc", + "libc", + "pkg-config", + "vcpkg", +] + [[package]] name = "os_str_bytes" version = "6.3.0" @@ -2493,6 +2315,12 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" +[[package]] +name = "pkg-config" +version = "0.3.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1df8c4ec4b0627e53bdf214615ad287367e482558cf84b109250b37464dc03ae" + [[package]] name = "plotters" version = "0.3.4" @@ -2543,12 +2371,12 @@ dependencies = [ "byteorder", "bytes", "fallible-iterator", - "hmac", + "hmac 0.12.1", "lazy_static", - "md-5", + "md-5 0.10.5", "memchr", "rand", - "sha2", + "sha2 0.10.6", "stringprep", ] @@ -2750,7 +2578,7 @@ dependencies = [ "git-version", "hashbrown", "hex", - "hmac", + "hmac 0.12.1", "hyper", "itertools", "md5", @@ -2769,7 +2597,7 @@ dependencies = [ "scopeguard", "serde", "serde_json", - "sha2", + "sha2 0.10.6", "socket2", "thiserror", "tokio", @@ -2888,6 +2716,17 @@ dependencies = [ "bitflags", ] +[[package]] +name = "redox_users" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b033d837a7cf162d7993aded9304e30a83213c648b6e389db233191f891e5c2b" +dependencies = [ + "getrandom", + "redox_syscall", + "thiserror", +] + [[package]] name = "regex" version = "1.6.0" @@ -2920,13 +2759,10 @@ version = "0.1.0" dependencies = [ "anyhow", "async-trait", - "aws-config", - "aws-sdk-s3", - "aws-smithy-http", - "aws-types", - "hyper", "metrics", "once_cell", + "rusoto_core", + "rusoto_s3", "serde", "serde_json", "tempfile", @@ -3080,6 +2916,88 @@ dependencies = [ "syn", ] +[[package]] +name = "rusoto_core" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1db30db44ea73551326269adcf7a2169428a054f14faf9e1768f2163494f2fa2" +dependencies = [ + "async-trait", + "base64", + "bytes", + "crc32fast", + "futures", + "http", + "hyper", + "hyper-tls", + "lazy_static", + "log", + "rusoto_credential", + "rusoto_signature", + "rustc_version 0.4.0", + "serde", + "serde_json", + "tokio", + "xml-rs", +] + +[[package]] +name = "rusoto_credential" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee0a6c13db5aad6047b6a44ef023dbbc21a056b6dab5be3b79ce4283d5c02d05" +dependencies = [ + "async-trait", + "chrono", + "dirs-next", + "futures", + "hyper", + "serde", + "serde_json", + "shlex", + "tokio", + "zeroize", +] + +[[package]] +name = "rusoto_s3" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7aae4677183411f6b0b412d66194ef5403293917d66e70ab118f07cc24c5b14d" +dependencies = [ + "async-trait", + "bytes", + "futures", + "rusoto_core", + "xml-rs", +] + +[[package]] +name = "rusoto_signature" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5ae95491c8b4847931e291b151127eccd6ff8ca13f33603eb3d0035ecb05272" +dependencies = [ + "base64", + "bytes", + "chrono", + "digest 0.9.0", + "futures", + "hex", + "hmac 0.11.0", + "http", + "hyper", + "log", + "md-5 0.9.1", + "percent-encoding", + "pin-project-lite", + "rusoto_credential", + "rustc_version 0.4.0", + "serde", + "sha2 0.9.9", + "tokio", +] + [[package]] name = "rustc-demangle" version = "0.1.21" @@ -3131,18 +3049,6 @@ dependencies = [ "webpki", ] -[[package]] -name = "rustls-native-certs" -version = "0.6.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0167bac7a9f490495f3c33013e7722b53cb087ecbe082fb0c6387c96f634ea50" -dependencies = [ - "openssl-probe", - "rustls-pemfile", - "schannel", - "security-framework", -] - [[package]] name = "rustls-pemfile" version = "1.0.1" @@ -3384,14 +3290,16 @@ dependencies = [ ] [[package]] -name = "sha1" -version = "0.10.5" +name = "sha2" +version = "0.9.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f04293dc80c3993519f2d7f6f511707ee7094fe0c6d3406feb330cdb3540eba3" +checksum = "4d58a1e1bf39749807d89cf2d98ac2dfa0ff1cb3faa38fbb64dd88ac8013d800" dependencies = [ + "block-buffer 0.9.0", "cfg-if", "cpufeatures", - "digest", + "digest 0.9.0", + "opaque-debug", ] [[package]] @@ -3402,7 +3310,7 @@ checksum = "82e6b795fe2e3b1e845bafcb27aa35405c4d47cdfc92af5fc8d3002f76cebdc0" dependencies = [ "cfg-if", "cpufeatures", - "digest", + "digest 0.10.5", ] [[package]] @@ -3791,6 +3699,16 @@ dependencies = [ "syn", ] +[[package]] +name = "tokio-native-tls" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7d995660bd2b7f8c1568414c1126076c13fbb725c40112dc0120b78eb9b717b" +dependencies = [ + "native-tls", + "tokio", +] + [[package]] name = "tokio-postgres" version = "0.7.6" @@ -4128,12 +4046,6 @@ dependencies = [ "percent-encoding", ] -[[package]] -name = "urlencoding" -version = "2.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8db7427f936968176eaa7cdf81b7f98b980b18495ec28f1b5791ac3bfe3eea9" - [[package]] name = "utils" version = "0.1.0" @@ -4201,6 +4113,12 @@ version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "77439c1b53d2303b20d9459b1ade71a83c716e3f9c34f3228c00e6f185d6c002" +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + [[package]] name = "version_check" version = "0.9.4" @@ -4466,6 +4384,7 @@ dependencies = [ "ahash", "anyhow", "bytes", + "chrono", "clap 4.0.15", "crossbeam-utils", "either", @@ -4494,7 +4413,6 @@ dependencies = [ "time 0.3.15", "tokio", "tokio-util", - "tower", "tracing", "tracing-core", ] @@ -4527,10 +4445,10 @@ dependencies = [ ] [[package]] -name = "xmlparser" -version = "0.13.5" +name = "xml-rs" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d25c75bf9ea12c4040a97f829154768bbbce366287e2dc044af160cd79a13fd" +checksum = "d2d7d3948613f75c98fd9328cfdcc45acc4d360655289d0a7d4ec931392200a3" [[package]] name = "yasna" diff --git a/Makefile b/Makefile index 6e8b659171..4711dc1c7d 100644 --- a/Makefile +++ b/Makefile @@ -20,18 +20,18 @@ else $(error Bad build type '$(BUILD_TYPE)', see Makefile for options) endif -# Seccomp BPF is only available for Linux UNAME_S := $(shell uname -s) ifeq ($(UNAME_S),Linux) + # Seccomp BPF is only available for Linux PG_CONFIGURE_OPTS += --with-libseccomp -endif - -# macOS with brew-installed openssl requires explicit paths -# It can be configured with OPENSSL_PREFIX variable -UNAME_S := $(shell uname -s) -ifeq ($(UNAME_S),Darwin) - OPENSSL_PREFIX ?= $(shell brew --prefix openssl@3) - PG_CONFIGURE_OPTS += --with-includes=$(OPENSSL_PREFIX)/include --with-libraries=$(OPENSSL_PREFIX)/lib +else ifeq ($(UNAME_S),Darwin) + # macOS with brew-installed openssl requires explicit paths + # It can be configured with OPENSSL_PREFIX variable + OPENSSL_PREFIX ?= $(shell brew --prefix openssl@3) + PG_CONFIGURE_OPTS += --with-includes=$(OPENSSL_PREFIX)/include --with-libraries=$(OPENSSL_PREFIX)/lib + # macOS already has bison and flex in the system, but they are old and result in postgres-v14 target failure + # brew formulae are keg-only and not symlinked into HOMEBREW_PREFIX, force their usage + EXTRA_PATH_OVERRIDES += $(shell brew --prefix bison)/bin/:$(shell brew --prefix flex)/bin/: endif # Use -C option so that when PostgreSQL "make install" installs the @@ -73,7 +73,8 @@ $(POSTGRES_INSTALL_DIR)/build/v14/config.status: +@echo "Configuring Postgres v14 build" mkdir -p $(POSTGRES_INSTALL_DIR)/build/v14 (cd $(POSTGRES_INSTALL_DIR)/build/v14 && \ - $(ROOT_PROJECT_DIR)/vendor/postgres-v14/configure CFLAGS='$(PG_CFLAGS)' \ + env PATH="$(EXTRA_PATH_OVERRIDES):$$PATH" $(ROOT_PROJECT_DIR)/vendor/postgres-v14/configure \ + CFLAGS='$(PG_CFLAGS)' \ $(PG_CONFIGURE_OPTS) \ --prefix=$(abspath $(POSTGRES_INSTALL_DIR))/v14 > configure.log) @@ -81,7 +82,8 @@ $(POSTGRES_INSTALL_DIR)/build/v15/config.status: +@echo "Configuring Postgres v15 build" mkdir -p $(POSTGRES_INSTALL_DIR)/build/v15 (cd $(POSTGRES_INSTALL_DIR)/build/v15 && \ - $(ROOT_PROJECT_DIR)/vendor/postgres-v15/configure CFLAGS='$(PG_CFLAGS)' \ + env PATH="$(EXTRA_PATH_OVERRIDES):$$PATH" $(ROOT_PROJECT_DIR)/vendor/postgres-v15/configure \ + CFLAGS='$(PG_CFLAGS)' \ $(PG_CONFIGURE_OPTS) \ --prefix=$(abspath $(POSTGRES_INSTALL_DIR))/v15 > configure.log) @@ -111,6 +113,8 @@ postgres-v14: postgres-v14-configure \ $(MAKE) -C $(POSTGRES_INSTALL_DIR)/build/v14 MAKELEVEL=0 install +@echo "Compiling libpq v14" $(MAKE) -C $(POSTGRES_INSTALL_DIR)/build/v14/src/interfaces/libpq install + +@echo "Compiling pg_prewarm v14" + $(MAKE) -C $(POSTGRES_INSTALL_DIR)/build/v14/contrib/pg_prewarm install +@echo "Compiling pg_buffercache v14" $(MAKE) -C $(POSTGRES_INSTALL_DIR)/build/v14/contrib/pg_buffercache install +@echo "Compiling pageinspect v14" @@ -123,6 +127,8 @@ postgres-v15: postgres-v15-configure \ $(MAKE) -C $(POSTGRES_INSTALL_DIR)/build/v15 MAKELEVEL=0 install +@echo "Compiling libpq v15" $(MAKE) -C $(POSTGRES_INSTALL_DIR)/build/v15/src/interfaces/libpq install + +@echo "Compiling pg_prewarm v15" + $(MAKE) -C $(POSTGRES_INSTALL_DIR)/build/v15/contrib/pg_prewarm install +@echo "Compiling pg_buffercache v15" $(MAKE) -C $(POSTGRES_INSTALL_DIR)/build/v15/contrib/pg_buffercache install +@echo "Compiling pageinspect v15" diff --git a/README.md b/README.md index e9c30668e0..770c24d11f 100644 --- a/README.md +++ b/README.md @@ -53,7 +53,7 @@ curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh 1. Install XCode and dependencies ``` xcode-select --install -brew install protobuf etcd openssl +brew install protobuf etcd openssl flex bison ``` 2. [Install Rust](https://www.rust-lang.org/tools/install) @@ -125,24 +125,23 @@ Python (3.9 or higher), and install python3 packages using `./scripts/pysync` (r # Create repository in .neon with proper paths to binaries and data # Later that would be responsibility of a package install script > ./target/debug/neon_local init -Starting pageserver at '127.0.0.1:64000' in '.neon' - -Pageserver started -Successfully initialized timeline 7dd0907914ac399ff3be45fb252bfdb7 -Stopping pageserver gracefully...done! +Starting pageserver at '127.0.0.1:64000' in '.neon'. +pageserver started, pid: 2545906 +Successfully initialized timeline de200bd42b49cc1814412c7e592dd6e9 +Stopped pageserver 1 process with pid 2545906 # start pageserver and safekeeper > ./target/debug/neon_local start -Starting etcd broker using /usr/bin/etcd -Starting pageserver at '127.0.0.1:64000' in '.neon' - -Pageserver started -Starting safekeeper at '127.0.0.1:5454' in '.neon/safekeepers/sk1' -Safekeeper started +Starting etcd broker using "/usr/bin/etcd" +etcd started, pid: 2545996 +Starting pageserver at '127.0.0.1:64000' in '.neon'. +pageserver started, pid: 2546005 +Starting safekeeper at '127.0.0.1:5454' in '.neon/safekeepers/sk1'. +safekeeper 1 started, pid: 2546041 # start postgres compute node > ./target/debug/neon_local pg start main -Starting new postgres main on timeline de200bd42b49cc1814412c7e592dd6e9 ... +Starting new postgres (v14) main on timeline de200bd42b49cc1814412c7e592dd6e9 ... Extracting base backup to create postgres instance: path=.neon/pgdatadirs/tenants/9ef87a5bf0d92544f6fafeeb3239695c/main port=55432 Starting postgres node at 'host=127.0.0.1 port=55432 user=cloud_admin dbname=postgres' diff --git a/control_plane/src/background_process.rs b/control_plane/src/background_process.rs index 2f8098b7c9..c558c09854 100644 --- a/control_plane/src/background_process.rs +++ b/control_plane/src/background_process.rs @@ -26,8 +26,18 @@ use nix::unistd::Pid; use utils::lock_file; -const RETRIES: u32 = 15; -const RETRY_TIMEOUT_MILLIS: u64 = 500; +// These constants control the loop used to poll for process start / stop. +// +// The loop waits for at most 10 seconds, polling every 100 ms. +// Once a second, it prints a dot ("."), to give the user an indication that +// it's waiting. If the process hasn't started/stopped after 5 seconds, +// it prints a notice that it's taking long, but keeps waiting. +// +const RETRY_UNTIL_SECS: u64 = 10; +const RETRIES: u64 = (RETRY_UNTIL_SECS * 1000) / RETRY_INTERVAL_MILLIS; +const RETRY_INTERVAL_MILLIS: u64 = 100; +const DOT_EVERY_RETRIES: u64 = 10; +const NOTICE_AFTER_RETRIES: u64 = 50; /// Argument to `start_process`, to indicate whether it should create pidfile or if the process creates /// it itself. @@ -107,16 +117,16 @@ where return Ok(spawned_process); } Ok(false) => { - if retries < 5 { + if retries == NOTICE_AFTER_RETRIES { + // The process is taking a long time to start up. Keep waiting, but + // print a message + print!("\n{process_name} has not started yet, continuing to wait"); + } + if retries % DOT_EVERY_RETRIES == 0 { print!("."); io::stdout().flush().unwrap(); - } else { - if retries == 5 { - println!() // put a line break after dots for second message - } - println!("{process_name} has not started yet, retrying ({retries})..."); } - thread::sleep(Duration::from_millis(RETRY_TIMEOUT_MILLIS)); + thread::sleep(Duration::from_millis(RETRY_INTERVAL_MILLIS)); } Err(e) => { println!("{process_name} failed to start: {e:#}"); @@ -127,7 +137,8 @@ where } } } - anyhow::bail!("{process_name} could not start in {RETRIES} attempts"); + println!(); + anyhow::bail!("{process_name} did not start in {RETRY_UNTIL_SECS} seconds"); } /// Stops the process, using the pid file given. Returns Ok also if the process is already not running. @@ -158,7 +169,7 @@ pub fn stop_process(immediate: bool, process_name: &str, pid_file: &Path) -> any } // Wait until process is gone - for _ in 0..RETRIES { + for retries in 0..RETRIES { match process_has_stopped(pid) { Ok(true) => { println!("\n{process_name} stopped"); @@ -170,9 +181,16 @@ pub fn stop_process(immediate: bool, process_name: &str, pid_file: &Path) -> any return Ok(()); } Ok(false) => { - print!("."); - io::stdout().flush().unwrap(); - thread::sleep(Duration::from_secs(1)) + if retries == NOTICE_AFTER_RETRIES { + // The process is taking a long time to start up. Keep waiting, but + // print a message + print!("\n{process_name} has not stopped yet, continuing to wait"); + } + if retries % DOT_EVERY_RETRIES == 0 { + print!("."); + io::stdout().flush().unwrap(); + } + thread::sleep(Duration::from_millis(RETRY_INTERVAL_MILLIS)); } Err(e) => { println!("{process_name} with pid {pid} failed to stop: {e:#}"); @@ -180,8 +198,8 @@ pub fn stop_process(immediate: bool, process_name: &str, pid_file: &Path) -> any } } } - - anyhow::bail!("{process_name} with pid {pid} failed to stop in {RETRIES} attempts"); + println!(); + anyhow::bail!("{process_name} with pid {pid} did not stop in {RETRY_UNTIL_SECS} seconds"); } fn fill_rust_env_vars(cmd: &mut Command) -> &mut Command { diff --git a/control_plane/src/etcd.rs b/control_plane/src/etcd.rs index 4c15914e24..60aa5da780 100644 --- a/control_plane/src/etcd.rs +++ b/control_plane/src/etcd.rs @@ -6,7 +6,7 @@ use crate::{background_process, local_env}; pub fn start_etcd_process(env: &local_env::LocalEnv) -> anyhow::Result<()> { let etcd_broker = &env.etcd_broker; - println!( + print!( "Starting etcd broker using {:?}", etcd_broker.etcd_binary_path ); diff --git a/control_plane/src/pageserver.rs b/control_plane/src/pageserver.rs index 18d6aee68d..aec6f5bc2c 100644 --- a/control_plane/src/pageserver.rs +++ b/control_plane/src/pageserver.rs @@ -237,7 +237,7 @@ impl PageServerNode { datadir: &Path, update_config: bool, ) -> anyhow::Result { - println!( + print!( "Starting pageserver at '{}' in '{}'", self.pg_connection_config.raw_address(), datadir.display() diff --git a/docs/sourcetree.md b/docs/sourcetree.md index 4b4efcecd7..4ea83dd068 100644 --- a/docs/sourcetree.md +++ b/docs/sourcetree.md @@ -83,6 +83,16 @@ A subject for future modularization. `/libs/metrics`: Helpers for exposing Prometheus metrics from the server. +### Adding dependencies +When you add a Cargo dependency, you should update hakari manifest by running commands below and committing the updated `Cargo.lock` and `workspace_hack/`. There may be no changes, that's fine. + +```bash +cargo hakari generate +cargo hakari manage-deps +``` + +If you don't have hakari installed (`error: no such subcommand: hakari`), install it by running `cargo install cargo-hakari`. + ## Using Python Note that Debian/Ubuntu Python packages are stale, as it commonly happens, so manual installation of dependencies is not recommended. diff --git a/libs/remote_storage/Cargo.toml b/libs/remote_storage/Cargo.toml index 85a1104c46..f54d91905c 100644 --- a/libs/remote_storage/Cargo.toml +++ b/libs/remote_storage/Cargo.toml @@ -9,11 +9,8 @@ async-trait = "0.1" metrics = { version = "0.1", path = "../metrics" } utils = { version = "0.1", path = "../utils" } once_cell = "1.13.0" -aws-smithy-http = "0.51.0" -aws-types = "0.51.0" -aws-config = { version = "0.51.0", default-features = false } -aws-sdk-s3 = "0.21.0" -hyper = { version = "0.14", features = ["stream"] } +rusoto_core = "0.48" +rusoto_s3 = "0.48" serde = { version = "1.0", features = ["derive"] } serde_json = "1" tokio = { version = "1.17", features = ["sync", "macros", "fs", "io-util"] } diff --git a/libs/remote_storage/src/s3_bucket.rs b/libs/remote_storage/src/s3_bucket.rs index 3869ef8557..74632430cd 100644 --- a/libs/remote_storage/src/s3_bucket.rs +++ b/libs/remote_storage/src/s3_bucket.rs @@ -5,32 +5,27 @@ //! their bucket prefixes are both specified and different. use std::path::{Path, PathBuf}; -use std::sync::Arc; use anyhow::Context; -use aws_config::{ - environment::credentials::EnvironmentVariableCredentialsProvider, - imds::credentials::ImdsCredentialsProvider, meta::credentials::CredentialsProviderChain, +use rusoto_core::{ + credential::{InstanceMetadataProvider, StaticProvider}, + HttpClient, Region, RusotoError, }; -use aws_sdk_s3::{ - config::Config, - error::{GetObjectError, GetObjectErrorKind}, - types::{ByteStream, SdkError}, - Client, Endpoint, Region, +use rusoto_s3::{ + DeleteObjectRequest, GetObjectError, GetObjectRequest, ListObjectsV2Request, PutObjectRequest, + S3Client, StreamingBody, S3, }; -use aws_smithy_http::body::SdkBody; -use aws_types::credentials::SharedCredentialsProvider; -use hyper::Body; use tokio::{io, sync::Semaphore}; use tokio_util::io::ReaderStream; use tracing::debug; -use super::StorageMetadata; use crate::{ strip_path_prefix, Download, DownloadError, RemoteObjectId, RemoteStorage, S3Config, REMOTE_STORAGE_PREFIX_SEPARATOR, }; +use super::StorageMetadata; + pub(super) mod metrics { use metrics::{register_int_counter_vec, IntCounterVec}; use once_cell::sync::Lazy; @@ -121,7 +116,7 @@ fn download_destination( /// AWS S3 storage. pub struct S3Bucket { workdir: PathBuf, - client: Client, + client: S3Client, bucket_name: String, prefix_in_bucket: Option, // Every request to S3 can be throttled or cancelled, if a certain number of requests per second is exceeded. @@ -130,13 +125,6 @@ pub struct S3Bucket { concurrency_limiter: Semaphore, } -#[derive(Default)] -struct GetObjectRequest { - bucket: String, - key: String, - range: Option, -} - impl S3Bucket { /// Creates the S3 storage, errors if incorrect AWS S3 configuration provided. pub fn new(aws_config: &S3Config, workdir: PathBuf) -> anyhow::Result { @@ -144,25 +132,43 @@ impl S3Bucket { "Creating s3 remote storage for S3 bucket {}", aws_config.bucket_name ); - let provider = CredentialsProviderChain::first_try( - "Environment", - EnvironmentVariableCredentialsProvider::new(), - ) - .or_else("IAM", ImdsCredentialsProvider::builder().build()); + let region = match aws_config.endpoint.clone() { + Some(custom_endpoint) => Region::Custom { + name: aws_config.bucket_region.clone(), + endpoint: custom_endpoint, + }, + None => aws_config + .bucket_region + .parse::() + .context("Failed to parse the s3 region from config")?, + }; + let request_dispatcher = HttpClient::new().context("Failed to create S3 http client")?; - let mut config_builder = Config::builder() - .region(Region::new(aws_config.bucket_region.clone())) - .credentials_provider(SharedCredentialsProvider::new(provider)); + let access_key_id = std::env::var("AWS_ACCESS_KEY_ID").ok(); + let secret_access_key = std::env::var("AWS_SECRET_ACCESS_KEY").ok(); + // session token is used when authorizing through sso + // which is typically the case when testing locally on developer machine + let session_token = std::env::var("AWS_SESSION_TOKEN").ok(); - if let Some(custom_endpoint) = aws_config.endpoint.clone() { - let endpoint = Endpoint::immutable( - custom_endpoint - .parse() - .expect("Failed to parse S3 custom endpoint"), + let client = if access_key_id.is_none() && secret_access_key.is_none() { + debug!("Using IAM-based AWS access"); + S3Client::new_with(request_dispatcher, InstanceMetadataProvider::new(), region) + } else { + debug!( + "Using credentials-based AWS access. Session token is set: {}", + session_token.is_some() ); - config_builder.set_endpoint_resolver(Some(Arc::new(endpoint))); - } - let client = Client::from_conf(config_builder.build()); + S3Client::new_with( + request_dispatcher, + StaticProvider::new( + access_key_id.unwrap_or_default(), + secret_access_key.unwrap_or_default(), + session_token, + None, + ), + region, + ) + }; let prefix_in_bucket = aws_config.prefix_in_bucket.as_deref().map(|prefix| { let mut prefix = prefix; @@ -196,33 +202,20 @@ impl S3Bucket { metrics::inc_get_object(); - let get_object = self - .client - .get_object() - .bucket(request.bucket) - .key(request.key) - .set_range(request.range) - .send() - .await; - - match get_object { - Ok(object_output) => { - let metadata = object_output.metadata().cloned().map(StorageMetadata); - Ok(Download { - metadata, - download_stream: Box::pin(io::BufReader::new( - object_output.body.into_async_read(), - )), - }) - } - Err(SdkError::ServiceError { - err: - GetObjectError { - kind: GetObjectErrorKind::NoSuchKey(..), - .. - }, - .. - }) => Err(DownloadError::NotFound), + match self.client.get_object(request).await { + Ok(object_output) => match object_output.body { + None => { + metrics::inc_get_object_fail(); + Err(DownloadError::Other(anyhow::anyhow!( + "Got no body for the S3 object given" + ))) + } + Some(body) => Ok(Download { + metadata: object_output.metadata.map(StorageMetadata), + download_stream: Box::pin(io::BufReader::new(body.into_async_read())), + }), + }, + Err(RusotoError::Service(GetObjectError::NoSuchKey(_))) => Err(DownloadError::NotFound), Err(e) => { metrics::inc_get_object_fail(); Err(DownloadError::Other(anyhow::anyhow!( @@ -268,11 +261,12 @@ impl RemoteStorage for S3Bucket { let fetch_response = self .client - .list_objects_v2() - .bucket(self.bucket_name.clone()) - .set_prefix(self.prefix_in_bucket.clone()) - .set_continuation_token(continuation_token) - .send() + .list_objects_v2(ListObjectsV2Request { + bucket: self.bucket_name.clone(), + prefix: self.prefix_in_bucket.clone(), + continuation_token, + ..ListObjectsV2Request::default() + }) .await .map_err(|e| { metrics::inc_list_objects_fail(); @@ -328,12 +322,13 @@ impl RemoteStorage for S3Bucket { let fetch_response = self .client - .list_objects_v2() - .bucket(self.bucket_name.clone()) - .set_prefix(list_prefix.clone()) - .set_continuation_token(continuation_token) - .delimiter(REMOTE_STORAGE_PREFIX_SEPARATOR.to_string()) - .send() + .list_objects_v2(ListObjectsV2Request { + bucket: self.bucket_name.clone(), + prefix: list_prefix.clone(), + continuation_token, + delimiter: Some(REMOTE_STORAGE_PREFIX_SEPARATOR.to_string()), + ..ListObjectsV2Request::default() + }) .await .map_err(|e| { metrics::inc_list_objects_fail(); @@ -371,18 +366,17 @@ impl RemoteStorage for S3Bucket { .context("Concurrency limiter semaphore got closed during S3 upload")?; metrics::inc_put_object(); - - let body = Body::wrap_stream(ReaderStream::new(from)); - let bytes_stream = ByteStream::new(SdkBody::from(body)); - self.client - .put_object() - .bucket(self.bucket_name.clone()) - .key(to.0.to_owned()) - .set_metadata(metadata.map(|m| m.0)) - .content_length(from_size_bytes.try_into()?) - .body(bytes_stream) - .send() + .put_object(PutObjectRequest { + body: Some(StreamingBody::new_with_size( + ReaderStream::new(from), + from_size_bytes, + )), + bucket: self.bucket_name.clone(), + key: to.0.to_owned(), + metadata: metadata.map(|m| m.0), + ..PutObjectRequest::default() + }) .await .map_err(|e| { metrics::inc_put_object_fail(); @@ -418,6 +412,7 @@ impl RemoteStorage for S3Bucket { bucket: self.bucket_name.clone(), key: from.0.to_owned(), range, + ..GetObjectRequest::default() }) .await } @@ -432,10 +427,11 @@ impl RemoteStorage for S3Bucket { metrics::inc_delete_object(); self.client - .delete_object() - .bucket(self.bucket_name.clone()) - .key(remote_object_id.0.to_owned()) - .send() + .delete_object(DeleteObjectRequest { + bucket: self.bucket_name.clone(), + key: remote_object_id.0.to_owned(), + ..DeleteObjectRequest::default() + }) .await .map_err(|e| { metrics::inc_delete_object_fail(); @@ -604,7 +600,7 @@ mod tests { fn dummy_storage(workdir: PathBuf) -> S3Bucket { S3Bucket { workdir, - client: Client::new(&aws_config::SdkConfig::builder().build()), + client: S3Client::new("us-east-1".parse().unwrap()), bucket_name: "dummy-bucket".to_string(), prefix_in_bucket: Some("dummy_prefix/".to_string()), concurrency_limiter: Semaphore::new(1), diff --git a/pageserver/src/tenant_tasks.rs b/pageserver/src/tenant_tasks.rs index 5a9c5aa3a5..d8a4d5521c 100644 --- a/pageserver/src/tenant_tasks.rs +++ b/pageserver/src/tenant_tasks.rs @@ -71,7 +71,7 @@ async fn compaction_loop(tenant_id: TenantId) { let mut sleep_duration = tenant.get_compaction_period(); if let Err(e) = tenant.compaction_iteration() { sleep_duration = wait_duration; - error!("Compaction failed, retrying in {:?}: {e:#}", sleep_duration); + error!("Compaction failed, retrying in {:?}: {e:?}", sleep_duration); } // Sleep @@ -120,7 +120,7 @@ async fn gc_loop(tenant_id: TenantId) { if let Err(e) = tenant.gc_iteration(None, gc_horizon, tenant.get_pitr_interval(), false).await { sleep_duration = wait_duration; - error!("Gc failed, retrying in {:?}: {e:#}", sleep_duration); + error!("Gc failed, retrying in {:?}: {e:?}", sleep_duration); } } diff --git a/pgxn/neon/libpagestore.c b/pgxn/neon/libpagestore.c index d8e9d8b52c..df92a1e2f4 100644 --- a/pgxn/neon/libpagestore.c +++ b/pgxn/neon/libpagestore.c @@ -32,11 +32,6 @@ #define PageStoreTrace DEBUG5 -#define NEON_TAG "[NEON_SMGR] " -#define neon_log(tag, fmt, ...) ereport(tag, \ - (errmsg(NEON_TAG fmt, ##__VA_ARGS__), \ - errhidestmt(true), errhidecontext(true))) - bool connected = false; PGconn *pageserver_conn = NULL; @@ -97,11 +92,10 @@ pageserver_connect() while (PQisBusy(pageserver_conn)) { - int wc; WaitEvent event; /* Sleep until there's something to do */ - wc = WaitEventSetWait(pageserver_conn_wes, -1L, &event, 1, PG_WAIT_EXTENSION); + (void) WaitEventSetWait(pageserver_conn_wes, -1L, &event, 1, PG_WAIT_EXTENSION); ResetLatch(MyLatch); CHECK_FOR_INTERRUPTS(); @@ -141,11 +135,10 @@ retry: if (ret == 0) { - int wc; WaitEvent event; /* Sleep until there's something to do */ - wc = WaitEventSetWait(pageserver_conn_wes, -1L, &event, 1, PG_WAIT_EXTENSION); + (void) WaitEventSetWait(pageserver_conn_wes, -1L, &event, 1, PG_WAIT_EXTENSION); ResetLatch(MyLatch); CHECK_FOR_INTERRUPTS(); @@ -239,6 +232,9 @@ pageserver_receive(void) StringInfoData resp_buff; NeonResponse *resp; + if (!connected) + return NULL; + PG_TRY(); { /* read response */ @@ -248,7 +244,10 @@ pageserver_receive(void) if (resp_buff.len < 0) { if (resp_buff.len == -1) - neon_log(ERROR, "end of COPY"); + { + pageserver_disconnect(); + return NULL; + } else if (resp_buff.len == -2) neon_log(ERROR, "could not read COPY data: %s", PQerrorMessage(pageserver_conn)); } diff --git a/pgxn/neon/pagestore_client.h b/pgxn/neon/pagestore_client.h index 9b8081065c..170a0cb72d 100644 --- a/pgxn/neon/pagestore_client.h +++ b/pgxn/neon/pagestore_client.h @@ -49,6 +49,11 @@ typedef struct #define messageTag(m) (((const NeonMessage *)(m))->tag) +#define NEON_TAG "[NEON_SMGR] " +#define neon_log(tag, fmt, ...) ereport(tag, \ + (errmsg(NEON_TAG fmt, ##__VA_ARGS__), \ + errhidestmt(true), errhidecontext(true))) + /* * supertype of all the Neon*Request structs below * diff --git a/pgxn/neon/pagestore_smgr.c b/pgxn/neon/pagestore_smgr.c index d6fa7c46c9..21067fb56f 100644 --- a/pgxn/neon/pagestore_smgr.c +++ b/pgxn/neon/pagestore_smgr.c @@ -251,9 +251,9 @@ XLogRecPtr prefetch_lsn = 0; static void consume_prefetch_responses(void); static uint64 prefetch_register_buffer(BufferTag tag, bool *force_latest, XLogRecPtr *force_lsn); -static void prefetch_read(PrefetchRequest *slot); +static bool prefetch_read(PrefetchRequest *slot); static void prefetch_do_request(PrefetchRequest *slot, bool *force_latest, XLogRecPtr *force_lsn); -static void prefetch_wait_for(uint64 ring_index); +static bool prefetch_wait_for(uint64 ring_index); static void prefetch_cleanup(void); static inline void prefetch_set_unused(uint64 ring_index); @@ -393,7 +393,7 @@ prefetch_cleanup(void) * NOTE: this function may indirectly update MyPState->pfs_hash; which * invalidates any active pointers into the hash table. */ -static void +static bool prefetch_wait_for(uint64 ring_index) { PrefetchRequest *entry; @@ -412,8 +412,10 @@ prefetch_wait_for(uint64 ring_index) entry = GetPrfSlot(MyPState->ring_receive); Assert(entry->status == PRFS_REQUESTED); - prefetch_read(entry); + if (!prefetch_read(entry)) + return false; } + return true; } /* @@ -425,7 +427,7 @@ prefetch_wait_for(uint64 ring_index) * NOTE: this function may indirectly update MyPState->pfs_hash; which * invalidates any active pointers into the hash table. */ -static void +static bool prefetch_read(PrefetchRequest *slot) { NeonResponse *response; @@ -438,15 +440,22 @@ prefetch_read(PrefetchRequest *slot) old = MemoryContextSwitchTo(MyPState->errctx); response = (NeonResponse *) page_server->receive(); MemoryContextSwitchTo(old); - - /* update prefetch state */ - MyPState->n_responses_buffered += 1; - MyPState->n_requests_inflight -= 1; - MyPState->ring_receive += 1; + if (response) + { + /* update prefetch state */ + MyPState->n_responses_buffered += 1; + MyPState->n_requests_inflight -= 1; + MyPState->ring_receive += 1; - /* update slot state */ - slot->status = PRFS_RECEIVED; - slot->response = response; + /* update slot state */ + slot->status = PRFS_RECEIVED; + slot->response = response; + return true; + } + else + { + return false; + } } /* @@ -746,11 +755,16 @@ prefetch_register_buffer(BufferTag tag, bool *force_latest, XLogRecPtr *force_ls static NeonResponse * page_server_request(void const *req) { - page_server->send((NeonRequest *) req); - page_server->flush(); - MyPState->ring_flush = MyPState->ring_unused; - consume_prefetch_responses(); - return page_server->receive(); + NeonResponse* resp; + do { + page_server->send((NeonRequest *) req); + page_server->flush(); + MyPState->ring_flush = MyPState->ring_unused; + consume_prefetch_responses(); + resp = page_server->receive(); + } while (resp == NULL); + return resp; + } @@ -1635,7 +1649,8 @@ neon_close(SMgrRelation reln, ForkNumber forknum) bool neon_prefetch(SMgrRelation reln, ForkNumber forknum, BlockNumber blocknum) { - uint64 ring_index PG_USED_FOR_ASSERTS_ONLY; + BufferTag tag; + uint64 ring_index PG_USED_FOR_ASSERTS_ONLY; switch (reln->smgr_relpersistence) { @@ -1651,7 +1666,7 @@ neon_prefetch(SMgrRelation reln, ForkNumber forknum, BlockNumber blocknum) elog(ERROR, "unknown relpersistence '%c'", reln->smgr_relpersistence); } - BufferTag tag = (BufferTag) { + tag = (BufferTag) { .rnode = reln->smgr_rnode.node, .forkNum = forknum, .blockNum = blocknum @@ -1755,22 +1770,24 @@ neon_read_at_lsn(RelFileNode rnode, ForkNumber forkNum, BlockNumber blkno, } } - if (entry == NULL) + do { - n_prefetch_misses += 1; + if (entry == NULL) + { + n_prefetch_misses += 1; - ring_index = prefetch_register_buffer(buftag, &request_latest, - &request_lsn); - slot = GetPrfSlot(ring_index); - } + ring_index = prefetch_register_buffer(buftag, &request_latest, + &request_lsn); + slot = GetPrfSlot(ring_index); + } - Assert(slot->my_ring_index == ring_index); - Assert(MyPState->ring_last <= ring_index && - MyPState->ring_unused > ring_index); - Assert(slot->status != PRFS_UNUSED); - Assert(GetPrfSlot(ring_index) == slot); + Assert(slot->my_ring_index == ring_index); + Assert(MyPState->ring_last <= ring_index && + MyPState->ring_unused > ring_index); + Assert(slot->status != PRFS_UNUSED); + Assert(GetPrfSlot(ring_index) == slot); - prefetch_wait_for(ring_index); + } while (!prefetch_wait_for(ring_index)); Assert(slot->status == PRFS_RECEIVED); diff --git a/pgxn/neon/walproposer.c b/pgxn/neon/walproposer.c index c24142dca1..8323811b0d 100644 --- a/pgxn/neon/walproposer.c +++ b/pgxn/neon/walproposer.c @@ -119,6 +119,7 @@ static TimestampTz last_reconnect_attempt; static WalproposerShmemState * walprop_shared; /* Prototypes for private functions */ +static void WalProposerRegister(void); static void WalProposerInit(XLogRecPtr flushRecPtr, uint64 systemId); static void WalProposerStart(void); static void WalProposerLoop(void); @@ -455,7 +456,7 @@ WalProposerPoll(void) /* * Register a background worker proposing WAL to wal acceptors. */ -void +static void WalProposerRegister(void) { BackgroundWorker bgw; diff --git a/pgxn/neon/walproposer.h b/pgxn/neon/walproposer.h index 3c4f080353..362b194b32 100644 --- a/pgxn/neon/walproposer.h +++ b/pgxn/neon/walproposer.h @@ -377,18 +377,18 @@ typedef struct Safekeeper AppendResponse appendResponse; /* feedback for master */ } Safekeeper; -extern PGDLLIMPORT void WalProposerMain(Datum main_arg); -void WalProposerBroadcast(XLogRecPtr startpos, XLogRecPtr endpos); -void WalProposerPoll(void); -void WalProposerRegister(void); -void ParseReplicationFeedbackMessage(StringInfo reply_message, - ReplicationFeedback * rf); +extern void WalProposerSync(int argc, char *argv[]); +extern void WalProposerMain(Datum main_arg); +extern void WalProposerBroadcast(XLogRecPtr startpos, XLogRecPtr endpos); +extern void WalProposerPoll(void); +extern void ParseReplicationFeedbackMessage(StringInfo reply_message, + ReplicationFeedback *rf); extern void StartProposerReplication(StartReplicationCmd *cmd); -Size WalproposerShmemSize(void); -bool WalproposerShmemInit(void); -void replication_feedback_set(ReplicationFeedback * rf); -void replication_feedback_get_lsns(XLogRecPtr *writeLsn, XLogRecPtr *flushLsn, XLogRecPtr *applyLsn); +extern Size WalproposerShmemSize(void); +extern bool WalproposerShmemInit(void); +extern void replication_feedback_set(ReplicationFeedback *rf); +extern void replication_feedback_get_lsns(XLogRecPtr *writeLsn, XLogRecPtr *flushLsn, XLogRecPtr *applyLsn); /* libpqwalproposer hooks & helper type */ diff --git a/test_runner/fixtures/neon_fixtures.py b/test_runner/fixtures/neon_fixtures.py index 051c140836..236330e16a 100644 --- a/test_runner/fixtures/neon_fixtures.py +++ b/test_runner/fixtures/neon_fixtures.py @@ -2073,9 +2073,9 @@ class NeonProxy(PgProtocol): self, proxy_port: int, http_port: int, + mgmt_port: int, neon_binpath: Path, auth_endpoint=None, - mgmt_port=None, ): super().__init__(dsn=auth_endpoint, port=proxy_port) self.host = "127.0.0.1" @@ -2099,6 +2099,7 @@ class NeonProxy(PgProtocol): str(self.neon_binpath / "proxy"), *["--http", f"{self.host}:{self.http_port}"], *["--proxy", f"{self.host}:{self.proxy_port}"], + *["--mgmt", f"{self.host}:{self.mgmt_port}"], *["--auth-backend", "postgres"], *["--auth-endpoint", self.auth_endpoint], ] @@ -2175,11 +2176,13 @@ def static_proxy( auth_endpoint = f"postgres://proxy:password@{host}:{port}/{dbname}" proxy_port = port_distributor.get_port() + mgmt_port = port_distributor.get_port() http_port = port_distributor.get_port() with NeonProxy( proxy_port=proxy_port, http_port=http_port, + mgmt_port=mgmt_port, neon_binpath=neon_binpath, auth_endpoint=auth_endpoint, ) as proxy: diff --git a/test_runner/performance/test_seqscans.py b/test_runner/performance/test_seqscans.py index c681c50ff5..1755c70324 100644 --- a/test_runner/performance/test_seqscans.py +++ b/test_runner/performance/test_seqscans.py @@ -6,6 +6,7 @@ import pytest from fixtures.benchmark_fixture import MetricReport from fixtures.compare_fixtures import PgCompare from fixtures.log_helper import log +from pytest_lazyfixture import lazy_fixture # type: ignore @pytest.mark.parametrize( @@ -20,11 +21,19 @@ from fixtures.log_helper import log pytest.param(10000000, 1, 4), ], ) -def test_seqscans(neon_with_baseline: PgCompare, rows: int, iters: int, workers: int): - env = neon_with_baseline - +@pytest.mark.parametrize( + "env", + [ + # Run on all envs + pytest.param(lazy_fixture("neon_compare"), id="neon"), + pytest.param(lazy_fixture("vanilla_compare"), id="vanilla"), + pytest.param(lazy_fixture("remote_compare"), id="remote", marks=pytest.mark.remote_cluster), + ], +) +def test_seqscans(env: PgCompare, rows: int, iters: int, workers: int): with closing(env.pg.connect()) as conn: with conn.cursor() as cur: + cur.execute("drop table if exists t;") cur.execute("create table t (i integer);") cur.execute(f"insert into t values (generate_series(1,{rows}));") diff --git a/test_runner/regress/test_compatibility.py b/test_runner/regress/test_compatibility.py index 9ad8cd393f..6b3324b7a7 100644 --- a/test_runner/regress/test_compatibility.py +++ b/test_runner/regress/test_compatibility.py @@ -2,7 +2,7 @@ import os import shutil import subprocess from pathlib import Path -from typing import Any +from typing import Any, Optional import pytest import toml # TODO: replace with tomllib for Python >= 3.11 @@ -160,6 +160,7 @@ def test_forward_compatibility( from_dir=compatibility_snapshot_dir, to_dir=test_output_dir / "compatibility_snapshot", port_distributor=port_distributor, + pg_distrib_dir=compatibility_postgres_distrib_dir, ) breaking_changes_allowed = ( @@ -189,7 +190,12 @@ def test_forward_compatibility( ), "Breaking changes are allowed by ALLOW_FORWARD_COMPATIBILITY_BREAKAGE, but the test has passed without any breakage" -def prepare_snapshot(from_dir: Path, to_dir: Path, port_distributor: PortDistributor): +def prepare_snapshot( + from_dir: Path, + to_dir: Path, + port_distributor: PortDistributor, + pg_distrib_dir: Optional[Path] = None, +): assert from_dir.exists(), f"Snapshot '{from_dir}' doesn't exist" assert (from_dir / "repo").exists(), f"Snapshot '{from_dir}' doesn't contain a repo directory" assert (from_dir / "dump.sql").exists(), f"Snapshot '{from_dir}' doesn't contain a dump.sql" @@ -214,7 +220,7 @@ def prepare_snapshot(from_dir: Path, to_dir: Path, port_distributor: PortDistrib # Update paths and ports in config files pageserver_toml = repo_dir / "pageserver.toml" pageserver_config = toml.load(pageserver_toml) - pageserver_config["remote_storage"]["local_path"] = repo_dir / "local_fs_remote_storage" + pageserver_config["remote_storage"]["local_path"] = str(repo_dir / "local_fs_remote_storage") pageserver_config["listen_http_addr"] = port_distributor.replace_with_new_port( pageserver_config["listen_http_addr"] ) @@ -225,6 +231,9 @@ def prepare_snapshot(from_dir: Path, to_dir: Path, port_distributor: PortDistrib port_distributor.replace_with_new_port(ep) for ep in pageserver_config["broker_endpoints"] ] + if pg_distrib_dir: + pageserver_config["pg_distrib_dir"] = str(pg_distrib_dir) + with pageserver_toml.open("w") as f: toml.dump(pageserver_config, f) @@ -244,7 +253,10 @@ def prepare_snapshot(from_dir: Path, to_dir: Path, port_distributor: PortDistrib sk["http_port"] = port_distributor.replace_with_new_port(sk["http_port"]) sk["pg_port"] = port_distributor.replace_with_new_port(sk["pg_port"]) - with (snapshot_config_toml).open("w") as f: + if pg_distrib_dir: + snapshot_config["pg_distrib_dir"] = str(pg_distrib_dir) + + with snapshot_config_toml.open("w") as f: toml.dump(snapshot_config, f) # Ensure that snapshot doesn't contain references to the original path diff --git a/test_runner/regress/test_gc_cutoff.py b/test_runner/regress/test_gc_cutoff.py index f760c993f4..1b98a414da 100644 --- a/test_runner/regress/test_gc_cutoff.py +++ b/test_runner/regress/test_gc_cutoff.py @@ -8,6 +8,7 @@ from fixtures.neon_fixtures import NeonEnvBuilder, PgBin # normally restarts after it. Also, there should be GC ERRORs in the log, # but the fixture checks the log for any unexpected ERRORs after every # test anyway, so it doesn't need any special attention here. +@pytest.mark.timeout(600) def test_gc_cutoff(neon_env_builder: NeonEnvBuilder, pg_bin: PgBin): env = neon_env_builder.init_start() @@ -38,7 +39,7 @@ def test_gc_cutoff(neon_env_builder: NeonEnvBuilder, pg_bin: PgBin): for _ in range(5): with pytest.raises(Exception): - pg_bin.run_capture(["pgbench", "-N", "-c5", "-T100", "-Mprepared", connstr]) + pg_bin.run_capture(["pgbench", "-P1", "-N", "-c5", "-T500", "-Mprepared", connstr]) env.pageserver.stop() env.pageserver.start() pageserver_http.configure_failpoints(("after-timeline-gc-removed-layers", "exit")) diff --git a/test_runner/regress/test_proxy.py b/test_runner/regress/test_proxy.py index b4647ebbe9..b8cfb21a5b 100644 --- a/test_runner/regress/test_proxy.py +++ b/test_runner/regress/test_proxy.py @@ -8,11 +8,11 @@ from fixtures.log_helper import log from fixtures.neon_fixtures import PSQL, NeonProxy, VanillaPostgres -def test_proxy_select_1(static_proxy): +def test_proxy_select_1(static_proxy: NeonProxy): static_proxy.safe_psql("select 1", options="project=generic-project-name") -def test_password_hack(static_proxy): +def test_password_hack(static_proxy: NeonProxy): user = "borat" password = "password" static_proxy.safe_psql( @@ -24,7 +24,7 @@ def test_password_hack(static_proxy): static_proxy.safe_psql("select 1", sslsni=0, user=user, password=magic) # Must also check that invalid magic won't be accepted. - with pytest.raises(psycopg2.errors.OperationalError): + with pytest.raises(psycopg2.OperationalError): magic = "broken" static_proxy.safe_psql("select 1", sslsni=0, user=user, password=magic) @@ -135,7 +135,7 @@ async def test_psql_session_id(vanilla_pg: VanillaPostgres, link_proxy: NeonProx # Pass extra options to the server. -def test_proxy_options(static_proxy): +def test_proxy_options(static_proxy: NeonProxy): with static_proxy.connect(options="project=irrelevant -cproxytest.option=value") as conn: with conn.cursor() as cur: cur.execute("SHOW proxytest.option") diff --git a/test_runner/regress/test_tenant_size.py b/test_runner/regress/test_tenant_size.py index 03e7129ff7..d9aed351a5 100644 --- a/test_runner/regress/test_tenant_size.py +++ b/test_runner/regress/test_tenant_size.py @@ -166,6 +166,10 @@ def test_get_tenant_size_with_multiple_branches(neon_env_builder: NeonEnvBuilder env = neon_env_builder.init_start() + # FIXME: we have a race condition between GC and delete timeline. GC might fail with this + # error. Similar to https://github.com/neondatabase/neon/issues/2671 + env.pageserver.allowed_errors.append(".*InternalServerError\\(No such file or directory.*") + tenant_id = env.initial_tenant main_branch_name, main_timeline_id = env.neon_cli.list_timelines(tenant_id)[0] @@ -263,6 +267,8 @@ def test_get_tenant_size_with_multiple_branches(neon_env_builder: NeonEnvBuilder except PageserverApiException as e: # compaction is ok but just retry if this fails; related to #2442 if "cannot lock compaction critical section" in str(e): + # also ignore it in the log + env.pageserver.allowed_errors.append(".*cannot lock compaction critical section.*") time.sleep(1) continue raise diff --git a/workspace_hack/Cargo.toml b/workspace_hack/Cargo.toml index 67f9b19446..2daa08c9b6 100644 --- a/workspace_hack/Cargo.toml +++ b/workspace_hack/Cargo.toml @@ -16,13 +16,14 @@ publish = false ahash = { version = "0.7", features = ["std"] } anyhow = { version = "1", features = ["backtrace", "std"] } bytes = { version = "1", features = ["serde", "std"] } +chrono = { version = "0.4", features = ["clock", "iana-time-zone", "js-sys", "oldtime", "serde", "std", "time", "wasm-bindgen", "wasmbind", "winapi"] } clap = { version = "4", features = ["color", "error-context", "help", "std", "string", "suggestions", "usage"] } crossbeam-utils = { version = "0.8", features = ["once_cell", "std"] } either = { version = "1", features = ["use_std"] } fail = { version = "0.5", default-features = false, features = ["failpoints"] } futures-channel = { version = "0.3", features = ["alloc", "futures-sink", "sink", "std"] } futures-task = { version = "0.3", default-features = false, features = ["alloc", "std"] } -futures-util = { version = "0.3", features = ["alloc", "async-await", "async-await-macro", "channel", "futures-channel", "futures-io", "futures-macro", "futures-sink", "io", "memchr", "sink", "slab", "std"] } +futures-util = { version = "0.3", default-features = false, features = ["alloc", "async-await", "async-await-macro", "channel", "futures-channel", "futures-io", "futures-macro", "futures-sink", "io", "memchr", "sink", "slab", "std"] } hashbrown = { version = "0.12", features = ["ahash", "inline-more", "raw"] } indexmap = { version = "1", default-features = false, features = ["std"] } libc = { version = "0.2", features = ["extra_traits", "std"] } @@ -30,7 +31,7 @@ log = { version = "0.4", default-features = false, features = ["serde", "std"] } memchr = { version = "2", features = ["std"] } nom = { version = "7", features = ["alloc", "std"] } num-bigint = { version = "0.4", features = ["std"] } -num-integer = { version = "0.1", features = ["i128", "std"] } +num-integer = { version = "0.1", default-features = false, features = ["i128", "std"] } num-traits = { version = "0.2", features = ["i128", "libm", "std"] } prost = { version = "0.10", features = ["prost-derive", "std"] } rand = { version = "0.8", features = ["alloc", "getrandom", "libc", "rand_chacha", "rand_hc", "small_rng", "std", "std_rng"] } @@ -43,7 +44,6 @@ stable_deref_trait = { version = "1", features = ["alloc", "std"] } time = { version = "0.3", features = ["alloc", "formatting", "itoa", "macros", "parsing", "std", "time-macros"] } tokio = { version = "1", features = ["bytes", "fs", "io-std", "io-util", "libc", "macros", "memchr", "mio", "net", "num_cpus", "once_cell", "process", "rt", "rt-multi-thread", "signal-hook-registry", "socket2", "sync", "time", "tokio-macros"] } tokio-util = { version = "0.7", features = ["codec", "io", "io-util", "tracing"] } -tower = { version = "0.4", features = ["__common", "balance", "buffer", "discover", "futures-core", "futures-util", "indexmap", "limit", "load", "log", "make", "pin-project", "pin-project-lite", "rand", "ready-cache", "retry", "slab", "timeout", "tokio", "tokio-util", "tracing", "util"] } tracing = { version = "0.1", features = ["attributes", "log", "std", "tracing-attributes"] } tracing-core = { version = "0.1", features = ["once_cell", "std"] }