From e5a397cf9612d32b17a1599ae032b93f68b33380 Mon Sep 17 00:00:00 2001 From: Anastasia Lubennikova Date: Tue, 29 Aug 2023 20:30:24 +0300 Subject: [PATCH 01/10] Form archive_path for remote extensions on the fly --- compute_tools/src/compute.rs | 3 ++- compute_tools/src/http/api.rs | 7 ++++++- libs/compute_api/src/spec.rs | 19 +++++++++++++++---- 3 files changed, 23 insertions(+), 6 deletions(-) diff --git a/compute_tools/src/compute.rs b/compute_tools/src/compute.rs index 595ecb453e..03ae39f79f 100644 --- a/compute_tools/src/compute.rs +++ b/compute_tools/src/compute.rs @@ -1078,7 +1078,8 @@ LIMIT 100", let mut download_tasks = Vec::new(); for library in &libs_vec { - let (ext_name, ext_path) = remote_extensions.get_ext(library, true)?; + let (ext_name, ext_path) = + remote_extensions.get_ext(library, true, &self.build_tag, &self.pgversion)?; download_tasks.push(self.download_extension(ext_name, ext_path)); } let results = join_all(download_tasks).await; diff --git a/compute_tools/src/http/api.rs b/compute_tools/src/http/api.rs index 841e533a3a..a571628770 100644 --- a/compute_tools/src/http/api.rs +++ b/compute_tools/src/http/api.rs @@ -169,7 +169,12 @@ async fn routes(req: Request, compute: &Arc) -> Response anyhow::Result<(String, RemotePath)> { let mut real_ext_name = ext_name; if is_library { @@ -105,10 +107,19 @@ impl RemoteExtSpec { } match self.extension_data.get(real_ext_name) { - Some(ext_data) => Ok(( - real_ext_name.to_string(), - RemotePath::from_string(&ext_data.archive_path)?, - )), + Some(_ext_data) => { + // Construct the path to the extension archive + // BUILD_TAG/PG_MAJOR_VERSION/extensions/EXTENSION_NAME.tar.zst + // + // Keep it in sync with path generation in + // https://github.com/neondatabase/build-custom-extensions/tree/main + let archive_path_str = + format!("{build_tag}/{pg_major_version}/extensions/{real_ext_name}.tar.zst"); + Ok(( + real_ext_name.to_string(), + RemotePath::from_string(&archive_path_str)?, + )) + } None => Err(anyhow::anyhow!( "real_ext_name {} is not found", real_ext_name From 3b81e0c86d755f1127026912c885cca32dad5c0d Mon Sep 17 00:00:00 2001 From: Conrad Ludgate Date: Wed, 30 Aug 2023 15:14:03 +0100 Subject: [PATCH 02/10] chore: remove webpki (#5069) ## Problem webpki is unmaintained Closes https://github.com/neondatabase/neon/security/dependabot/33 ## Summary of changes Update all dependents of webpki. --- Cargo.lock | 423 +++++++++++++++++++----------------- Cargo.toml | 26 +-- proxy/src/http/websocket.rs | 2 +- proxy/src/stream.rs | 2 +- workspace_hack/Cargo.toml | 9 +- 5 files changed, 246 insertions(+), 216 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index c213cd4d0a..867008808b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -221,9 +221,9 @@ checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" [[package]] name = "aws-config" -version = "0.55.3" +version = "0.56.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bcdcf0d683fe9c23d32cf5b53c9918ea0a500375a9fb20109802552658e576c9" +checksum = "de3d533e0263bf453cc80af4c8bcc4d64e2aca293bd16f81633a36f1bf4a97cb" dependencies = [ "aws-credential-types", "aws-http", @@ -236,7 +236,7 @@ dependencies = [ "aws-smithy-types", "aws-types", "bytes", - "fastrand", + "fastrand 2.0.0", "http", "hyper", "time", @@ -247,37 +247,23 @@ dependencies = [ [[package]] name = "aws-credential-types" -version = "0.55.3" +version = "0.56.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fcdb2f7acbc076ff5ad05e7864bdb191ca70a6fd07668dc3a1a8bcd051de5ae" +checksum = "e4834ba01c5ad1ed9740aa222de62190e3c565d11ab7e72cc68314a258994567" dependencies = [ "aws-smithy-async", "aws-smithy-types", - "fastrand", + "fastrand 2.0.0", "tokio", "tracing", "zeroize", ] -[[package]] -name = "aws-endpoint" -version = "0.55.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8cce1c41a6cfaa726adee9ebb9a56fcd2bbfd8be49fd8a04c5e20fd968330b04" -dependencies = [ - "aws-smithy-http", - "aws-smithy-types", - "aws-types", - "http", - "regex", - "tracing", -] - [[package]] name = "aws-http" -version = "0.55.3" +version = "0.56.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aadbc44e7a8f3e71c8b374e03ecd972869eb91dd2bc89ed018954a52ba84bc44" +checksum = "72badf9de83cc7d66b21b004f09241836823b8302afb25a24708769e576a8d8f" dependencies = [ "aws-credential-types", "aws-smithy-http", @@ -293,23 +279,45 @@ dependencies = [ ] [[package]] -name = "aws-sdk-s3" -version = "0.27.0" +name = "aws-runtime" +version = "0.56.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37c77060408d653d3efa6ea7b66c1389bc35a0342352984c8bf8bcb814a8fc27" +checksum = "cf832f522111225c02547e1e1c28137e840e4b082399d93a236e4b29193a4667" dependencies = [ "aws-credential-types", - "aws-endpoint", "aws-http", - "aws-sig-auth", + "aws-sigv4", + "aws-smithy-async", + "aws-smithy-eventstream", + "aws-smithy-http", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-types", + "fastrand 2.0.0", + "http", + "percent-encoding", + "tracing", + "uuid", +] + +[[package]] +name = "aws-sdk-s3" +version = "0.29.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e30370b61599168d38190ad272bb91842cd81870a6ca035c05dd5726d22832c" +dependencies = [ + "aws-credential-types", + "aws-http", + "aws-runtime", "aws-sigv4", "aws-smithy-async", "aws-smithy-checksums", "aws-smithy-client", "aws-smithy-eventstream", "aws-smithy-http", - "aws-smithy-http-tower", "aws-smithy-json", + "aws-smithy-runtime", + "aws-smithy-runtime-api", "aws-smithy-types", "aws-smithy-xml", "aws-types", @@ -320,57 +328,39 @@ dependencies = [ "percent-encoding", "regex", "tokio-stream", - "tower", "tracing", "url", ] [[package]] name = "aws-sdk-sts" -version = "0.28.0" +version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "265fac131fbfc188e5c3d96652ea90ecc676a934e3174eaaee523c6cec040b3b" +checksum = "79e21aa1a5b0853969a1ef96ccfaa8ff5d57c761549786a4d5f86c1902b2586a" dependencies = [ "aws-credential-types", - "aws-endpoint", "aws-http", - "aws-sig-auth", + "aws-runtime", "aws-smithy-async", "aws-smithy-client", "aws-smithy-http", - "aws-smithy-http-tower", "aws-smithy-json", "aws-smithy-query", + "aws-smithy-runtime", + "aws-smithy-runtime-api", "aws-smithy-types", "aws-smithy-xml", "aws-types", - "bytes", "http", "regex", - "tower", - "tracing", -] - -[[package]] -name = "aws-sig-auth" -version = "0.55.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b94acb10af0c879ecd5c7bdf51cda6679a0a4f4643ce630905a77673bfa3c61" -dependencies = [ - "aws-credential-types", - "aws-sigv4", - "aws-smithy-eventstream", - "aws-smithy-http", - "aws-types", - "http", "tracing", ] [[package]] name = "aws-sigv4" -version = "0.55.3" +version = "0.56.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d2ce6f507be68e968a33485ced670111d1cbad161ddbbab1e313c03d37d8f4c" +checksum = "2cb40a93429794065f41f0581734fc56a345f6a38d8e2e3c25c7448d930cd132" dependencies = [ "aws-smithy-eventstream", "aws-smithy-http", @@ -389,9 +379,9 @@ dependencies = [ [[package]] name = "aws-smithy-async" -version = "0.55.3" +version = "0.56.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13bda3996044c202d75b91afeb11a9afae9db9a721c6a7a427410018e286b880" +checksum = "6ee6d17d487c8b579423067718b3580c0908d0f01d7461813f94ec4323bad623" dependencies = [ "futures-util", "pin-project-lite", @@ -401,9 +391,9 @@ dependencies = [ [[package]] name = "aws-smithy-checksums" -version = "0.55.3" +version = "0.56.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07ed8b96d95402f3f6b8b57eb4e0e45ee365f78b1a924faf20ff6e97abf1eae6" +checksum = "0d1849fd5916904513fb0862543b36f8faab43c07984dbc476132b7da1aed056" dependencies = [ "aws-smithy-http", "aws-smithy-types", @@ -422,23 +412,23 @@ dependencies = [ [[package]] name = "aws-smithy-client" -version = "0.55.3" +version = "0.56.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0a86aa6e21e86c4252ad6a0e3e74da9617295d8d6e374d552be7d3059c41cedd" +checksum = "bdbe0a3ad15283cc5f863a68cb6adc8e256e7c109c43c01bdd09be407219a1e9" dependencies = [ "aws-smithy-async", "aws-smithy-http", "aws-smithy-http-tower", "aws-smithy-types", "bytes", - "fastrand", + "fastrand 2.0.0", "http", "http-body", "hyper", - "hyper-rustls 0.23.2", + "hyper-rustls", "lazy_static", "pin-project-lite", - "rustls 0.20.8", + "rustls", "tokio", "tower", "tracing", @@ -446,9 +436,9 @@ dependencies = [ [[package]] name = "aws-smithy-eventstream" -version = "0.55.3" +version = "0.56.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "460c8da5110835e3d9a717c61f5556b20d03c32a1dec57f8fc559b360f733bb8" +checksum = "a56afef1aa766f512b4970b4c3150b9bf2df8035939723830df4b30267e2d7cb" dependencies = [ "aws-smithy-types", "bytes", @@ -457,9 +447,9 @@ dependencies = [ [[package]] name = "aws-smithy-http" -version = "0.55.3" +version = "0.56.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b3b693869133551f135e1f2c77cb0b8277d9e3e17feaf2213f735857c4f0d28" +checksum = "34dc313472d727f5ef44fdda93e668ebfe17380c99dee512c403e3ca51863bb9" dependencies = [ "aws-smithy-eventstream", "aws-smithy-types", @@ -480,9 +470,9 @@ dependencies = [ [[package]] name = "aws-smithy-http-tower" -version = "0.55.3" +version = "0.56.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ae4f6c5798a247fac98a867698197d9ac22643596dc3777f0c76b91917616b9" +checksum = "1dd50fca5a4ea4ec3771689ee93bf06b32de02a80af01ed93a8f8a4ed90e8483" dependencies = [ "aws-smithy-http", "aws-smithy-types", @@ -496,50 +486,88 @@ dependencies = [ [[package]] name = "aws-smithy-json" -version = "0.55.3" +version = "0.56.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23f9f42fbfa96d095194a632fbac19f60077748eba536eb0b9fecc28659807f8" +checksum = "3591dd7c2fe01ab8025e4847a0a0f6d0c2b2269714688ffb856f9cf6c6d465cf" dependencies = [ "aws-smithy-types", ] [[package]] name = "aws-smithy-query" -version = "0.55.3" +version = "0.56.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "98819eb0b04020a1c791903533b638534ae6c12e2aceda3e6e6fba015608d51d" +checksum = "dbabb1145e65dd57ae72d91a2619d3f5fba40b68a5f40ba009c30571dfd60aff" dependencies = [ "aws-smithy-types", "urlencoding", ] [[package]] -name = "aws-smithy-types" -version = "0.55.3" +name = "aws-smithy-runtime" +version = "0.56.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "16a3d0bf4f324f4ef9793b86a1701d9700fbcdbd12a846da45eed104c634c6e8" +checksum = "3687fb838d4ad1c883b62eb59115bc9fb02c4f308aac49a7df89627067f6eb0d" +dependencies = [ + "aws-smithy-async", + "aws-smithy-client", + "aws-smithy-http", + "aws-smithy-runtime-api", + "aws-smithy-types", + "bytes", + "fastrand 2.0.0", + "http", + "http-body", + "once_cell", + "pin-project-lite", + "pin-utils", + "tokio", + "tracing", +] + +[[package]] +name = "aws-smithy-runtime-api" +version = "0.56.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5cfbf1e5c2108b41f5ca607cde40dd5109fecc448f5d30c8e614b61f36dce704" +dependencies = [ + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-types", + "bytes", + "http", + "tokio", + "tracing", +] + +[[package]] +name = "aws-smithy-types" +version = "0.56.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eed0a94eefd845a2a78677f1b72f02fa75802d38f7f59be675add140279aa8bf" dependencies = [ "base64-simd", "itoa", "num-integer", "ryu", + "serde", "time", ] [[package]] name = "aws-smithy-xml" -version = "0.55.3" +version = "0.56.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1b9d12875731bd07e767be7baad95700c3137b56730ec9ddeedb52a5e5ca63b" +checksum = "c88052c812f696143ad7ba729c63535209ff0e0f49e31a6d2b1205208ea6ea79" dependencies = [ "xmlparser", ] [[package]] name = "aws-types" -version = "0.55.3" +version = "0.56.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6dd209616cc8d7bfb82f87811a5c655dc97537f592689b18743bddf5dc5c4829" +checksum = "6bceb8cf724ad057ad7f327d0d256d7147b3eac777b39849a26189e003dc9782" dependencies = [ "aws-credential-types", "aws-smithy-async", @@ -1402,6 +1430,12 @@ dependencies = [ "instant", ] +[[package]] +name = "fastrand" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6999dc1837253364c2ebb0704ba97994bd874e8f195d665c50b7548f6ea92764" + [[package]] name = "filetime" version = "0.2.21" @@ -1837,21 +1871,6 @@ dependencies = [ "want", ] -[[package]] -name = "hyper-rustls" -version = "0.23.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1788965e61b367cd03a62950836d5cd41560c3577d90e40e0819373194d1661c" -dependencies = [ - "http", - "hyper", - "log", - "rustls 0.20.8", - "rustls-native-certs", - "tokio", - "tokio-rustls 0.23.4", -] - [[package]] name = "hyper-rustls" version = "0.24.0" @@ -1860,9 +1879,11 @@ checksum = "0646026eb1b3eea4cd9ba47912ea5ce9cc07713d105b1a14698f4e6433d348b7" dependencies = [ "http", "hyper", - "rustls 0.21.1", + "log", + "rustls", + "rustls-native-certs", "tokio", - "tokio-rustls 0.24.0", + "tokio-rustls", ] [[package]] @@ -2054,7 +2075,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6971da4d9c3aa03c3d8f3ff0f4155b534aad021292003895a469716b2a230378" dependencies = [ "base64 0.21.1", - "pem", + "pem 1.1.1", "ring", "serde", "serde_json", @@ -2780,6 +2801,16 @@ dependencies = [ "base64 0.13.1", ] +[[package]] +name = "pem" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b13fe415cdf3c8e44518e18a7c95a13431d9bdf6d15367d82b23c377fdd441a" +dependencies = [ + "base64 0.21.1", + "serde", +] + [[package]] name = "percent-encoding" version = "2.2.0" @@ -2942,14 +2973,14 @@ dependencies = [ "futures", "once_cell", "pq_proto", - "rustls 0.20.8", + "rustls", "rustls-pemfile", "serde", "thiserror", "tokio", "tokio-postgres", "tokio-postgres-rustls", - "tokio-rustls 0.23.4", + "tokio-rustls", "tracing", "workspace_hack", ] @@ -3174,7 +3205,7 @@ dependencies = [ "reqwest-tracing", "routerify", "rstest", - "rustls 0.20.8", + "rustls", "rustls-pemfile", "scopeguard", "serde", @@ -3187,7 +3218,7 @@ dependencies = [ "tokio", "tokio-postgres", "tokio-postgres-rustls", - "tokio-rustls 0.23.4", + "tokio-rustls", "tokio-util", "tracing", "tracing-opentelemetry", @@ -3196,7 +3227,7 @@ dependencies = [ "url", "utils", "uuid", - "webpki-roots 0.23.0", + "webpki-roots 0.25.2", "workspace_hack", "x509-parser", ] @@ -3264,11 +3295,11 @@ dependencies = [ [[package]] name = "rcgen" -version = "0.10.0" +version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ffbe84efe2f38dea12e9bfc1f65377fdf03e53a18cb3b995faedf7934c7e785b" +checksum = "4954fbc00dcd4d8282c987710e50ba513d351400dbdd00e803a05172a90d8976" dependencies = [ - "pem", + "pem 2.0.1", "ring", "time", "yasna", @@ -3324,6 +3355,12 @@ version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "436b050e76ed2903236f032a59761c1eb99e1b0aead2c257922771dab1fc8c78" +[[package]] +name = "relative-path" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c707298afce11da2efef2f600116fa93ffa7a032b5d7b628aa17711ec81383ca" + [[package]] name = "remote_storage" version = "0.1.0" @@ -3354,9 +3391,9 @@ dependencies = [ [[package]] name = "reqwest" -version = "0.11.18" +version = "0.11.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cde824a14b7c14f85caff81225f411faacc04a2013f41670f41443742b1c1c55" +checksum = "20b9b67e2ca7dd9e9f9285b759de30ff538aab981abaaf7bc9bd90b84a0126c3" dependencies = [ "base64 0.21.1", "bytes", @@ -3367,7 +3404,7 @@ dependencies = [ "http", "http-body", "hyper", - "hyper-rustls 0.24.0", + "hyper-rustls", "ipnet", "js-sys", "log", @@ -3376,19 +3413,19 @@ dependencies = [ "once_cell", "percent-encoding", "pin-project-lite", - "rustls 0.21.1", + "rustls", "rustls-pemfile", "serde", "serde_json", "serde_urlencoded", "tokio", - "tokio-rustls 0.24.0", + "tokio-rustls", "tower-service", "url", "wasm-bindgen", "wasm-bindgen-futures", "web-sys", - "webpki-roots 0.22.6", + "webpki-roots 0.25.2", "winreg", ] @@ -3498,9 +3535,9 @@ dependencies = [ [[package]] name = "rstest" -version = "0.17.0" +version = "0.18.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de1bb486a691878cd320c2f0d319ba91eeaa2e894066d8b5f8f117c000e9d962" +checksum = "97eeab2f3c0a199bc4be135c36c924b6590b88c377d416494288c14f2db30199" dependencies = [ "futures", "futures-timer", @@ -3510,15 +3547,18 @@ dependencies = [ [[package]] name = "rstest_macros" -version = "0.17.0" +version = "0.18.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "290ca1a1c8ca7edb7c3283bd44dc35dd54fdec6253a3912e201ba1072018fca8" +checksum = "d428f8247852f894ee1be110b375111b586d4fa431f6c46e64ba5a0dcccbe605" dependencies = [ "cfg-if", + "glob", "proc-macro2", "quote", + "regex", + "relative-path", "rustc_version", - "syn 1.0.109", + "syn 2.0.28", "unicode-ident", ] @@ -3582,25 +3622,13 @@ dependencies = [ [[package]] name = "rustls" -version = "0.20.8" +version = "0.21.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fff78fc74d175294f4e83b28343315ffcfb114b156f0185e9741cb5570f50e2f" +checksum = "1d1feddffcfcc0b33f5c6ce9a29e341e4cd59c3f78e7ee45f4a40c038b1d6cbb" dependencies = [ "log", "ring", - "sct", - "webpki", -] - -[[package]] -name = "rustls" -version = "0.21.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c911ba11bc8433e811ce56fde130ccf32f5127cab0e0194e9c68c5a5b671791e" -dependencies = [ - "log", - "ring", - "rustls-webpki", + "rustls-webpki 0.101.4", "sct", ] @@ -3635,6 +3663,16 @@ dependencies = [ "untrusted", ] +[[package]] +name = "rustls-webpki" +version = "0.101.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d93931baf2d282fff8d3a532bbfd7653f734643161b87e3e01e59a04439bf0d" +dependencies = [ + "ring", + "untrusted", +] + [[package]] name = "rustversion" version = "1.0.12" @@ -3772,27 +3810,28 @@ checksum = "bebd363326d05ec3e2f532ab7660680f3b02130d780c299bca73469d521bc0ed" [[package]] name = "sentry" -version = "0.30.0" +version = "0.31.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5ce6d3512e2617c209ec1e86b0ca2fea06454cd34653c91092bf0f3ec41f8e3" +checksum = "2e95efd0cefa32028cdb9766c96de71d96671072f9fb494dc9fb84c0ef93e52b" dependencies = [ "httpdate", "reqwest", - "rustls 0.20.8", + "rustls", "sentry-backtrace", "sentry-contexts", "sentry-core", "sentry-panic", + "sentry-tracing", "tokio", "ureq", - "webpki-roots 0.22.6", + "webpki-roots 0.25.2", ] [[package]] name = "sentry-backtrace" -version = "0.30.0" +version = "0.31.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e7fe408d4d1f8de188a9309916e02e129cbe51ca19e55badea5a64899399b1a" +checksum = "6ac2bac6f310c4c4c4bb094d1541d32ae497f8c5c23405e85492cefdfe0971a9" dependencies = [ "backtrace", "once_cell", @@ -3802,9 +3841,9 @@ dependencies = [ [[package]] name = "sentry-contexts" -version = "0.30.0" +version = "0.31.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5695096a059a89973ec541062d331ff4c9aeef9c2951416c894f0fff76340e7d" +checksum = "6c3e17295cecdbacf66c5bd38d6e1147e09e1e9d824d2d5341f76638eda02a3a" dependencies = [ "hostname", "libc", @@ -3816,9 +3855,9 @@ dependencies = [ [[package]] name = "sentry-core" -version = "0.30.0" +version = "0.31.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b22828bfd118a7b660cf7a155002a494755c0424cebb7061e4743ecde9c7dbc" +checksum = "8339474f587f36cb110fa1ed1b64229eea6d47b0b886375579297b7e47aeb055" dependencies = [ "once_cell", "rand", @@ -3829,19 +3868,31 @@ dependencies = [ [[package]] name = "sentry-panic" -version = "0.30.0" +version = "0.31.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f4ced2a7a8c14899d58eec402d946f69d5ed26a3fc363a7e8b1e5cb88473a01" +checksum = "875b69f506da75bd664029eafb05f8934297d2990192896d17325f066bd665b7" dependencies = [ "sentry-backtrace", "sentry-core", ] [[package]] -name = "sentry-types" -version = "0.30.0" +name = "sentry-tracing" +version = "0.31.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "360ee3270f7a4a1eee6c667f7d38360b995431598a73b740dfe420da548d9cc9" +checksum = "89feead9bdd116f8035e89567651340fc382db29240b6c55ef412078b08d1aa3" +dependencies = [ + "sentry-backtrace", + "sentry-core", + "tracing-core", + "tracing-subscriber", +] + +[[package]] +name = "sentry-types" +version = "0.31.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "99dc599bd6646884fc403d593cdcb9816dd67c50cff3271c01ff123617908dcd" dependencies = [ "debugid", "getrandom", @@ -4248,7 +4299,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b9fbec84f381d5795b08656e4912bec604d162bff9291d6189a78f4c8ab87998" dependencies = [ "cfg-if", - "fastrand", + "fastrand 1.9.0", "redox_syscall 0.3.5", "rustix 0.37.19", "windows-sys 0.45.0", @@ -4378,16 +4429,16 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tls-listener" -version = "0.6.0" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97abcaa5d5850d3b469898d1e0939b57c3afb4475122e792cdd1c82b07f5de06" +checksum = "81294c017957a1a69794f506723519255879e15a870507faf45dfed288b763dd" dependencies = [ "futures-util", "hyper", "pin-project-lite", "thiserror", "tokio", - "tokio-rustls 0.23.4", + "tokio-rustls", ] [[package]] @@ -4464,27 +4515,16 @@ dependencies = [ [[package]] name = "tokio-postgres-rustls" -version = "0.9.0" +version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "606f2b73660439474394432239c82249c0d45eb5f23d91f401be1e33590444a7" +checksum = "dd5831152cb0d3f79ef5523b357319ba154795d64c7078b2daa95a803b54057f" dependencies = [ "futures", "ring", - "rustls 0.20.8", + "rustls", "tokio", "tokio-postgres", - "tokio-rustls 0.23.4", -] - -[[package]] -name = "tokio-rustls" -version = "0.23.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c43ee83903113e03984cb9e5cebe6c04a5116269e900e3ddba8f068a62adda59" -dependencies = [ - "rustls 0.20.8", - "tokio", - "webpki", + "tokio-rustls", ] [[package]] @@ -4493,7 +4533,7 @@ version = "0.24.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e0d409377ff5b1e3ca6437aa86c1eb7d40c134bfec254e44c830defa92669db5" dependencies = [ - "rustls 0.21.1", + "rustls", "tokio", ] @@ -4651,7 +4691,7 @@ dependencies = [ "rustls-native-certs", "rustls-pemfile", "tokio", - "tokio-rustls 0.24.0", + "tokio-rustls", "tokio-stream", "tower", "tower-layer", @@ -4949,17 +4989,17 @@ checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" [[package]] name = "ureq" -version = "2.6.2" +version = "2.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "338b31dd1314f68f3aabf3ed57ab922df95ffcd902476ca7ba3c4ce7b908c46d" +checksum = "0b11c96ac7ee530603dcdf68ed1557050f374ce55a5a07193ebf8cbc9f8927e9" dependencies = [ - "base64 0.13.1", + "base64 0.21.1", "log", "once_cell", - "rustls 0.20.8", + "rustls", + "rustls-webpki 0.100.2", "url", - "webpki", - "webpki-roots 0.22.6", + "webpki-roots 0.23.1", ] [[package]] @@ -5230,32 +5270,19 @@ dependencies = [ ] [[package]] -name = "webpki" -version = "0.22.0" +name = "webpki-roots" +version = "0.23.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f095d78192e208183081cc07bc5515ef55216397af48b873e5edcd72637fa1bd" +checksum = "b03058f88386e5ff5310d9111d53f48b17d732b401aeb83a8d5190f2ac459338" dependencies = [ - "ring", - "untrusted", + "rustls-webpki 0.100.2", ] [[package]] name = "webpki-roots" -version = "0.22.6" +version = "0.25.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6c71e40d7d2c34a5106301fb632274ca37242cd0c9d3e64dbece371a40a2d87" -dependencies = [ - "webpki", -] - -[[package]] -name = "webpki-roots" -version = "0.23.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa54963694b65584e170cf5dc46aeb4dcaa5584e652ff5f3952e56d66aff0125" -dependencies = [ - "rustls-webpki", -] +checksum = "14247bb57be4f377dfb94c72830b8ce8fc6beac03cf4bf7b9732eadd414123fc" [[package]] name = "which" @@ -5466,11 +5493,12 @@ dependencies = [ [[package]] name = "winreg" -version = "0.10.1" +version = "0.50.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "80d0f4e272c85def139476380b12f9ac60926689dd2e01d4923222f40580869d" +checksum = "524e57b2c537c0f9b1e69f1965311ec12182b4122e45035b1508cd24d2adadb1" dependencies = [ - "winapi", + "cfg-if", + "windows-sys 0.48.0", ] [[package]] @@ -5479,6 +5507,7 @@ version = "0.1.0" dependencies = [ "anyhow", "axum", + "base64 0.21.1", "bytes", "cc", "chrono", @@ -5509,7 +5538,7 @@ dependencies = [ "regex-syntax 0.7.2", "reqwest", "ring", - "rustls 0.20.8", + "rustls", "scopeguard", "serde", "serde_json", @@ -5518,7 +5547,7 @@ dependencies = [ "syn 1.0.109", "syn 2.0.28", "tokio", - "tokio-rustls 0.23.4", + "tokio-rustls", "tokio-util", "toml_datetime", "toml_edit", diff --git a/Cargo.toml b/Cargo.toml index f38e67f487..d545be266f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -37,11 +37,11 @@ async-compression = { version = "0.4.0", features = ["tokio", "gzip"] } flate2 = "1.0.26" async-stream = "0.3" async-trait = "0.1" -aws-config = { version = "0.55", default-features = false, features=["rustls"] } -aws-sdk-s3 = "0.27" -aws-smithy-http = "0.55" -aws-credential-types = "0.55" -aws-types = "0.55" +aws-config = { version = "0.56", default-features = false, features=["rustls"] } +aws-sdk-s3 = "0.29" +aws-smithy-http = "0.56" +aws-credential-types = "0.56" +aws-types = "0.56" axum = { version = "0.6.20", features = ["ws"] } base64 = "0.13.0" bincode = "1.3" @@ -105,12 +105,12 @@ reqwest-middleware = "0.2.0" reqwest-retry = "0.2.2" routerify = "3" rpds = "0.13" -rustls = "0.20" +rustls = "0.21" rustls-pemfile = "1" rustls-split = "0.3" scopeguard = "1.1" sysinfo = "0.29.2" -sentry = { version = "0.30", default-features = false, features = ["backtrace", "contexts", "panic", "rustls", "reqwest" ] } +sentry = { version = "0.31", default-features = false, features = ["backtrace", "contexts", "panic", "rustls", "reqwest" ] } serde = { version = "1.0", features = ["derive"] } serde_json = "1" serde_with = "2.0" @@ -125,11 +125,11 @@ sync_wrapper = "0.1.2" tar = "0.4" test-context = "0.1" thiserror = "1.0" -tls-listener = { version = "0.6", features = ["rustls", "hyper-h1"] } +tls-listener = { version = "0.7", features = ["rustls", "hyper-h1"] } tokio = { version = "1.17", features = ["macros"] } tokio-io-timeout = "1.2.0" -tokio-postgres-rustls = "0.9.0" -tokio-rustls = "0.23" +tokio-postgres-rustls = "0.10.0" +tokio-rustls = "0.24" tokio-stream = "0.1" tokio-tar = "0.3" tokio-util = { version = "0.7", features = ["io"] } @@ -143,7 +143,7 @@ tracing-subscriber = { version = "0.3", default_features = false, features = ["s url = "2.2" uuid = { version = "1.2", features = ["v4", "serde"] } walkdir = "2.3.2" -webpki-roots = "0.23" +webpki-roots = "0.25" x509-parser = "0.15" ## TODO replace this with tracing @@ -182,8 +182,8 @@ workspace_hack = { version = "0.1", path = "./workspace_hack/" } ## Build dependencies criterion = "0.5.1" -rcgen = "0.10" -rstest = "0.17" +rcgen = "0.11" +rstest = "0.18" tempfile = "3.4" tonic-build = "0.9" diff --git a/proxy/src/http/websocket.rs b/proxy/src/http/websocket.rs index c85450a074..72ae3dc26f 100644 --- a/proxy/src/http/websocket.rs +++ b/proxy/src/http/websocket.rs @@ -304,7 +304,7 @@ pub async fn task_main( let make_svc = hyper::service::make_service_fn(|stream: &tokio_rustls::server::TlsStream| { - let sni_name = stream.get_ref().1.sni_hostname().map(|s| s.to_string()); + let sni_name = stream.get_ref().1.server_name().map(|s| s.to_string()); let conn_pool = conn_pool.clone(); async move { diff --git a/proxy/src/stream.rs b/proxy/src/stream.rs index 7cb292ed58..6210601a80 100644 --- a/proxy/src/stream.rs +++ b/proxy/src/stream.rs @@ -141,7 +141,7 @@ impl Stream { pub fn sni_hostname(&self) -> Option<&str> { match self { Stream::Raw { .. } => None, - Stream::Tls { tls } => tls.get_ref().1.sni_hostname(), + Stream::Tls { tls } => tls.get_ref().1.server_name(), } } } diff --git a/workspace_hack/Cargo.toml b/workspace_hack/Cargo.toml index 831cc6f6b1..4ec4b01f66 100644 --- a/workspace_hack/Cargo.toml +++ b/workspace_hack/Cargo.toml @@ -15,6 +15,7 @@ publish = false [dependencies] anyhow = { version = "1", features = ["backtrace"] } axum = { version = "0.6", features = ["ws"] } +base64 = { version = "0.21", features = ["alloc"] } bytes = { version = "1", features = ["serde"] } chrono = { version = "0.4", default-features = false, features = ["clock", "serde"] } clap = { version = "4", features = ["derive", "string"] } @@ -44,14 +45,14 @@ regex = { version = "1" } regex-syntax = { version = "0.7" } reqwest = { version = "0.11", default-features = false, features = ["blocking", "json", "multipart", "rustls-tls"] } ring = { version = "0.16", features = ["std"] } -rustls = { version = "0.20", features = ["dangerous_configuration"] } +rustls = { version = "0.21", features = ["dangerous_configuration"] } scopeguard = { version = "1" } serde = { version = "1", features = ["alloc", "derive"] } serde_json = { version = "1", features = ["raw_value"] } smallvec = { version = "1", default-features = false, features = ["write"] } socket2 = { version = "0.4", default-features = false, features = ["all"] } tokio = { version = "1", features = ["fs", "io-std", "io-util", "macros", "net", "process", "rt-multi-thread", "signal", "test-util"] } -tokio-rustls = { version = "0.23" } +tokio-rustls = { version = "0.24" } tokio-util = { version = "0.7", features = ["codec", "io"] } toml_datetime = { version = "0.6", default-features = false, features = ["serde"] } toml_edit = { version = "0.19", features = ["serde"] } @@ -74,7 +75,7 @@ prost = { version = "0.11" } regex = { version = "1" } regex-syntax = { version = "0.7" } serde = { version = "1", features = ["alloc", "derive"] } -syn-dff4ba8e3ae991db = { package = "syn", version = "1", features = ["extra-traits", "full", "visit", "visit-mut"] } -syn-f595c2ba2a3f28df = { package = "syn", version = "2", features = ["extra-traits", "full", "visit-mut"] } +syn-dff4ba8e3ae991db = { package = "syn", version = "1", features = ["extra-traits", "full", "visit"] } +syn-f595c2ba2a3f28df = { package = "syn", version = "2", features = ["extra-traits", "full", "visit", "visit-mut"] } ### END HAKARI SECTION From a7c0e4dcd0ae6f61a84d2a36e9e40a3113cb54f0 Mon Sep 17 00:00:00 2001 From: Anastasia Lubennikova Date: Wed, 30 Aug 2023 16:04:48 +0300 Subject: [PATCH 03/10] Check if custiom extension is enabled. This check was lost in the latest refactoring. If extension is not present in 'public_extensions' or 'custom_extensions' don't download it --- compute_tools/src/extension_server.rs | 14 +++++++++++++- libs/compute_api/src/spec.rs | 12 ++++++++++++ 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/compute_tools/src/extension_server.rs b/compute_tools/src/extension_server.rs index cb54a603e0..54c22026e7 100644 --- a/compute_tools/src/extension_server.rs +++ b/compute_tools/src/extension_server.rs @@ -180,7 +180,19 @@ pub async fn download_extension( // Create extension control files from spec pub fn create_control_files(remote_extensions: &RemoteExtSpec, pgbin: &str) { let local_sharedir = Path::new(&get_pg_config("--sharedir", pgbin)).join("extension"); - for ext_data in remote_extensions.extension_data.values() { + for (ext_name, ext_data) in remote_extensions.extension_data.iter() { + // Check if extension is present in public or custom. + // If not, then it is not allowed to be used by this compute. + if let Some(public_extensions) = &remote_extensions.public_extensions { + if !public_extensions.contains(ext_name) { + if let Some(custom_extensions) = &remote_extensions.custom_extensions { + if !custom_extensions.contains(ext_name) { + continue; // skip this extension, it is not allowed + } + } + } + } + for (control_name, control_content) in &ext_data.control_data { let control_path = local_sharedir.join(control_name); if !control_path.exists() { diff --git a/libs/compute_api/src/spec.rs b/libs/compute_api/src/spec.rs index 78361ce07e..b41ca8c9cf 100644 --- a/libs/compute_api/src/spec.rs +++ b/libs/compute_api/src/spec.rs @@ -106,6 +106,18 @@ impl RemoteExtSpec { .ok_or(anyhow::anyhow!("library {} is not found", lib_raw_name))?; } + // Check if extension is present in public or custom. + // If not, then it is not allowed to be used by this compute. + if let Some(public_extensions) = &self.public_extensions { + if !public_extensions.contains(&real_ext_name.to_string()) { + if let Some(custom_extensions) = &self.custom_extensions { + if !custom_extensions.contains(&real_ext_name.to_string()) { + return Err(anyhow::anyhow!("extension {} is not found", real_ext_name)); + } + } + } + } + match self.extension_data.get(real_ext_name) { Some(_ext_data) => { // Construct the path to the extension archive From a93274b389a8445bbf3df11abdcc8cc0b17210e4 Mon Sep 17 00:00:00 2001 From: John Spray Date: Wed, 30 Aug 2023 16:14:04 +0100 Subject: [PATCH 04/10] pageserver: remove vestigial `timeline_layers` attribute (#5153) ## Problem `timeline_layers` was write-only since b95addddd54dc1b25850b0784206941ebaea6af4 We deployed the version that no longer requires it for deserializing, so now we can stop including it when serializing. ## Summary of changes Fully remove `timeline_layers`. --- .../tenant/remote_timeline_client/index.rs | 66 +++++++++++++------ 1 file changed, 45 insertions(+), 21 deletions(-) diff --git a/pageserver/src/tenant/remote_timeline_client/index.rs b/pageserver/src/tenant/remote_timeline_client/index.rs index 28177b097f..bcde6589c5 100644 --- a/pageserver/src/tenant/remote_timeline_client/index.rs +++ b/pageserver/src/tenant/remote_timeline_client/index.rs @@ -2,7 +2,7 @@ //! Able to restore itself from the storage index parts, that are located in every timeline's remote directory and contain all data about //! remote timeline layers and its metadata. -use std::collections::{HashMap, HashSet}; +use std::collections::HashMap; use chrono::NaiveDateTime; use serde::{Deserialize, Serialize}; @@ -62,10 +62,6 @@ pub struct IndexPart { #[serde(skip_serializing_if = "Option::is_none")] pub deleted_at: Option, - /// Legacy field: equal to the keys of `layer_metadata`, only written out for forward compat - #[serde(default, skip_deserializing)] - timeline_layers: HashSet, - /// Per layer file name metadata, which can be present for a present or missing layer file. /// /// Older versions of `IndexPart` will not have this property or have only a part of metadata @@ -91,7 +87,8 @@ impl IndexPart { /// - 2: added `deleted_at` /// - 3: no longer deserialize `timeline_layers` (serialized format is the same, but timeline_layers /// is always generated from the keys of `layer_metadata`) - const LATEST_VERSION: usize = 3; + /// - 4: timeline_layers is fully removed. + const LATEST_VERSION: usize = 4; pub const FILE_NAME: &'static str = "index_part.json"; pub fn new( @@ -99,18 +96,14 @@ impl IndexPart { disk_consistent_lsn: Lsn, metadata: TimelineMetadata, ) -> Self { - let mut timeline_layers = HashSet::with_capacity(layers_and_metadata.len()); - let mut layer_metadata = HashMap::with_capacity(layers_and_metadata.len()); - - for (remote_name, metadata) in &layers_and_metadata { - timeline_layers.insert(remote_name.to_owned()); - let metadata = IndexLayerMetadata::from(metadata); - layer_metadata.insert(remote_name.to_owned(), metadata); - } + // Transform LayerFileMetadata into IndexLayerMetadata + let layer_metadata = layers_and_metadata + .into_iter() + .map(|(k, v)| (k, IndexLayerMetadata::from(v))) + .collect(); Self { version: Self::LATEST_VERSION, - timeline_layers, layer_metadata, disk_consistent_lsn, metadata, @@ -140,8 +133,8 @@ pub struct IndexLayerMetadata { pub(super) file_size: u64, } -impl From<&'_ LayerFileMetadata> for IndexLayerMetadata { - fn from(other: &'_ LayerFileMetadata) -> Self { +impl From for IndexLayerMetadata { + fn from(other: LayerFileMetadata) -> Self { IndexLayerMetadata { file_size: other.file_size, } @@ -168,7 +161,6 @@ mod tests { let expected = IndexPart { // note this is not verified, could be anything, but exists for humans debugging.. could be the git version instead? version: 1, - timeline_layers: HashSet::new(), layer_metadata: HashMap::from([ ("000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9".parse().unwrap(), IndexLayerMetadata { file_size: 25600000, @@ -205,7 +197,6 @@ mod tests { let expected = IndexPart { // note this is not verified, could be anything, but exists for humans debugging.. could be the git version instead? version: 1, - timeline_layers: HashSet::new(), layer_metadata: HashMap::from([ ("000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9".parse().unwrap(), IndexLayerMetadata { file_size: 25600000, @@ -243,7 +234,6 @@ mod tests { let expected = IndexPart { // note this is not verified, could be anything, but exists for humans debugging.. could be the git version instead? version: 2, - timeline_layers: HashSet::new(), layer_metadata: HashMap::from([ ("000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9".parse().unwrap(), IndexLayerMetadata { file_size: 25600000, @@ -276,7 +266,6 @@ mod tests { let expected = IndexPart { version: 1, - timeline_layers: HashSet::new(), layer_metadata: HashMap::new(), disk_consistent_lsn: "0/2532648".parse::().unwrap(), metadata: TimelineMetadata::from_bytes(&[ @@ -309,4 +298,39 @@ mod tests { assert_eq!(empty_layers_parsed, expected); } + + #[test] + fn v4_indexpart_is_parsed() { + let example = r#"{ + "version":4, + "layer_metadata":{ + "000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9": { "file_size": 25600000 }, + "000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__00000000016B59D8-00000000016B5A51": { "file_size": 9007199254741001 } + }, + "disk_consistent_lsn":"0/16960E8", + "metadata_bytes":[113,11,159,210,0,54,0,4,0,0,0,0,1,105,96,232,1,0,0,0,0,1,105,96,112,0,0,0,0,0,0,0,0,0,0,0,0,0,1,105,96,112,0,0,0,0,1,105,96,112,0,0,0,14,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0], + "deleted_at": "2023-07-31T09:00:00.123" + }"#; + + let expected = IndexPart { + version: 4, + layer_metadata: HashMap::from([ + ("000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9".parse().unwrap(), IndexLayerMetadata { + file_size: 25600000, + }), + ("000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__00000000016B59D8-00000000016B5A51".parse().unwrap(), IndexLayerMetadata { + // serde_json should always parse this but this might be a double with jq for + // example. + file_size: 9007199254741001, + }) + ]), + disk_consistent_lsn: "0/16960E8".parse::().unwrap(), + metadata: TimelineMetadata::from_bytes(&[113,11,159,210,0,54,0,4,0,0,0,0,1,105,96,232,1,0,0,0,0,1,105,96,112,0,0,0,0,0,0,0,0,0,0,0,0,0,1,105,96,112,0,0,0,0,1,105,96,112,0,0,0,14,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]).unwrap(), + deleted_at: Some(chrono::NaiveDateTime::parse_from_str( + "2023-07-31T09:00:00.123000000", "%Y-%m-%dT%H:%M:%S.%f").unwrap()) + }; + + let part = serde_json::from_str::(example).unwrap(); + assert_eq!(part, expected); + } } From 93dcdb293afa1bf3709e49138b5018f3f013b0d0 Mon Sep 17 00:00:00 2001 From: Conrad Ludgate Date: Wed, 30 Aug 2023 16:20:27 +0100 Subject: [PATCH 05/10] proxy: password hack hack (#5126) ## Problem fixes #4881 ## Summary of changes --- proxy/src/auth/password_hack.rs | 37 +++++++++++++++++++++++++++------ 1 file changed, 31 insertions(+), 6 deletions(-) diff --git a/proxy/src/auth/password_hack.rs b/proxy/src/auth/password_hack.rs index 33441e8c88..d1da208fef 100644 --- a/proxy/src/auth/password_hack.rs +++ b/proxy/src/auth/password_hack.rs @@ -12,13 +12,19 @@ pub struct PasswordHackPayload { impl PasswordHackPayload { pub fn parse(bytes: &[u8]) -> Option { - // The format is `project=;`. - let mut iter = bytes.splitn_str(2, ";"); - let endpoint = iter.next()?.to_str().ok()?; - let endpoint = parse_endpoint_param(endpoint)?.to_owned(); - let password = iter.next()?.to_owned(); + // The format is `project=;` or `project=$`. + let separators = [";", "$"]; + for sep in separators { + if let Some((endpoint, password)) = bytes.split_once_str(sep) { + let endpoint = endpoint.to_str().ok()?; + return Some(Self { + endpoint: parse_endpoint_param(endpoint)?.to_owned(), + password: password.to_owned(), + }); + } + } - Some(Self { endpoint, password }) + None } } @@ -91,4 +97,23 @@ mod tests { assert_eq!(payload.endpoint, "foobar"); assert_eq!(payload.password, b"pass;word"); } + + #[test] + fn parse_password_hack_payload_dollar() { + let bytes = b""; + assert!(PasswordHackPayload::parse(bytes).is_none()); + + let bytes = b"endpoint="; + assert!(PasswordHackPayload::parse(bytes).is_none()); + + let bytes = b"endpoint=$"; + let payload = PasswordHackPayload::parse(bytes).expect("parsing failed"); + assert_eq!(payload.endpoint, ""); + assert_eq!(payload.password, b""); + + let bytes = b"endpoint=foobar$pass$word"; + let payload = PasswordHackPayload::parse(bytes).expect("parsing failed"); + assert_eq!(payload.endpoint, "foobar"); + assert_eq!(payload.password, b"pass$word"); + } } From f2c21447cec25280af8701b2ada50bcb04896336 Mon Sep 17 00:00:00 2001 From: Alexey Kondratov Date: Wed, 30 Aug 2023 17:44:28 +0200 Subject: [PATCH 06/10] [compute_ctl] Create check availability data during full configuration (#5084) I've moved it to the API handler in the 589cf1ed2 to do not delay compute start. Yet, we now skip full configuration and catalog updates in the most hot path -- waking up suspended compute, and only do it at: - first start - start with applying new configuration - start for availability check so it doesn't really matter anymore. The problem with creating the table and test record in the API handler is that someone can fill up timeline till the logical limit. Then it's suspended and availability check is scheduled, so it fails. If table + test row are created at the very beginning, we reserve a 8 KB page for future checks, which theoretically will last almost forever. For example, my ~1y old branch still has 8 KB sized test table: ```sql cloud_admin@postgres=# select pg_relation_size('health_check'); pg_relation_size ------------------ 8192 (1 row) ``` --------- Co-authored-by: Anastasia Lubennikova --- compute_tools/src/checker.rs | 39 ++++++++++++++++++++++++++++------- compute_tools/src/compute.rs | 2 ++ control_plane/src/endpoint.rs | 10 +++++++-- 3 files changed, 41 insertions(+), 10 deletions(-) diff --git a/compute_tools/src/checker.rs b/compute_tools/src/checker.rs index b6a287bdeb..6f52004fa8 100644 --- a/compute_tools/src/checker.rs +++ b/compute_tools/src/checker.rs @@ -1,12 +1,39 @@ -use anyhow::{anyhow, Result}; +use anyhow::{anyhow, Ok, Result}; +use postgres::Client; use tokio_postgres::NoTls; use tracing::{error, instrument}; use crate::compute::ComputeNode; +/// Create a special service table for availability checks +/// only if it does not exist already. +pub fn create_availability_check_data(client: &mut Client) -> Result<()> { + let query = " + DO $$ + BEGIN + IF NOT EXISTS( + SELECT 1 + FROM pg_catalog.pg_tables + WHERE tablename = 'health_check' + ) + THEN + CREATE TABLE health_check ( + id serial primary key, + updated_at timestamptz default now() + ); + INSERT INTO health_check VALUES (1, now()) + ON CONFLICT (id) DO UPDATE + SET updated_at = now(); + END IF; + END + $$;"; + client.execute(query, &[])?; + + Ok(()) +} + /// Update timestamp in a row in a special service table to check /// that we can actually write some data in this particular timeline. -/// Create table if it's missing. #[instrument(skip_all)] pub async fn check_writability(compute: &ComputeNode) -> Result<()> { // Connect to the database. @@ -24,19 +51,15 @@ pub async fn check_writability(compute: &ComputeNode) -> Result<()> { }); let query = " - CREATE TABLE IF NOT EXISTS health_check ( - id serial primary key, - updated_at timestamptz default now() - ); INSERT INTO health_check VALUES (1, now()) ON CONFLICT (id) DO UPDATE SET updated_at = now();"; let result = client.simple_query(query).await?; - if result.len() != 2 { + if result.len() != 1 { return Err(anyhow::format_err!( - "expected 2 query results, but got {}", + "expected 1 query result, but got {}", result.len() )); } diff --git a/compute_tools/src/compute.rs b/compute_tools/src/compute.rs index 03ae39f79f..5c08ebe06a 100644 --- a/compute_tools/src/compute.rs +++ b/compute_tools/src/compute.rs @@ -27,6 +27,7 @@ use utils::measured_stream::MeasuredReader; use remote_storage::{DownloadError, GenericRemoteStorage, RemotePath}; +use crate::checker::create_availability_check_data; use crate::pg_helpers::*; use crate::spec::*; use crate::sync_sk::{check_if_synced, ping_safekeeper}; @@ -696,6 +697,7 @@ impl ComputeNode { handle_role_deletions(spec, self.connstr.as_str(), &mut client)?; handle_grants(spec, self.connstr.as_str())?; handle_extensions(spec, &mut client)?; + create_availability_check_data(&mut client)?; // 'Close' connection drop(client); diff --git a/control_plane/src/endpoint.rs b/control_plane/src/endpoint.rs index 7ebcf98ab0..4ed03c8771 100644 --- a/control_plane/src/endpoint.rs +++ b/control_plane/src/endpoint.rs @@ -138,7 +138,13 @@ impl ComputeControlPlane { mode, tenant_id, pg_version, - skip_pg_catalog_updates: false, + // We don't setup roles and databases in the spec locally, so we don't need to + // do catalog updates. Catalog updates also include check availability + // data creation. Yet, we have tests that check that size and db dump + // before and after start are the same. So, skip catalog updates, + // with this we basically test a case of waking up an idle compute, where + // we also skip catalog updates in the cloud. + skip_pg_catalog_updates: true, }); ep.create_endpoint_dir()?; @@ -152,7 +158,7 @@ impl ComputeControlPlane { http_port, pg_port, pg_version, - skip_pg_catalog_updates: false, + skip_pg_catalog_updates: true, })?, )?; std::fs::write( From 83ae2bd82cd6ed7ff5c44acf96f2dcc1e2e480b4 Mon Sep 17 00:00:00 2001 From: John Spray Date: Thu, 31 Aug 2023 09:19:34 +0100 Subject: [PATCH 07/10] pageserver: generation number support in keys and indices (#5140) ## Problem To implement split brain protection, we need tenants and timelines to be aware of their current generation, and use it when composing S3 keys. ## Summary of changes - A `Generation` type is introduced in the `utils` crate -- it is in this broadly-visible location because it will later be used from `control_plane/` as well as `pageserver/`. Generations can be a number, None, or Broken, to support legacy content (None), and Tenants in the broken state (Broken). - Tenant, Timeline, and RemoteTimelineClient all get a generation attribute - IndexPart's IndexLayerMetadata has a new `generation` attribute. Legacy layers' metadata will deserialize to Generation::none(). - Remote paths are composed with a trailing generation suffix. If a generation is equal to Generation::none() (as it currently always is), then this suffix is an empty string. - Functions for composing remote storage paths added in remote_timeline_client: these avoid the way that we currently always compose a local path and then strip the prefix, and avoid requiring a PageserverConf reference on functions that want to create remote paths (the conf is only needed for local paths). These are less DRY than the old functions, but remote storage paths are a very rarely changing thing, so it's better to write out our paths clearly in the functions than to compose timeline paths from tenant paths, etc. - Code paths that construct a Tenant take a `generation` argument in anticipation that we will soon load generations on startup before constructing Tenant. Until the whole feature is done, we don't want any generation-ful keys though: so initially we will carry this everywhere with the special Generation::none() value. Closes: https://github.com/neondatabase/neon/issues/5135 Co-authored-by: Christian Schwarz --- libs/utils/src/generation.rs | 113 +++++++++++++ libs/utils/src/lib.rs | 3 + pageserver/src/config.rs | 17 -- pageserver/src/tenant.rs | 27 +++- pageserver/src/tenant/mgr.rs | 12 +- .../src/tenant/remote_timeline_client.rs | 150 +++++++++++++++--- .../tenant/remote_timeline_client/delete.rs | 15 +- .../tenant/remote_timeline_client/download.rs | 117 +++++++------- .../tenant/remote_timeline_client/index.rs | 26 ++- .../tenant/remote_timeline_client/upload.rs | 46 +++--- pageserver/src/tenant/timeline.rs | 34 +++- pageserver/src/tenant/timeline/init.rs | 18 ++- pageserver/src/tenant/upload_queue.rs | 16 +- 13 files changed, 434 insertions(+), 160 deletions(-) create mode 100644 libs/utils/src/generation.rs diff --git a/libs/utils/src/generation.rs b/libs/utils/src/generation.rs new file mode 100644 index 0000000000..87c6361255 --- /dev/null +++ b/libs/utils/src/generation.rs @@ -0,0 +1,113 @@ +use std::fmt::Debug; + +use serde::{Deserialize, Serialize}; + +/// Tenant generations are used to provide split-brain safety and allow +/// multiple pageservers to attach the same tenant concurrently. +/// +/// See docs/rfcs/025-generation-numbers.md for detail on how generation +/// numbers are used. +#[derive(Copy, Clone, Eq, PartialEq, PartialOrd, Ord)] +pub enum Generation { + // Generations with this magic value will not add a suffix to S3 keys, and will not + // be included in persisted index_part.json. This value is only to be used + // during migration from pre-generation metadata to generation-aware metadata, + // and should eventually go away. + // + // A special Generation is used rather than always wrapping Generation in an Option, + // so that code handling generations doesn't have to be aware of the legacy + // case everywhere it touches a generation. + None, + // Generations with this magic value may never be used to construct S3 keys: + // we will panic if someone tries to. This is for Tenants in the "Broken" state, + // so that we can satisfy their constructor with a Generation without risking + // a code bug using it in an S3 write (broken tenants should never write) + Broken, + Valid(u32), +} + +/// The Generation type represents a number associated with a Tenant, which +/// increments every time the tenant is attached to a new pageserver, or +/// an attached pageserver restarts. +/// +/// It is included as a suffix in S3 keys, as a protection against split-brain +/// scenarios where pageservers might otherwise issue conflicting writes to +/// remote storage +impl Generation { + /// Create a new Generation that represents a legacy key format with + /// no generation suffix + pub fn none() -> Self { + Self::None + } + + // Create a new generation that will panic if you try to use get_suffix + pub fn broken() -> Self { + Self::Broken + } + + pub fn new(v: u32) -> Self { + Self::Valid(v) + } + + pub fn is_none(&self) -> bool { + matches!(self, Self::None) + } + + pub fn get_suffix(&self) -> String { + match self { + Self::Valid(v) => { + format!("-{:08x}", v) + } + Self::None => "".into(), + Self::Broken => { + panic!("Tried to use a broken generation"); + } + } + } +} + +impl Serialize for Generation { + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + if let Self::Valid(v) = self { + v.serialize(serializer) + } else { + // We should never be asked to serialize a None or Broken. Structures + // that include an optional generation should convert None to an + // Option::None + Err(serde::ser::Error::custom( + "Tried to serialize invalid generation ({self})", + )) + } + } +} + +impl<'de> Deserialize<'de> for Generation { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + Ok(Self::Valid(u32::deserialize(deserializer)?)) + } +} + +// We intentionally do not implement Display for Generation, to reduce the +// risk of a bug where the generation is used in a format!() string directly +// instead of using get_suffix(). +impl Debug for Generation { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Valid(v) => { + write!(f, "{:08x}", v) + } + Self::None => { + write!(f, "") + } + Self::Broken => { + write!(f, "") + } + } + } +} diff --git a/libs/utils/src/lib.rs b/libs/utils/src/lib.rs index 6cf829a67c..160e082833 100644 --- a/libs/utils/src/lib.rs +++ b/libs/utils/src/lib.rs @@ -27,6 +27,9 @@ pub mod id; // http endpoint utils pub mod http; +// definition of the Generation type for pageserver attachment APIs +pub mod generation; + // common log initialisation routine pub mod logging; diff --git a/pageserver/src/config.rs b/pageserver/src/config.rs index f2aa2f365e..5394f17398 100644 --- a/pageserver/src/config.rs +++ b/pageserver/src/config.rs @@ -643,23 +643,6 @@ impl PageServerConf { .join(METADATA_FILE_NAME) } - /// Files on the remote storage are stored with paths, relative to the workdir. - /// That path includes in itself both tenant and timeline ids, allowing to have a unique remote storage path. - /// - /// Errors if the path provided does not start from pageserver's workdir. - pub fn remote_path(&self, local_path: &Path) -> anyhow::Result { - local_path - .strip_prefix(&self.workdir) - .context("Failed to strip workdir prefix") - .and_then(RemotePath::new) - .with_context(|| { - format!( - "Failed to resolve remote part of path {:?} for base {:?}", - local_path, self.workdir - ) - }) - } - /// Turns storage remote path of a file into its local path. pub fn local_path(&self, remote_path: &RemotePath) -> PathBuf { remote_path.with_base(&self.workdir) diff --git a/pageserver/src/tenant.rs b/pageserver/src/tenant.rs index a204f8a22b..2168db57de 100644 --- a/pageserver/src/tenant.rs +++ b/pageserver/src/tenant.rs @@ -85,6 +85,7 @@ pub use pageserver_api::models::TenantState; use toml_edit; use utils::{ crashsafe, + generation::Generation, id::{TenantId, TimelineId}, lsn::{Lsn, RecordLsn}, }; @@ -178,6 +179,11 @@ pub struct Tenant { tenant_conf: Arc>, tenant_id: TenantId, + + /// The remote storage generation, used to protect S3 objects from split-brain. + /// Does not change over the lifetime of the [`Tenant`] object. + generation: Generation, + timelines: Mutex>>, // This mutex prevents creation of new timelines during GC. // Adding yet another mutex (in addition to `timelines`) is needed because holding @@ -522,6 +528,7 @@ impl Tenant { pub(crate) fn spawn_attach( conf: &'static PageServerConf, tenant_id: TenantId, + generation: Generation, broker_client: storage_broker::BrokerClientChannel, tenants: &'static tokio::sync::RwLock, remote_storage: GenericRemoteStorage, @@ -538,6 +545,7 @@ impl Tenant { tenant_conf, wal_redo_manager, tenant_id, + generation, Some(remote_storage.clone()), )); @@ -648,12 +656,8 @@ impl Tenant { .as_ref() .ok_or_else(|| anyhow::anyhow!("cannot attach without remote storage"))?; - let remote_timeline_ids = remote_timeline_client::list_remote_timelines( - remote_storage, - self.conf, - self.tenant_id, - ) - .await?; + let remote_timeline_ids = + remote_timeline_client::list_remote_timelines(remote_storage, self.tenant_id).await?; info!("found {} timelines", remote_timeline_ids.len()); @@ -665,6 +669,7 @@ impl Tenant { self.conf, self.tenant_id, timeline_id, + self.generation, ); part_downloads.spawn( async move { @@ -851,6 +856,7 @@ impl Tenant { TenantConfOpt::default(), wal_redo_manager, tenant_id, + Generation::broken(), None, )) } @@ -868,6 +874,7 @@ impl Tenant { pub(crate) fn spawn_load( conf: &'static PageServerConf, tenant_id: TenantId, + generation: Generation, resources: TenantSharedResources, init_order: Option, tenants: &'static tokio::sync::RwLock, @@ -893,6 +900,7 @@ impl Tenant { tenant_conf, wal_redo_manager, tenant_id, + generation, remote_storage.clone(), ); let tenant = Arc::new(tenant); @@ -2274,6 +2282,7 @@ impl Tenant { ancestor, new_timeline_id, self.tenant_id, + self.generation, Arc::clone(&self.walredo_mgr), resources, pg_version, @@ -2291,6 +2300,7 @@ impl Tenant { tenant_conf: TenantConfOpt, walredo_mgr: Arc, tenant_id: TenantId, + generation: Generation, remote_storage: Option, ) -> Tenant { let (state, mut rx) = watch::channel(state); @@ -2349,6 +2359,7 @@ impl Tenant { Tenant { tenant_id, + generation, conf, // using now here is good enough approximation to catch tenants with really long // activation times. @@ -2931,6 +2942,7 @@ impl Tenant { self.conf, self.tenant_id, timeline_id, + self.generation, ); Some(remote_client) } else { @@ -3454,6 +3466,7 @@ pub mod harness { pub conf: &'static PageServerConf, pub tenant_conf: TenantConf, pub tenant_id: TenantId, + pub generation: Generation, } static LOG_HANDLE: OnceCell<()> = OnceCell::new(); @@ -3495,6 +3508,7 @@ pub mod harness { conf, tenant_conf, tenant_id, + generation: Generation::new(0xdeadbeef), }) } @@ -3521,6 +3535,7 @@ pub mod harness { TenantConfOpt::from(self.tenant_conf), walredo_mgr, self.tenant_id, + self.generation, remote_storage, )); tenant diff --git a/pageserver/src/tenant/mgr.rs b/pageserver/src/tenant/mgr.rs index a558c7d0ba..87617b544c 100644 --- a/pageserver/src/tenant/mgr.rs +++ b/pageserver/src/tenant/mgr.rs @@ -25,6 +25,7 @@ use crate::tenant::{create_tenant_files, CreateTenantFilesMode, Tenant, TenantSt use crate::{InitializationOrder, IGNORED_TENANT_FILE_NAME}; use utils::fs_ext::PathExt; +use utils::generation::Generation; use utils::id::{TenantId, TimelineId}; use super::delete::DeleteTenantError; @@ -202,6 +203,7 @@ pub(crate) fn schedule_local_tenant_processing( match Tenant::spawn_attach( conf, tenant_id, + Generation::none(), resources.broker_client, tenants, remote_storage, @@ -224,7 +226,15 @@ pub(crate) fn schedule_local_tenant_processing( } else { info!("tenant {tenant_id} is assumed to be loadable, starting load operation"); // Start loading the tenant into memory. It will initially be in Loading state. - Tenant::spawn_load(conf, tenant_id, resources, init_order, tenants, ctx) + Tenant::spawn_load( + conf, + tenant_id, + Generation::none(), + resources, + init_order, + tenants, + ctx, + ) }; Ok(tenant) } diff --git a/pageserver/src/tenant/remote_timeline_client.rs b/pageserver/src/tenant/remote_timeline_client.rs index e46205810a..50bb8b43de 100644 --- a/pageserver/src/tenant/remote_timeline_client.rs +++ b/pageserver/src/tenant/remote_timeline_client.rs @@ -216,7 +216,7 @@ use utils::backoff::{ }; use std::collections::{HashMap, VecDeque}; -use std::path::Path; +use std::path::{Path, PathBuf}; use std::sync::atomic::{AtomicU32, Ordering}; use std::sync::{Arc, Mutex}; @@ -235,6 +235,7 @@ use crate::task_mgr::shutdown_token; use crate::tenant::debug_assert_current_span_has_tenant_and_timeline_id; use crate::tenant::remote_timeline_client::index::LayerFileMetadata; use crate::tenant::upload_queue::Delete; +use crate::tenant::TIMELINES_SEGMENT_NAME; use crate::{ config::PageServerConf, task_mgr, @@ -252,6 +253,7 @@ use self::index::IndexPart; use super::storage_layer::LayerFileName; use super::upload_queue::SetDeletedFlagProgress; +use super::Generation; // Occasional network issues and such can cause remote operations to fail, and // that's expected. If a download fails, we log it at info-level, and retry. @@ -315,6 +317,7 @@ pub struct RemoteTimelineClient { tenant_id: TenantId, timeline_id: TimelineId, + generation: Generation, upload_queue: Mutex, @@ -335,12 +338,14 @@ impl RemoteTimelineClient { conf: &'static PageServerConf, tenant_id: TenantId, timeline_id: TimelineId, + generation: Generation, ) -> RemoteTimelineClient { RemoteTimelineClient { conf, runtime: BACKGROUND_RUNTIME.handle().to_owned(), tenant_id, timeline_id, + generation, storage_impl: remote_storage, upload_queue: Mutex::new(UploadQueue::Uninitialized), metrics: Arc::new(RemoteTimelineClientMetrics::new(&tenant_id, &timeline_id)), @@ -449,10 +454,10 @@ impl RemoteTimelineClient { ); let index_part = download::download_index_part( - self.conf, &self.storage_impl, &self.tenant_id, &self.timeline_id, + self.generation, ) .measure_remote_op( self.tenant_id, @@ -650,22 +655,41 @@ impl RemoteTimelineClient { // from latest_files, but not yet scheduled for deletion. Use a closure // to syntactically forbid ? or bail! calls here. let no_bail_here = || { - for name in names { - if upload_queue.latest_files.remove(name).is_some() { - upload_queue.latest_files_changes_since_metadata_upload_scheduled += 1; - } - } + // Decorate our list of names with each name's generation, dropping + // makes that are unexpectedly missing from our metadata. + let with_generations: Vec<_> = names + .iter() + .filter_map(|name| { + // Remove from latest_files, learning the file's remote generation in the process + let meta = upload_queue.latest_files.remove(name); + + if let Some(meta) = meta { + upload_queue.latest_files_changes_since_metadata_upload_scheduled += 1; + Some((name, meta.generation)) + } else { + // This can only happen if we forgot to to schedule the file upload + // before scheduling the delete. Log it because it is a rare/strange + // situation, and in case something is misbehaving, we'd like to know which + // layers experienced this. + info!( + "Deleting layer {name} not found in latest_files list, never uploaded?" + ); + None + } + }) + .collect(); if upload_queue.latest_files_changes_since_metadata_upload_scheduled > 0 { self.schedule_index_upload(upload_queue, metadata); } // schedule the actual deletions - for name in names { + for (name, generation) in with_generations { let op = UploadOp::Delete(Delete { file_kind: RemoteOpFileKind::Layer, layer_file_name: name.clone(), scheduled_from_timeline_delete: false, + generation, }); self.calls_unfinished_metric_begin(&op); upload_queue.queued_operations.push_back(op); @@ -761,10 +785,10 @@ impl RemoteTimelineClient { backoff::retry( || { upload::upload_index_part( - self.conf, &self.storage_impl, &self.tenant_id, &self.timeline_id, + self.generation, &index_part_with_deleted_at, ) }, @@ -822,12 +846,14 @@ impl RemoteTimelineClient { .reserve(stopped.upload_queue_for_deletion.latest_files.len()); // schedule the actual deletions - for name in stopped.upload_queue_for_deletion.latest_files.keys() { + for (name, meta) in &stopped.upload_queue_for_deletion.latest_files { let op = UploadOp::Delete(Delete { file_kind: RemoteOpFileKind::Layer, layer_file_name: name.clone(), scheduled_from_timeline_delete: true, + generation: meta.generation, }); + self.calls_unfinished_metric_begin(&op); stopped .upload_queue_for_deletion @@ -850,8 +876,7 @@ impl RemoteTimelineClient { // Do not delete index part yet, it is needed for possible retry. If we remove it first // and retry will arrive to different pageserver there wont be any traces of it on remote storage - let timeline_path = self.conf.timeline_path(&self.tenant_id, &self.timeline_id); - let timeline_storage_path = self.conf.remote_path(&timeline_path)?; + let timeline_storage_path = remote_timeline_path(&self.tenant_id, &self.timeline_id); let remaining = backoff::retry( || async { @@ -1055,15 +1080,17 @@ impl RemoteTimelineClient { let upload_result: anyhow::Result<()> = match &task.op { UploadOp::UploadLayer(ref layer_file_name, ref layer_metadata) => { - let path = &self + let path = self .conf .timeline_path(&self.tenant_id, &self.timeline_id) .join(layer_file_name.file_name()); + upload::upload_timeline_layer( self.conf, &self.storage_impl, - path, + &path, layer_metadata, + self.generation, ) .measure_remote_op( self.tenant_id, @@ -1085,10 +1112,10 @@ impl RemoteTimelineClient { }; let res = upload::upload_index_part( - self.conf, &self.storage_impl, &self.tenant_id, &self.timeline_id, + self.generation, index_part, ) .measure_remote_op( @@ -1113,7 +1140,7 @@ impl RemoteTimelineClient { .conf .timeline_path(&self.tenant_id, &self.timeline_id) .join(delete.layer_file_name.file_name()); - delete::delete_layer(self.conf, &self.storage_impl, path) + delete::delete_layer(self.conf, &self.storage_impl, path, delete.generation) .measure_remote_op( self.tenant_id, self.timeline_id, @@ -1360,6 +1387,71 @@ impl RemoteTimelineClient { } } +pub fn remote_timelines_path(tenant_id: &TenantId) -> RemotePath { + let path = format!("tenants/{tenant_id}/{TIMELINES_SEGMENT_NAME}"); + RemotePath::from_string(&path).expect("Failed to construct path") +} + +pub fn remote_timeline_path(tenant_id: &TenantId, timeline_id: &TimelineId) -> RemotePath { + remote_timelines_path(tenant_id).join(&PathBuf::from(timeline_id.to_string())) +} + +pub fn remote_layer_path( + tenant_id: &TenantId, + timeline_id: &TimelineId, + layer_file_name: &LayerFileName, + layer_meta: &LayerFileMetadata, +) -> RemotePath { + // Generation-aware key format + let path = format!( + "tenants/{tenant_id}/{TIMELINES_SEGMENT_NAME}/{timeline_id}/{0}{1}", + layer_file_name.file_name(), + layer_meta.generation.get_suffix() + ); + + RemotePath::from_string(&path).expect("Failed to construct path") +} + +pub fn remote_index_path( + tenant_id: &TenantId, + timeline_id: &TimelineId, + generation: Generation, +) -> RemotePath { + RemotePath::from_string(&format!( + "tenants/{tenant_id}/{TIMELINES_SEGMENT_NAME}/{timeline_id}/{0}{1}", + IndexPart::FILE_NAME, + generation.get_suffix() + )) + .expect("Failed to construct path") +} + +/// Files on the remote storage are stored with paths, relative to the workdir. +/// That path includes in itself both tenant and timeline ids, allowing to have a unique remote storage path. +/// +/// Errors if the path provided does not start from pageserver's workdir. +pub fn remote_path( + conf: &PageServerConf, + local_path: &Path, + generation: Generation, +) -> anyhow::Result { + let stripped = local_path + .strip_prefix(&conf.workdir) + .context("Failed to strip workdir prefix")?; + + let suffixed = format!( + "{0}{1}", + stripped.to_string_lossy(), + generation.get_suffix() + ); + + RemotePath::new(&PathBuf::from(suffixed)).with_context(|| { + format!( + "to resolve remote part of path {:?} for base {:?}", + local_path, conf.workdir + ) + }) +} + #[cfg(test)] mod tests { use super::*; @@ -1367,7 +1459,7 @@ mod tests { context::RequestContext, tenant::{ harness::{TenantHarness, TIMELINE_ID}, - Tenant, Timeline, + Generation, Tenant, Timeline, }, DEFAULT_PG_VERSION, }; @@ -1409,8 +1501,11 @@ mod tests { assert_eq!(avec, bvec); } - fn assert_remote_files(expected: &[&str], remote_path: &Path) { - let mut expected: Vec = expected.iter().map(|x| String::from(*x)).collect(); + fn assert_remote_files(expected: &[&str], remote_path: &Path, generation: Generation) { + let mut expected: Vec = expected + .iter() + .map(|x| format!("{}{}", x, generation.get_suffix())) + .collect(); expected.sort(); let mut found: Vec = Vec::new(); @@ -1461,6 +1556,8 @@ mod tests { storage: RemoteStorageKind::LocalFs(remote_fs_dir.clone()), }; + let generation = Generation::new(0xdeadbeef); + let storage = GenericRemoteStorage::from_config(&storage_config).unwrap(); let client = Arc::new(RemoteTimelineClient { @@ -1468,6 +1565,7 @@ mod tests { runtime: tokio::runtime::Handle::current(), tenant_id: harness.tenant_id, timeline_id: TIMELINE_ID, + generation, storage_impl: storage, upload_queue: Mutex::new(UploadQueue::Uninitialized), metrics: Arc::new(RemoteTimelineClientMetrics::new( @@ -1526,6 +1624,8 @@ mod tests { .init_upload_queue_for_empty_remote(&metadata) .unwrap(); + let generation = Generation::new(0xdeadbeef); + // Create a couple of dummy files, schedule upload for them let layer_file_name_1: LayerFileName = "000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__00000000016B59D8-00000000016B5A51".parse().unwrap(); let layer_file_name_2: LayerFileName = "000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__00000000016B59D9-00000000016B5A52".parse().unwrap(); @@ -1545,13 +1645,13 @@ mod tests { client .schedule_layer_file_upload( &layer_file_name_1, - &LayerFileMetadata::new(content_1.len() as u64), + &LayerFileMetadata::new(content_1.len() as u64, generation), ) .unwrap(); client .schedule_layer_file_upload( &layer_file_name_2, - &LayerFileMetadata::new(content_2.len() as u64), + &LayerFileMetadata::new(content_2.len() as u64, generation), ) .unwrap(); @@ -1615,7 +1715,7 @@ mod tests { client .schedule_layer_file_upload( &layer_file_name_3, - &LayerFileMetadata::new(content_3.len() as u64), + &LayerFileMetadata::new(content_3.len() as u64, generation), ) .unwrap(); client @@ -1639,6 +1739,7 @@ mod tests { "index_part.json", ], &remote_timeline_dir, + generation, ); // Finish them @@ -1651,6 +1752,7 @@ mod tests { "index_part.json", ], &remote_timeline_dir, + generation, ); } @@ -1703,12 +1805,14 @@ mod tests { // Test + let generation = Generation::new(0xdeadbeef); + let init = get_bytes_started_stopped(); client .schedule_layer_file_upload( &layer_file_name_1, - &LayerFileMetadata::new(content_1.len() as u64), + &LayerFileMetadata::new(content_1.len() as u64, generation), ) .unwrap(); diff --git a/pageserver/src/tenant/remote_timeline_client/delete.rs b/pageserver/src/tenant/remote_timeline_client/delete.rs index 3f505d45ab..7324559223 100644 --- a/pageserver/src/tenant/remote_timeline_client/delete.rs +++ b/pageserver/src/tenant/remote_timeline_client/delete.rs @@ -5,25 +5,30 @@ use tracing::debug; use remote_storage::GenericRemoteStorage; -use crate::config::PageServerConf; +use crate::{ + config::PageServerConf, + tenant::{remote_timeline_client::remote_path, Generation}, +}; pub(super) async fn delete_layer<'a>( conf: &'static PageServerConf, storage: &'a GenericRemoteStorage, local_layer_path: &'a Path, + generation: Generation, ) -> anyhow::Result<()> { fail::fail_point!("before-delete-layer", |_| { anyhow::bail!("failpoint before-delete-layer") }); debug!("Deleting layer from remote storage: {local_layer_path:?}",); - let path_to_delete = conf.remote_path(local_layer_path)?; + let path_to_delete = remote_path(conf, local_layer_path, generation)?; // We don't want to print an error if the delete failed if the file has // already been deleted. Thankfully, in this situation S3 already // does not yield an error. While OS-provided local file system APIs do yield // errors, we avoid them in the `LocalFs` wrapper. - storage.delete(&path_to_delete).await.with_context(|| { - format!("Failed to delete remote layer from storage at {path_to_delete:?}") - }) + storage + .delete(&path_to_delete) + .await + .with_context(|| format!("delete remote layer from storage at {path_to_delete:?}")) } diff --git a/pageserver/src/tenant/remote_timeline_client/download.rs b/pageserver/src/tenant/remote_timeline_client/download.rs index 2cb33f07c9..dc8d87b9e1 100644 --- a/pageserver/src/tenant/remote_timeline_client/download.rs +++ b/pageserver/src/tenant/remote_timeline_client/download.rs @@ -15,14 +15,16 @@ use tokio_util::sync::CancellationToken; use utils::{backoff, crashsafe}; use crate::config::PageServerConf; +use crate::tenant::remote_timeline_client::{remote_layer_path, remote_timelines_path}; use crate::tenant::storage_layer::LayerFileName; use crate::tenant::timeline::span::debug_assert_current_span_has_tenant_and_timeline_id; +use crate::tenant::Generation; use remote_storage::{DownloadError, GenericRemoteStorage}; use utils::crashsafe::path_with_suffix_extension; use utils::id::{TenantId, TimelineId}; use super::index::{IndexPart, LayerFileMetadata}; -use super::{FAILED_DOWNLOAD_WARN_THRESHOLD, FAILED_REMOTE_OP_RETRIES}; +use super::{remote_index_path, FAILED_DOWNLOAD_WARN_THRESHOLD, FAILED_REMOTE_OP_RETRIES}; static MAX_DOWNLOAD_DURATION: Duration = Duration::from_secs(120); @@ -41,13 +43,11 @@ pub async fn download_layer_file<'a>( ) -> Result { debug_assert_current_span_has_tenant_and_timeline_id(); - let timeline_path = conf.timeline_path(&tenant_id, &timeline_id); + let local_path = conf + .timeline_path(&tenant_id, &timeline_id) + .join(layer_file_name.file_name()); - let local_path = timeline_path.join(layer_file_name.file_name()); - - let remote_path = conf - .remote_path(&local_path) - .map_err(DownloadError::Other)?; + let remote_path = remote_layer_path(&tenant_id, &timeline_id, layer_file_name, layer_metadata); // Perform a rename inspired by durable_rename from file_utils.c. // The sequence: @@ -64,33 +64,43 @@ pub async fn download_layer_file<'a>( let (mut destination_file, bytes_amount) = download_retry( || async { // TODO: this doesn't use the cached fd for some reason? - let mut destination_file = fs::File::create(&temp_file_path).await.with_context(|| { - format!( - "create a destination file for layer '{}'", - temp_file_path.display() - ) - }) - .map_err(DownloadError::Other)?; - let mut download = storage.download(&remote_path).await.with_context(|| { - format!( + let mut destination_file = fs::File::create(&temp_file_path) + .await + .with_context(|| { + format!( + "create a destination file for layer '{}'", + temp_file_path.display() + ) + }) + .map_err(DownloadError::Other)?; + let mut download = storage + .download(&remote_path) + .await + .with_context(|| { + format!( "open a download stream for layer with remote storage path '{remote_path:?}'" ) - }) - .map_err(DownloadError::Other)?; - - let bytes_amount = tokio::time::timeout(MAX_DOWNLOAD_DURATION, tokio::io::copy(&mut download.download_stream, &mut destination_file)) - .await - .map_err(|e| DownloadError::Other(anyhow::anyhow!("Timed out {:?}", e)))? - .with_context(|| { - format!("Failed to download layer with remote storage path '{remote_path:?}' into file {temp_file_path:?}") }) .map_err(DownloadError::Other)?; - Ok((destination_file, bytes_amount)) + let bytes_amount = tokio::time::timeout( + MAX_DOWNLOAD_DURATION, + tokio::io::copy(&mut download.download_stream, &mut destination_file), + ) + .await + .map_err(|e| DownloadError::Other(anyhow::anyhow!("Timed out {:?}", e)))? + .with_context(|| { + format!( + "download layer at remote path '{remote_path:?}' into file {temp_file_path:?}" + ) + }) + .map_err(DownloadError::Other)?; + Ok((destination_file, bytes_amount)) }, &format!("download {remote_path:?}"), - ).await?; + ) + .await?; // Tokio doc here: https://docs.rs/tokio/1.17.0/tokio/fs/struct.File.html states that: // A file will not be closed immediately when it goes out of scope if there are any IO operations @@ -103,12 +113,7 @@ pub async fn download_layer_file<'a>( destination_file .flush() .await - .with_context(|| { - format!( - "failed to flush source file at {}", - temp_file_path.display() - ) - }) + .with_context(|| format!("flush source file at {}", temp_file_path.display())) .map_err(DownloadError::Other)?; let expected = layer_metadata.file_size(); @@ -139,17 +144,12 @@ pub async fn download_layer_file<'a>( fs::rename(&temp_file_path, &local_path) .await - .with_context(|| { - format!( - "Could not rename download layer file to {}", - local_path.display(), - ) - }) + .with_context(|| format!("rename download layer file to {}", local_path.display(),)) .map_err(DownloadError::Other)?; crashsafe::fsync_async(&local_path) .await - .with_context(|| format!("Could not fsync layer file {}", local_path.display(),)) + .with_context(|| format!("fsync layer file {}", local_path.display(),)) .map_err(DownloadError::Other)?; tracing::debug!("download complete: {}", local_path.display()); @@ -173,21 +173,19 @@ pub fn is_temp_download_file(path: &Path) -> bool { } /// List timelines of given tenant in remote storage -pub async fn list_remote_timelines<'a>( - storage: &'a GenericRemoteStorage, - conf: &'static PageServerConf, +pub async fn list_remote_timelines( + storage: &GenericRemoteStorage, tenant_id: TenantId, ) -> anyhow::Result> { - let tenant_path = conf.timelines_path(&tenant_id); - let tenant_storage_path = conf.remote_path(&tenant_path)?; + let remote_path = remote_timelines_path(&tenant_id); fail::fail_point!("storage-sync-list-remote-timelines", |_| { anyhow::bail!("storage-sync-list-remote-timelines"); }); let timelines = download_retry( - || storage.list_prefixes(Some(&tenant_storage_path)), - &format!("list prefixes for {tenant_path:?}"), + || storage.list_prefixes(Some(&remote_path)), + &format!("list prefixes for {tenant_id}"), ) .await?; @@ -202,9 +200,9 @@ pub async fn list_remote_timelines<'a>( anyhow::anyhow!("failed to get timeline id for remote tenant {tenant_id}") })?; - let timeline_id: TimelineId = object_name.parse().with_context(|| { - format!("failed to parse object name into timeline id '{object_name}'") - })?; + let timeline_id: TimelineId = object_name + .parse() + .with_context(|| format!("parse object name into timeline id '{object_name}'"))?; // list_prefixes is assumed to return unique names. Ensure this here. // NB: it's safer to bail out than warn-log this because the pageserver @@ -222,21 +220,16 @@ pub async fn list_remote_timelines<'a>( } pub(super) async fn download_index_part( - conf: &'static PageServerConf, storage: &GenericRemoteStorage, tenant_id: &TenantId, timeline_id: &TimelineId, + generation: Generation, ) -> Result { - let index_part_path = conf - .metadata_path(tenant_id, timeline_id) - .with_file_name(IndexPart::FILE_NAME); - let part_storage_path = conf - .remote_path(&index_part_path) - .map_err(DownloadError::BadInput)?; + let remote_path = remote_index_path(tenant_id, timeline_id, generation); let index_part_bytes = download_retry( || async { - let mut index_part_download = storage.download(&part_storage_path).await?; + let mut index_part_download = storage.download(&remote_path).await?; let mut index_part_bytes = Vec::new(); tokio::io::copy( @@ -244,20 +237,16 @@ pub(super) async fn download_index_part( &mut index_part_bytes, ) .await - .with_context(|| { - format!("Failed to download an index part into file {index_part_path:?}") - }) + .with_context(|| format!("download index part at {remote_path:?}")) .map_err(DownloadError::Other)?; Ok(index_part_bytes) }, - &format!("download {part_storage_path:?}"), + &format!("download {remote_path:?}"), ) .await?; let index_part: IndexPart = serde_json::from_slice(&index_part_bytes) - .with_context(|| { - format!("Failed to deserialize index part file into file {index_part_path:?}") - }) + .with_context(|| format!("download index part file at {remote_path:?}")) .map_err(DownloadError::Other)?; Ok(index_part) diff --git a/pageserver/src/tenant/remote_timeline_client/index.rs b/pageserver/src/tenant/remote_timeline_client/index.rs index bcde6589c5..37ed0e2c3f 100644 --- a/pageserver/src/tenant/remote_timeline_client/index.rs +++ b/pageserver/src/tenant/remote_timeline_client/index.rs @@ -12,6 +12,7 @@ use utils::bin_ser::SerializeError; use crate::tenant::metadata::TimelineMetadata; use crate::tenant::storage_layer::LayerFileName; use crate::tenant::upload_queue::UploadQueueInitialized; +use crate::tenant::Generation; use utils::lsn::Lsn; @@ -20,22 +21,28 @@ use utils::lsn::Lsn; /// Fields have to be `Option`s because remote [`IndexPart`]'s can be from different version, which /// might have less or more metadata depending if upgrading or rolling back an upgrade. #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)] -#[cfg_attr(test, derive(Default))] +//#[cfg_attr(test, derive(Default))] pub struct LayerFileMetadata { file_size: u64, + + pub(crate) generation: Generation, } impl From<&'_ IndexLayerMetadata> for LayerFileMetadata { fn from(other: &IndexLayerMetadata) -> Self { LayerFileMetadata { file_size: other.file_size, + generation: other.generation, } } } impl LayerFileMetadata { - pub fn new(file_size: u64) -> Self { - LayerFileMetadata { file_size } + pub fn new(file_size: u64, generation: Generation) -> Self { + LayerFileMetadata { + file_size, + generation, + } } pub fn file_size(&self) -> u64 { @@ -128,15 +135,20 @@ impl TryFrom<&UploadQueueInitialized> for IndexPart { } /// Serialized form of [`LayerFileMetadata`]. -#[derive(Debug, PartialEq, Eq, Clone, Serialize, Deserialize, Default)] +#[derive(Debug, PartialEq, Eq, Clone, Serialize, Deserialize)] pub struct IndexLayerMetadata { pub(super) file_size: u64, + + #[serde(default = "Generation::none")] + #[serde(skip_serializing_if = "Generation::is_none")] + pub(super) generation: Generation, } impl From for IndexLayerMetadata { fn from(other: LayerFileMetadata) -> Self { IndexLayerMetadata { file_size: other.file_size, + generation: other.generation, } } } @@ -164,11 +176,13 @@ mod tests { layer_metadata: HashMap::from([ ("000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9".parse().unwrap(), IndexLayerMetadata { file_size: 25600000, + generation: Generation::none() }), ("000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__00000000016B59D8-00000000016B5A51".parse().unwrap(), IndexLayerMetadata { // serde_json should always parse this but this might be a double with jq for // example. file_size: 9007199254741001, + generation: Generation::none() }) ]), disk_consistent_lsn: "0/16960E8".parse::().unwrap(), @@ -200,11 +214,13 @@ mod tests { layer_metadata: HashMap::from([ ("000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9".parse().unwrap(), IndexLayerMetadata { file_size: 25600000, + generation: Generation::none() }), ("000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__00000000016B59D8-00000000016B5A51".parse().unwrap(), IndexLayerMetadata { // serde_json should always parse this but this might be a double with jq for // example. file_size: 9007199254741001, + generation: Generation::none() }) ]), disk_consistent_lsn: "0/16960E8".parse::().unwrap(), @@ -237,11 +253,13 @@ mod tests { layer_metadata: HashMap::from([ ("000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9".parse().unwrap(), IndexLayerMetadata { file_size: 25600000, + generation: Generation::none() }), ("000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__00000000016B59D8-00000000016B5A51".parse().unwrap(), IndexLayerMetadata { // serde_json should always parse this but this might be a double with jq for // example. file_size: 9007199254741001, + generation: Generation::none() }) ]), disk_consistent_lsn: "0/16960E8".parse::().unwrap(), diff --git a/pageserver/src/tenant/remote_timeline_client/upload.rs b/pageserver/src/tenant/remote_timeline_client/upload.rs index a805e9bd60..c442c4f445 100644 --- a/pageserver/src/tenant/remote_timeline_client/upload.rs +++ b/pageserver/src/tenant/remote_timeline_client/upload.rs @@ -5,7 +5,11 @@ use fail::fail_point; use std::{io::ErrorKind, path::Path}; use tokio::fs; -use crate::{config::PageServerConf, tenant::remote_timeline_client::index::IndexPart}; +use super::Generation; +use crate::{ + config::PageServerConf, + tenant::remote_timeline_client::{index::IndexPart, remote_index_path, remote_path}, +}; use remote_storage::GenericRemoteStorage; use utils::id::{TenantId, TimelineId}; @@ -15,10 +19,10 @@ use tracing::info; /// Serializes and uploads the given index part data to the remote storage. pub(super) async fn upload_index_part<'a>( - conf: &'static PageServerConf, storage: &'a GenericRemoteStorage, tenant_id: &TenantId, timeline_id: &TimelineId, + generation: Generation, index_part: &'a IndexPart, ) -> anyhow::Result<()> { tracing::trace!("uploading new index part"); @@ -27,20 +31,16 @@ pub(super) async fn upload_index_part<'a>( bail!("failpoint before-upload-index") }); - let index_part_bytes = serde_json::to_vec(&index_part) - .context("Failed to serialize index part file into bytes")?; + let index_part_bytes = + serde_json::to_vec(&index_part).context("serialize index part file into bytes")?; let index_part_size = index_part_bytes.len(); let index_part_bytes = tokio::io::BufReader::new(std::io::Cursor::new(index_part_bytes)); - let index_part_path = conf - .metadata_path(tenant_id, timeline_id) - .with_file_name(IndexPart::FILE_NAME); - let storage_path = conf.remote_path(&index_part_path)?; - + let remote_path = remote_index_path(tenant_id, timeline_id, generation); storage - .upload_storage_object(Box::new(index_part_bytes), index_part_size, &storage_path) + .upload_storage_object(Box::new(index_part_bytes), index_part_size, &remote_path) .await - .with_context(|| format!("Failed to upload index part for '{tenant_id} / {timeline_id}'")) + .with_context(|| format!("upload index part for '{tenant_id} / {timeline_id}'")) } /// Attempts to upload given layer files. @@ -52,12 +52,13 @@ pub(super) async fn upload_timeline_layer<'a>( storage: &'a GenericRemoteStorage, source_path: &'a Path, known_metadata: &'a LayerFileMetadata, + generation: Generation, ) -> anyhow::Result<()> { fail_point!("before-upload-layer", |_| { bail!("failpoint before-upload-layer") }); - let storage_path = conf.remote_path(source_path)?; + let storage_path = remote_path(conf, source_path, generation)?; let source_file_res = fs::File::open(&source_path).await; let source_file = match source_file_res { Ok(source_file) => source_file, @@ -70,16 +71,15 @@ pub(super) async fn upload_timeline_layer<'a>( info!(path = %source_path.display(), "File to upload doesn't exist. Likely the file has been deleted and an upload is not required any more."); return Ok(()); } - Err(e) => Err(e) - .with_context(|| format!("Failed to open a source file for layer {source_path:?}"))?, + Err(e) => { + Err(e).with_context(|| format!("open a source file for layer {source_path:?}"))? + } }; let fs_size = source_file .metadata() .await - .with_context(|| { - format!("Failed to get the source file metadata for layer {source_path:?}") - })? + .with_context(|| format!("get the source file metadata for layer {source_path:?}"))? .len(); let metadata_size = known_metadata.file_size(); @@ -87,19 +87,13 @@ pub(super) async fn upload_timeline_layer<'a>( bail!("File {source_path:?} has its current FS size {fs_size} diferent from initially determined {metadata_size}"); } - let fs_size = usize::try_from(fs_size).with_context(|| { - format!("File {source_path:?} size {fs_size} could not be converted to usize") - })?; + let fs_size = usize::try_from(fs_size) + .with_context(|| format!("convert {source_path:?} size {fs_size} usize"))?; storage .upload(source_file, fs_size, &storage_path, None) .await - .with_context(|| { - format!( - "Failed to upload a layer from local path '{}'", - source_path.display() - ) - })?; + .with_context(|| format!("upload layer from local path '{}'", source_path.display()))?; Ok(()) } diff --git a/pageserver/src/tenant/timeline.rs b/pageserver/src/tenant/timeline.rs index 04da85a241..f0ae385806 100644 --- a/pageserver/src/tenant/timeline.rs +++ b/pageserver/src/tenant/timeline.rs @@ -67,6 +67,7 @@ use postgres_connection::PgConnectionConfig; use postgres_ffi::to_pg_timestamp; use utils::{ completion, + generation::Generation, id::{TenantId, TimelineId}, lsn::{AtomicLsn, Lsn, RecordLsn}, seqwait::SeqWait, @@ -152,6 +153,10 @@ pub struct Timeline { pub tenant_id: TenantId, pub timeline_id: TimelineId, + /// The generation of the tenant that instantiated us: this is used for safety when writing remote objects. + /// Never changes for the lifetime of this [`Timeline`] object. + generation: Generation, + pub pg_version: u32, /// The tuple has two elements. @@ -1199,7 +1204,7 @@ impl Timeline { Ok(delta) => Some(delta), }; - let layer_metadata = LayerFileMetadata::new(layer_file_size); + let layer_metadata = LayerFileMetadata::new(layer_file_size, self.generation); let new_remote_layer = Arc::new(match local_layer.filename() { LayerFileName::Image(image_name) => RemoteLayer::new_img( @@ -1377,6 +1382,7 @@ impl Timeline { ancestor: Option>, timeline_id: TimelineId, tenant_id: TenantId, + generation: Generation, walredo_mgr: Arc, resources: TimelineResources, pg_version: u32, @@ -1406,6 +1412,7 @@ impl Timeline { myself: myself.clone(), timeline_id, tenant_id, + generation, pg_version, layers: Arc::new(tokio::sync::RwLock::new(LayerManager::create())), wanted_image_layers: Mutex::new(None), @@ -1615,6 +1622,9 @@ impl Timeline { let (conf, tenant_id, timeline_id) = (self.conf, self.tenant_id, self.timeline_id); let span = tracing::Span::current(); + // Copy to move into the task we're about to spawn + let generation = self.generation; + let (loaded_layers, to_sync, total_physical_size) = tokio::task::spawn_blocking({ move || { let _g = span.entered(); @@ -1656,8 +1666,12 @@ impl Timeline { ); } - let decided = - init::reconcile(discovered_layers, index_part.as_ref(), disk_consistent_lsn); + let decided = init::reconcile( + discovered_layers, + index_part.as_ref(), + disk_consistent_lsn, + generation, + ); let mut loaded_layers = Vec::new(); let mut needs_upload = Vec::new(); @@ -2669,7 +2683,7 @@ impl Timeline { ( HashMap::from([( layer.filename(), - LayerFileMetadata::new(layer.layer_desc().file_size), + LayerFileMetadata::new(layer.layer_desc().file_size, self.generation), )]), Some(layer), ) @@ -3065,7 +3079,10 @@ impl Timeline { .metadata() .with_context(|| format!("reading metadata of layer file {}", path.file_name()))?; - layer_paths_to_upload.insert(path, LayerFileMetadata::new(metadata.len())); + layer_paths_to_upload.insert( + path, + LayerFileMetadata::new(metadata.len(), self.generation), + ); self.metrics .resident_physical_size_gauge @@ -3740,7 +3757,7 @@ impl Timeline { if let Some(remote_client) = &self.remote_client { remote_client.schedule_layer_file_upload( &l.filename(), - &LayerFileMetadata::new(metadata.len()), + &LayerFileMetadata::new(metadata.len(), self.generation), )?; } @@ -3749,7 +3766,10 @@ impl Timeline { .resident_physical_size_gauge .add(metadata.len()); - new_layer_paths.insert(new_delta_path, LayerFileMetadata::new(metadata.len())); + new_layer_paths.insert( + new_delta_path, + LayerFileMetadata::new(metadata.len(), self.generation), + ); l.access_stats().record_residence_event( LayerResidenceStatus::Resident, LayerResidenceEventReason::LayerCreate, diff --git a/pageserver/src/tenant/timeline/init.rs b/pageserver/src/tenant/timeline/init.rs index a270d96677..33effb4318 100644 --- a/pageserver/src/tenant/timeline/init.rs +++ b/pageserver/src/tenant/timeline/init.rs @@ -7,6 +7,7 @@ use crate::{ index::{IndexPart, LayerFileMetadata}, }, storage_layer::LayerFileName, + Generation, }, METADATA_FILE_NAME, }; @@ -104,6 +105,7 @@ pub(super) fn reconcile( discovered: Vec<(LayerFileName, u64)>, index_part: Option<&IndexPart>, disk_consistent_lsn: Lsn, + generation: Generation, ) -> Vec<(LayerFileName, Result)> { use Decision::*; @@ -112,7 +114,15 @@ pub(super) fn reconcile( let mut discovered = discovered .into_iter() - .map(|(name, file_size)| (name, (Some(LayerFileMetadata::new(file_size)), None))) + .map(|(name, file_size)| { + ( + name, + // The generation here will be corrected to match IndexPart in the merge below, unless + // it is not in IndexPart, in which case using our current generation makes sense + // because it will be uploaded in this generation. + (Some(LayerFileMetadata::new(file_size, generation)), None), + ) + }) .collect::(); // merge any index_part information, when available @@ -137,7 +147,11 @@ pub(super) fn reconcile( Err(FutureLayer { local }) } else { Ok(match (local, remote) { - (Some(local), Some(remote)) if local != remote => UseRemote { local, remote }, + (Some(local), Some(remote)) if local != remote => { + assert_eq!(local.generation, remote.generation); + + UseRemote { local, remote } + } (Some(x), Some(_)) => UseLocal(x), (None, Some(x)) => Evicted(x), (Some(x), None) => NeedsUpload(x), diff --git a/pageserver/src/tenant/upload_queue.rs b/pageserver/src/tenant/upload_queue.rs index 6026825b0d..28822335b0 100644 --- a/pageserver/src/tenant/upload_queue.rs +++ b/pageserver/src/tenant/upload_queue.rs @@ -1,6 +1,7 @@ use crate::metrics::RemoteOpFileKind; use super::storage_layer::LayerFileName; +use super::Generation; use crate::tenant::metadata::TimelineMetadata; use crate::tenant::remote_timeline_client::index::IndexPart; use crate::tenant::remote_timeline_client::index::LayerFileMetadata; @@ -205,6 +206,7 @@ pub(crate) struct Delete { pub(crate) file_kind: RemoteOpFileKind, pub(crate) layer_file_name: LayerFileName, pub(crate) scheduled_from_timeline_delete: bool, + pub(crate) generation: Generation, } #[derive(Debug)] @@ -228,17 +230,21 @@ impl std::fmt::Display for UploadOp { UploadOp::UploadLayer(path, metadata) => { write!( f, - "UploadLayer({}, size={:?})", + "UploadLayer({}, size={:?}, gen={:?})", path.file_name(), - metadata.file_size() + metadata.file_size(), + metadata.generation, ) } - UploadOp::UploadMetadata(_, lsn) => write!(f, "UploadMetadata(lsn: {})", lsn), + UploadOp::UploadMetadata(_, lsn) => { + write!(f, "UploadMetadata(lsn: {})", lsn) + } UploadOp::Delete(delete) => write!( f, - "Delete(path: {}, scheduled_from_timeline_delete: {})", + "Delete(path: {}, scheduled_from_timeline_delete: {}, gen: {:?})", delete.layer_file_name.file_name(), - delete.scheduled_from_timeline_delete + delete.scheduled_from_timeline_delete, + delete.generation ), UploadOp::Barrier(_) => write!(f, "Barrier"), } From b9c111962fd1ee274fd74090c81154cfc8272bf8 Mon Sep 17 00:00:00 2001 From: Nikita Kalyanov <44959448+nikitakalyanov@users.noreply.github.com> Date: Thu, 31 Aug 2023 12:23:51 +0300 Subject: [PATCH 08/10] pass JWT to management API (#5151) support authentication with JWT from env for proxy calls to mgmt API --- proxy/src/console/provider/neon.rs | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/proxy/src/console/provider/neon.rs b/proxy/src/console/provider/neon.rs index 3f4cee6e34..3322d5a5be 100644 --- a/proxy/src/console/provider/neon.rs +++ b/proxy/src/console/provider/neon.rs @@ -16,12 +16,21 @@ use tracing::{error, info, info_span, warn, Instrument}; pub struct Api { endpoint: http::Endpoint, caches: &'static ApiCaches, + jwt: String, } impl Api { /// Construct an API object containing the auth parameters. pub fn new(endpoint: http::Endpoint, caches: &'static ApiCaches) -> Self { - Self { endpoint, caches } + let jwt: String = match std::env::var("NEON_PROXY_TO_CONTROLPLANE_TOKEN") { + Ok(v) => v, + Err(_) => "".to_string(), + }; + Self { + endpoint, + caches, + jwt, + } } pub fn url(&self) -> &str { @@ -39,6 +48,7 @@ impl Api { .endpoint .get("proxy_get_role_secret") .header("X-Request-ID", &request_id) + .header("Authorization", &self.jwt) .query(&[("session_id", extra.session_id)]) .query(&[ ("application_name", extra.application_name), @@ -83,6 +93,7 @@ impl Api { .endpoint .get("proxy_wake_compute") .header("X-Request-ID", &request_id) + .header("Authorization", &self.jwt) .query(&[("session_id", extra.session_id)]) .query(&[ ("application_name", extra.application_name), From 300a5aa05e66899354b8d1e1a3ddce2c8d065603 Mon Sep 17 00:00:00 2001 From: John Spray Date: Thu, 31 Aug 2023 10:40:46 +0100 Subject: [PATCH 09/10] pageserver: fix test v4_indexpart_is_parsed (#5157) ## Problem Two recent PRs raced: - https://github.com/neondatabase/neon/pull/5153 - https://github.com/neondatabase/neon/pull/5140 ## Summary of changes Add missing `generation` argument to IndexLayerMetadata construction --- pageserver/src/tenant/remote_timeline_client/index.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pageserver/src/tenant/remote_timeline_client/index.rs b/pageserver/src/tenant/remote_timeline_client/index.rs index 37ed0e2c3f..9cc5256568 100644 --- a/pageserver/src/tenant/remote_timeline_client/index.rs +++ b/pageserver/src/tenant/remote_timeline_client/index.rs @@ -335,11 +335,13 @@ mod tests { layer_metadata: HashMap::from([ ("000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9".parse().unwrap(), IndexLayerMetadata { file_size: 25600000, + generation: Generation::none() }), ("000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__00000000016B59D8-00000000016B5A51".parse().unwrap(), IndexLayerMetadata { // serde_json should always parse this but this might be a double with jq for // example. file_size: 9007199254741001, + generation: Generation::none() }) ]), disk_consistent_lsn: "0/16960E8".parse::().unwrap(), From 43bb8bfdbba4b27773a975123b046dcb45242e7c Mon Sep 17 00:00:00 2001 From: John Spray Date: Thu, 31 Aug 2023 10:42:32 +0100 Subject: [PATCH 10/10] pageserver: fix flake in test_timeline_deletion_with_files_stuck_in_upload_queue (#5149) ## Problem Test failing on a different ERROR log than it anticipated. Closes: https://github.com/neondatabase/neon/issues/5148 ## Summary of changes Add the "could not flush frozen layer" error log to the permitted errors. --- test_runner/regress/test_remote_storage.py | 1 + 1 file changed, 1 insertion(+) diff --git a/test_runner/regress/test_remote_storage.py b/test_runner/regress/test_remote_storage.py index b865e3ce24..0bd365efaa 100644 --- a/test_runner/regress/test_remote_storage.py +++ b/test_runner/regress/test_remote_storage.py @@ -604,6 +604,7 @@ def test_timeline_deletion_with_files_stuck_in_upload_queue( checkpoint_allowed_to_fail.set() env.pageserver.allowed_errors.append( ".* ERROR .*Error processing HTTP request: InternalServerError\\(timeline is Stopping" + ".* ERROR .*[Cc]ould not flush frozen layer.*" ) # Generous timeout, because currently deletions can get blocked waiting for compaction