diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index 6dcf988191..6d89ce9994 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -661,6 +661,9 @@ jobs: project: nrdv0s4kcs push: true tags: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/neon:depot-${{needs.tag.outputs.build-tag}} + build-args: | + GIT_VERSION=${{ github.sha }} + REPOSITORY=369495373322.dkr.ecr.eu-central-1.amazonaws.com compute-tools-image: runs-on: [ self-hosted, gen3, large ] @@ -775,7 +778,7 @@ jobs: run: shell: sh -eu {0} env: - VM_BUILDER_VERSION: v0.7.3-alpha3 + VM_BUILDER_VERSION: v0.8.0 steps: - name: Checkout diff --git a/Cargo.lock b/Cargo.lock index 6501d9557d..d390df94e0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -17,17 +17,6 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" -[[package]] -name = "ahash" -version = "0.7.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fcb51a0695d8f838b1ee009b3fbf66bda078cd64590202a864a8f3e8c4315c47" -dependencies = [ - "getrandom", - "once_cell", - "version_check", -] - [[package]] name = "ahash" version = "0.8.3" @@ -41,9 +30,9 @@ dependencies = [ [[package]] name = "aho-corasick" -version = "0.7.20" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc936419f96fa211c1b9166887b38e5e40b19958e5b895be7c1f93adec7071ac" +checksum = "67fc08ce920c31afb70f013dcce1bfc3a3195de6a228474e45e1f145b36f8d04" dependencies = [ "memchr", ] @@ -65,9 +54,9 @@ checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" [[package]] name = "anstream" -version = "0.3.0" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e579a7752471abc2a8268df8b20005e3eadd975f585398f17efcfd8d4927371" +checksum = "0ca84f3628370c59db74ee214b3263d58f9aadd9b4fe7e711fd87dc452b7f163" dependencies = [ "anstyle", "anstyle-parse", @@ -104,9 +93,9 @@ dependencies = [ [[package]] name = "anstyle-wincon" -version = "1.0.0" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4bcd8291a340dd8ac70e18878bc4501dd7b4ff970cfa21c207d36ece51ea88fd" +checksum = "180abfa45703aebe0093f79badacc01b8fd4ea2e35118747e5811127f926e188" dependencies = [ "anstyle", "windows-sys 0.48.0", @@ -114,9 +103,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.70" +version = "1.0.71" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7de8ce5e0f9f8d88245311066a578d72b7af3e7088f32783804676302df237e4" +checksum = "9c7d0618f0e0b7e8ff11427422b64564d5fb0be1940354bfe2e0529b18a9d9b8" dependencies = [ "backtrace", ] @@ -188,7 +177,7 @@ checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193" dependencies = [ "proc-macro2", "quote", - "syn 2.0.15", + "syn 2.0.16", ] [[package]] @@ -199,7 +188,7 @@ checksum = "b9ccdd8f2a161be9bd5c023df56f1b2a0bd1d83872ae53b71a84a12c9bf6e842" dependencies = [ "proc-macro2", "quote", - "syn 2.0.15", + "syn 2.0.16", ] [[package]] @@ -230,9 +219,9 @@ checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" [[package]] name = "aws-config" -version = "0.55.2" +version = "0.55.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc00553f5f3c06ffd4510a9d576f92143618706c45ea6ff81e84ad9be9588abd" +checksum = "bcdcf0d683fe9c23d32cf5b53c9918ea0a500375a9fb20109802552658e576c9" dependencies = [ "aws-credential-types", "aws-http", @@ -256,9 +245,9 @@ dependencies = [ [[package]] name = "aws-credential-types" -version = "0.55.2" +version = "0.55.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4cb57ac6088805821f78d282c0ba8aec809f11cbee10dda19a97b03ab040ccc2" +checksum = "1fcdb2f7acbc076ff5ad05e7864bdb191ca70a6fd07668dc3a1a8bcd051de5ae" dependencies = [ "aws-smithy-async", "aws-smithy-types", @@ -270,9 +259,9 @@ dependencies = [ [[package]] name = "aws-endpoint" -version = "0.55.2" +version = "0.55.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c5f6f84a4f46f95a9bb71d9300b73cd67eb868bc43ae84f66ad34752299f4ac" +checksum = "8cce1c41a6cfaa726adee9ebb9a56fcd2bbfd8be49fd8a04c5e20fd968330b04" dependencies = [ "aws-smithy-http", "aws-smithy-types", @@ -284,9 +273,9 @@ dependencies = [ [[package]] name = "aws-http" -version = "0.55.2" +version = "0.55.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a754683c322f7dc5167484266489fdebdcd04d26e53c162cad1f3f949f2c5671" +checksum = "aadbc44e7a8f3e71c8b374e03ecd972869eb91dd2bc89ed018954a52ba84bc44" dependencies = [ "aws-credential-types", "aws-smithy-http", @@ -303,9 +292,9 @@ dependencies = [ [[package]] name = "aws-sdk-s3" -version = "0.25.1" +version = "0.27.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "392b9811ca489747ac84349790e49deaa1f16631949e7dd4156000251c260eae" +checksum = "37c77060408d653d3efa6ea7b66c1389bc35a0342352984c8bf8bcb814a8fc27" dependencies = [ "aws-credential-types", "aws-endpoint", @@ -336,9 +325,9 @@ dependencies = [ [[package]] name = "aws-sdk-sts" -version = "0.27.0" +version = "0.28.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d0fbe3c2c342bc8dfea4bb43937405a8ec06f99140a0dcb9c7b59e54dfa93a1" +checksum = "265fac131fbfc188e5c3d96652ea90ecc676a934e3174eaaee523c6cec040b3b" dependencies = [ "aws-credential-types", "aws-endpoint", @@ -362,9 +351,9 @@ dependencies = [ [[package]] name = "aws-sig-auth" -version = "0.55.2" +version = "0.55.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "84dc92a63ede3c2cbe43529cb87ffa58763520c96c6a46ca1ced80417afba845" +checksum = "3b94acb10af0c879ecd5c7bdf51cda6679a0a4f4643ce630905a77673bfa3c61" dependencies = [ "aws-credential-types", "aws-sigv4", @@ -377,9 +366,9 @@ dependencies = [ [[package]] name = "aws-sigv4" -version = "0.55.2" +version = "0.55.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "392fefab9d6fcbd76d518eb3b1c040b84728ab50f58df0c3c53ada4bea9d327e" +checksum = "9d2ce6f507be68e968a33485ced670111d1cbad161ddbbab1e313c03d37d8f4c" dependencies = [ "aws-smithy-eventstream", "aws-smithy-http", @@ -398,9 +387,9 @@ dependencies = [ [[package]] name = "aws-smithy-async" -version = "0.55.2" +version = "0.55.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae23b9fe7a07d0919000116c4c5c0578303fbce6fc8d32efca1f7759d4c20faf" +checksum = "13bda3996044c202d75b91afeb11a9afae9db9a721c6a7a427410018e286b880" dependencies = [ "futures-util", "pin-project-lite", @@ -410,9 +399,9 @@ dependencies = [ [[package]] name = "aws-smithy-checksums" -version = "0.55.2" +version = "0.55.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6367acbd6849b8c7c659e166955531274ae147bf83ab4312885991f6b6706cb" +checksum = "07ed8b96d95402f3f6b8b57eb4e0e45ee365f78b1a924faf20ff6e97abf1eae6" dependencies = [ "aws-smithy-http", "aws-smithy-types", @@ -431,9 +420,9 @@ dependencies = [ [[package]] name = "aws-smithy-client" -version = "0.55.2" +version = "0.55.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5230d25d244a51339273b8870f0f77874cd4449fb4f8f629b21188ae10cfc0ba" +checksum = "0a86aa6e21e86c4252ad6a0e3e74da9617295d8d6e374d552be7d3059c41cedd" dependencies = [ "aws-smithy-async", "aws-smithy-http", @@ -444,7 +433,7 @@ dependencies = [ "http", "http-body", "hyper", - "hyper-rustls", + "hyper-rustls 0.23.2", "lazy_static", "pin-project-lite", "rustls 0.20.8", @@ -455,9 +444,9 @@ dependencies = [ [[package]] name = "aws-smithy-eventstream" -version = "0.55.2" +version = "0.55.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22d2a2bcc16e5c4d949ffd2b851da852b9bbed4bb364ed4ae371b42137ca06d9" +checksum = "460c8da5110835e3d9a717c61f5556b20d03c32a1dec57f8fc559b360f733bb8" dependencies = [ "aws-smithy-types", "bytes", @@ -466,9 +455,9 @@ dependencies = [ [[package]] name = "aws-smithy-http" -version = "0.55.2" +version = "0.55.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b60e2133beb9fe6ffe0b70deca57aaeff0a35ad24a9c6fab2fd3b4f45b99fdb5" +checksum = "2b3b693869133551f135e1f2c77cb0b8277d9e3e17feaf2213f735857c4f0d28" dependencies = [ "aws-smithy-eventstream", "aws-smithy-types", @@ -489,9 +478,9 @@ dependencies = [ [[package]] name = "aws-smithy-http-tower" -version = "0.55.2" +version = "0.55.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a4d94f556c86a0dd916a5d7c39747157ea8cb909ca469703e20fee33e448b67" +checksum = "3ae4f6c5798a247fac98a867698197d9ac22643596dc3777f0c76b91917616b9" dependencies = [ "aws-smithy-http", "aws-smithy-types", @@ -505,18 +494,18 @@ dependencies = [ [[package]] name = "aws-smithy-json" -version = "0.55.2" +version = "0.55.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ce3d6e6ebb00b2cce379f079ad5ec508f9bcc3a9510d9b9c1840ed1d6f8af39" +checksum = "23f9f42fbfa96d095194a632fbac19f60077748eba536eb0b9fecc28659807f8" dependencies = [ "aws-smithy-types", ] [[package]] name = "aws-smithy-query" -version = "0.55.2" +version = "0.55.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d58edfca32ef9bfbc1ca394599e17ea329cb52d6a07359827be74235b64b3298" +checksum = "98819eb0b04020a1c791903533b638534ae6c12e2aceda3e6e6fba015608d51d" dependencies = [ "aws-smithy-types", "urlencoding", @@ -524,9 +513,9 @@ dependencies = [ [[package]] name = "aws-smithy-types" -version = "0.55.2" +version = "0.55.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "58db46fc1f4f26be01ebdb821751b4e2482cd43aa2b64a0348fb89762defaffa" +checksum = "16a3d0bf4f324f4ef9793b86a1701d9700fbcdbd12a846da45eed104c634c6e8" dependencies = [ "base64-simd", "itoa", @@ -537,18 +526,18 @@ dependencies = [ [[package]] name = "aws-smithy-xml" -version = "0.55.2" +version = "0.55.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb557fe4995bd9ec87fb244bbb254666a971dc902a783e9da8b7711610e9664c" +checksum = "b1b9d12875731bd07e767be7baad95700c3137b56730ec9ddeedb52a5e5ca63b" dependencies = [ "xmlparser", ] [[package]] name = "aws-types" -version = "0.55.2" +version = "0.55.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de0869598bfe46ec44ffe17e063ed33336e59df90356ca8ff0e8da6f7c1d994b" +checksum = "6dd209616cc8d7bfb82f87811a5c655dc97537f592689b18743bddf5dc5c4829" dependencies = [ "aws-credential-types", "aws-smithy-async", @@ -562,9 +551,9 @@ dependencies = [ [[package]] name = "axum" -version = "0.6.15" +version = "0.6.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b32c5ea3aabaf4deb5f5ced2d688ec0844c881c9e6c696a8b769a05fc691e62" +checksum = "f8175979259124331c1d7bf6586ee7e0da434155e4b2d48ec2c8386281d8df39" dependencies = [ "async-trait", "axum-core", @@ -634,9 +623,9 @@ checksum = "0ea22880d78093b0cbe17c89f64a7d457941e65759157ec6cb31a31d652b05e5" [[package]] name = "base64" -version = "0.21.0" +version = "0.21.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4a4ddaa51a5bc52a6948f74c06d20aaaddb71924eab79b8c97a8c556e942d6a" +checksum = "3f1e31e207a6b8fb791a38ea3105e6cb541f55e4d029902d3039a4ad07cc4105" [[package]] name = "base64-simd" @@ -670,13 +659,13 @@ dependencies = [ "lazycell", "log", "peeking_take_while", - "prettyplease 0.2.4", + "prettyplease 0.2.6", "proc-macro2", "quote", "regex", "rustc-hash", "shlex", - "syn 2.0.15", + "syn 2.0.16", "which", ] @@ -697,9 +686,9 @@ dependencies = [ [[package]] name = "bstr" -version = "1.4.0" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3d4260bcc2e8fc9df1eac4919a720effeb63a3f0952f5bf4944adfa18897f09" +checksum = "a246e68bb43f6cd9db24bea052a53e40405417c5fb372e3d1a8a7f770a564ef5" dependencies = [ "memchr", "once_cell", @@ -709,9 +698,9 @@ dependencies = [ [[package]] name = "bumpalo" -version = "3.12.0" +version = "3.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d261e256854913907f67ed06efbc3338dfe6179796deefc1ff763fc1aee5535" +checksum = "a3e2c3daef883ecc1b5d58c15adae93470a91d425f3532ba1695849656af3fc1" [[package]] name = "byteorder" @@ -780,9 +769,9 @@ dependencies = [ [[package]] name = "ciborium" -version = "0.2.0" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0c137568cc60b904a7724001b35ce2630fd00d5d84805fbb608ab89509d788f" +checksum = "effd91f6c78e5a4ace8a5d3c0b6bfaec9e2baaef55f3efc00e45fb2e477ee926" dependencies = [ "ciborium-io", "ciborium-ll", @@ -791,15 +780,15 @@ dependencies = [ [[package]] name = "ciborium-io" -version = "0.2.0" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "346de753af073cc87b52b2083a506b38ac176a44cfb05497b622e27be899b369" +checksum = "cdf919175532b369853f5d5e20b26b43112613fd6fe7aee757e35f7a44642656" [[package]] name = "ciborium-ll" -version = "0.2.0" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "213030a2b5a4e0c0892b6652260cf6ccac84827b83a85a534e178e3906c4cf1b" +checksum = "defaa24ecc093c77630e6c15e17c51f5e187bf35ee514f4e2d67baaa96dae22b" dependencies = [ "ciborium-io", "half", @@ -818,9 +807,9 @@ dependencies = [ [[package]] name = "clap" -version = "3.2.23" +version = "3.2.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "71655c45cb9845d3270c9d6df84ebe72b4dad3c2ba3f7023ad47c144e4e473a5" +checksum = "4ea181bf566f71cb9a5d17a59e1871af638180a18fb0035c92ae62b705207123" dependencies = [ "bitflags", "clap_lex 0.2.4", @@ -830,9 +819,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.2.2" +version = "4.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b802d85aaf3a1cdb02b224ba472ebdea62014fccfcb269b95a4d76443b5ee5a" +checksum = "93aae7a4192245f70fe75dd9157fc7b4a5bf53e88d30bd4396f7d8f9284d5acc" dependencies = [ "clap_builder", "clap_derive", @@ -841,27 +830,27 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.2.2" +version = "4.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14a1a858f532119338887a4b8e1af9c60de8249cd7bafd68036a489e261e37b6" +checksum = "4f423e341edefb78c9caba2d9c7f7687d0e72e89df3ce3394554754393ac3990" dependencies = [ "anstream", "anstyle", "bitflags", - "clap_lex 0.4.1", + "clap_lex 0.5.0", "strsim", ] [[package]] name = "clap_derive" -version = "4.2.0" +version = "4.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f9644cd56d6b87dbe899ef8b053e331c0637664e9e21a33dfcdc36093f5c5c4" +checksum = "191d9573962933b4027f932c600cd252ce27a8ad5979418fe78e43c07996f27b" dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.15", + "syn 2.0.16", ] [[package]] @@ -875,9 +864,9 @@ dependencies = [ [[package]] name = "clap_lex" -version = "0.4.1" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a2dd5a6fe8c6e3502f568a6353e5273bbb15193ad9a89e457b9970798efbea1" +checksum = "2da6da31387c7e4ef160ffab6d5e7f00c42626fe39aea70a7b0f1773f7dd6c1b" [[package]] name = "close_fds" @@ -889,16 +878,6 @@ dependencies = [ "libc", ] -[[package]] -name = "codespan-reporting" -version = "0.11.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3538270d33cc669650c4b093848450d380def10c331d38c768e34cac80576e6e" -dependencies = [ - "termcolor", - "unicode-width", -] - [[package]] name = "colorchoice" version = "1.0.0" @@ -936,7 +915,7 @@ version = "0.1.0" dependencies = [ "anyhow", "chrono", - "clap 4.2.2", + "clap 4.3.0", "compute_api", "futures", "hyper", @@ -998,7 +977,7 @@ name = "control_plane" version = "0.1.0" dependencies = [ "anyhow", - "clap 4.2.2", + "clap 4.3.0", "comfy-table", "compute_api", "git-version", @@ -1041,9 +1020,9 @@ checksum = "e496a50fda8aacccc86d7529e2c1e0892dbd0f898a6b5645b5561b89c3210efa" [[package]] name = "cpufeatures" -version = "0.2.6" +version = "0.2.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "280a9f2d8b3a38871a3c8a46fb80db65e5e5ed97da80c4d08bf27fb63e35e181" +checksum = "3e4c1eaa2012c47becbbad2ab175484c2a84d1185b566fb2cc5b8707343dfe58" dependencies = [ "libc", ] @@ -1076,7 +1055,7 @@ dependencies = [ "atty", "cast", "ciborium", - "clap 3.2.23", + "clap 3.2.25", "criterion-plot", "itertools", "lazy_static", @@ -1186,55 +1165,11 @@ dependencies = [ "typenum", ] -[[package]] -name = "cxx" -version = "1.0.94" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f61f1b6389c3fe1c316bf8a4dccc90a38208354b330925bce1f74a6c4756eb93" -dependencies = [ - "cc", - "cxxbridge-flags", - "cxxbridge-macro", - "link-cplusplus", -] - -[[package]] -name = "cxx-build" -version = "1.0.94" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12cee708e8962df2aeb38f594aae5d827c022b6460ac71a7a3e2c3c2aae5a07b" -dependencies = [ - "cc", - "codespan-reporting", - "once_cell", - "proc-macro2", - "quote", - "scratch", - "syn 2.0.15", -] - -[[package]] -name = "cxxbridge-flags" -version = "1.0.94" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7944172ae7e4068c533afbb984114a56c46e9ccddda550499caa222902c7f7bb" - -[[package]] -name = "cxxbridge-macro" -version = "1.0.94" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2345488264226bf682893e25de0769f3360aac9957980ec49361b083ddaa5bc5" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.15", -] - [[package]] name = "darling" -version = "0.14.4" +version = "0.20.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b750cb3417fd1b327431a470f388520309479ab0bf5e323505daf0290cd3850" +checksum = "0558d22a7b463ed0241e993f76f09f30b126687447751a8638587b864e4b3944" dependencies = [ "darling_core", "darling_macro", @@ -1242,27 +1177,27 @@ dependencies = [ [[package]] name = "darling_core" -version = "0.14.4" +version = "0.20.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "109c1ca6e6b7f82cc233a97004ea8ed7ca123a9af07a8230878fcfda9b158bf0" +checksum = "ab8bfa2e259f8ee1ce5e97824a3c55ec4404a0d772ca7fa96bf19f0752a046eb" dependencies = [ "fnv", "ident_case", "proc-macro2", "quote", "strsim", - "syn 1.0.109", + "syn 2.0.16", ] [[package]] name = "darling_macro" -version = "0.14.4" +version = "0.20.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4aab4dbc9f7611d8b55048a3a16d2d010c2c8334e46304b40ac1cc14bf3b48e" +checksum = "29a358ff9f12ec09c3e61fef9b5a9902623a695a46a917b07f269bff1445611a" dependencies = [ "darling_core", "quote", - "syn 1.0.109", + "syn 2.0.16", ] [[package]] @@ -1280,9 +1215,9 @@ dependencies = [ [[package]] name = "data-encoding" -version = "2.3.3" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23d8666cb01533c39dde32bcbab8e227b4ed6679b2c925eba05feabea39508fb" +checksum = "c2e66c9d817f1720209181c316d28635c050fa304f9c79e47a520882661b7308" [[package]] name = "debugid" @@ -1310,9 +1245,9 @@ dependencies = [ [[package]] name = "digest" -version = "0.10.6" +version = "0.10.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8168378f4e5023e7218c89c891c0fd8ecdb5e5e4f18cb78f38cf245dd021e76f" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ "block-buffer", "crypto-common", @@ -1321,13 +1256,13 @@ dependencies = [ [[package]] name = "displaydoc" -version = "0.2.3" +version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3bf95dc3f046b9da4f2d51833c0d3547d8564ef6910f5c1ed130306a75b92886" +checksum = "487585f4d0c6655fe74905e2504d8ad6908e4db67f744eb140876906c2f3175d" dependencies = [ "proc-macro2", "quote", - "syn 1.0.109", + "syn 2.0.16", ] [[package]] @@ -1367,23 +1302,23 @@ dependencies = [ [[package]] name = "enumset" -version = "1.0.12" +version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19be8061a06ab6f3a6cf21106c873578bf01bd42ad15e0311a9c76161cb1c753" +checksum = "e875f1719c16de097dee81ed675e2d9bb63096823ed3f0ca827b7dea3028bbbb" dependencies = [ "enumset_derive", ] [[package]] name = "enumset_derive" -version = "0.6.1" +version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03e7b551eba279bf0fa88b83a46330168c1560a52a94f5126f892f0b364ab3e0" +checksum = "e08b6c6ab82d70f08844964ba10c7babb716de2ecaeab9be5717918a5177d3af" dependencies = [ "darling", "proc-macro2", "quote", - "syn 1.0.109", + "syn 2.0.16", ] [[package]] @@ -1569,7 +1504,7 @@ checksum = "89ca545a94061b6365f2c7355b4b32bd20df3ff95f02da9329b34ccc3bd6ee72" dependencies = [ "proc-macro2", "quote", - "syn 2.0.15", + "syn 2.0.16", ] [[package]] @@ -1667,9 +1602,9 @@ checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b" [[package]] name = "h2" -version = "0.3.18" +version = "0.3.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17f8a914c2987b688368b5138aa05321db91f4090cf26118185672ad588bce21" +checksum = "d357c7ae988e7d2182f7d7871d0b963962420b0678b0997ce7de72001aeab782" dependencies = [ "bytes", "fnv", @@ -1704,9 +1639,6 @@ name = "hashbrown" version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" -dependencies = [ - "ahash 0.7.6", -] [[package]] name = "hashbrown" @@ -1714,16 +1646,16 @@ version = "0.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "43a3c133739dddd0d2990f9a4bdf8eb4b21ef50e4851ca85ab661199821d510e" dependencies = [ - "ahash 0.8.3", + "ahash", ] [[package]] name = "hashlink" -version = "0.8.1" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69fe1fcf8b4278d860ad0548329f892a3631fb63f82574df68275f34cdbe0ffa" +checksum = "0761a1b9491c4f2e3d66aa0f62d0fba0af9a0e2852e4d48ea506632a4b56e6aa" dependencies = [ - "hashbrown 0.12.3", + "hashbrown 0.13.2", ] [[package]] @@ -1892,6 +1824,19 @@ dependencies = [ "tokio-rustls 0.23.4", ] +[[package]] +name = "hyper-rustls" +version = "0.24.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0646026eb1b3eea4cd9ba47912ea5ce9cc07713d105b1a14698f4e6433d348b7" +dependencies = [ + "http", + "hyper", + "rustls 0.21.1", + "tokio", + "tokio-rustls 0.24.0", +] + [[package]] name = "hyper-timeout" version = "0.4.1" @@ -1933,12 +1878,11 @@ dependencies = [ [[package]] name = "iana-time-zone-haiku" -version = "0.1.1" +version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0703ae284fc167426161c2e3f1da3ea71d94b21bedbcc9494e92b28e334e3dca" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" dependencies = [ - "cxx", - "cxx-build", + "cc", ] [[package]] @@ -1999,9 +1943,9 @@ dependencies = [ [[package]] name = "io-lifetimes" -version = "1.0.10" +version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c66c74d2ae7e79a5a8f7ac924adbe38ee42a859c6539ad869eb51f0b52dc220" +checksum = "eae7b9aee968036d54dce06cebaefd919e4472e753296daccd6d344e3e2df0c2" dependencies = [ "hermit-abi 0.3.1", "libc", @@ -2022,7 +1966,7 @@ checksum = "adcf93614601c8129ddf72e2d5633df827ba6551541c6d8c59520a371475be1f" dependencies = [ "hermit-abi 0.3.1", "io-lifetimes", - "rustix 0.37.11", + "rustix 0.37.19", "windows-sys 0.48.0", ] @@ -2043,9 +1987,9 @@ checksum = "453ad9f582a441959e5f0d088b02ce04cfe8d51a8eaf077f12ac6d3e94164ca6" [[package]] name = "js-sys" -version = "0.3.61" +version = "0.3.63" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "445dde2150c55e483f3d8416706b97ec8e8237c307e5b7b4b8dd15e6af2a0730" +checksum = "2f37a4a5928311ac501dee68b3c7613a1037d0edb30c8e5427bd832d55d1b790" dependencies = [ "wasm-bindgen", ] @@ -2056,7 +2000,7 @@ version = "8.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6971da4d9c3aa03c3d8f3ff0f4155b534aad021292003895a469716b2a230378" dependencies = [ - "base64 0.21.0", + "base64 0.21.1", "pem", "ring", "serde", @@ -2098,9 +2042,9 @@ checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" [[package]] name = "libc" -version = "0.2.141" +version = "0.2.144" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3304a64d199bb964be99741b7a14d26972741915b3649639149b2479bb46f4b5" +checksum = "2b00cc1c228a6782d0f076e7b232802e0c5689d41bb5df366f2a6b6621cfdfe1" [[package]] name = "libloading" @@ -2112,15 +2056,6 @@ dependencies = [ "winapi", ] -[[package]] -name = "link-cplusplus" -version = "1.0.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ecd207c9c713c34f95a097a5b029ac2ce6010530c7b49d7fea24d977dede04f5" -dependencies = [ - "cc", -] - [[package]] name = "linux-raw-sys" version = "0.1.4" @@ -2129,9 +2064,9 @@ checksum = "f051f77a7c8e6957c0696eac88f26b0117e54f52d3fc682ab19397a8812846a4" [[package]] name = "linux-raw-sys" -version = "0.3.1" +version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d59d8c75012853d2e872fb56bc8a2e53718e2cafe1a4c823143141c6d90c322f" +checksum = "ef53942eb7bf7ff43a617b3e2c1c4a5ecf5944a7c1bc12d7ee39bbb15e5c1519" [[package]] name = "lock_api" @@ -2316,9 +2251,9 @@ dependencies = [ [[package]] name = "notify" -version = "5.1.0" +version = "5.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "58ea850aa68a06e48fdb069c0ec44d0d64c8dbffa49bf3b6f7f0a901fdea1ba9" +checksum = "729f63e1ca555a43fe3efa4f3efdf4801c479da85b432242a7b726f353c88486" dependencies = [ "bitflags", "crossbeam-channel", @@ -2329,7 +2264,7 @@ dependencies = [ "libc", "mio", "walkdir", - "windows-sys 0.42.0", + "windows-sys 0.45.0", ] [[package]] @@ -2435,7 +2370,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.15", + "syn 2.0.16", ] [[package]] @@ -2593,7 +2528,7 @@ version = "0.1.0" dependencies = [ "anyhow", "bytes", - "clap 4.2.2", + "clap 4.3.0", "git-version", "pageserver", "postgres_ffi", @@ -2612,7 +2547,7 @@ dependencies = [ "byteorder", "bytes", "chrono", - "clap 4.2.2", + "clap 4.3.0", "close_fds", "const_format", "consumption_metrics", @@ -2768,22 +2703,22 @@ dependencies = [ [[package]] name = "pin-project" -version = "1.0.12" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad29a609b6bcd67fee905812e544992d216af9d755757c05ed2d0e15a74c6ecc" +checksum = "c95a7476719eab1e366eaf73d0260af3021184f18177925b07f54b30089ceead" dependencies = [ "pin-project-internal", ] [[package]] name = "pin-project-internal" -version = "1.0.12" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "069bdb1e05adc7a8990dce9cc75370895fbe4e3d58b9b73bf1aee56359344a55" +checksum = "39407670928234ebc5e6e580247dd567ad73a3578460c5990f9503df207e8f07" dependencies = [ "proc-macro2", "quote", - "syn 1.0.109", + "syn 2.0.16", ] [[package]] @@ -2800,9 +2735,9 @@ checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" [[package]] name = "pkg-config" -version = "0.3.26" +version = "0.3.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ac9a59f73473f1b8d852421e59e64809f025994837ef743615c6d0c5b305160" +checksum = "26072860ba924cbfa98ea39c8c19b4dd6a4a25423dbdf219c1eca91aa0cf6964" [[package]] name = "plotters" @@ -2976,12 +2911,12 @@ dependencies = [ [[package]] name = "prettyplease" -version = "0.2.4" +version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ceca8aaf45b5c46ec7ed39fff75f57290368c1846d33d24a122ca81416ab058" +checksum = "3b69d39aab54d069e7f2fe8cb970493e7834601ca2d8c65fd7bbd183578080d1" dependencies = [ "proc-macro2", - "syn 2.0.15", + "syn 2.0.16", ] [[package]] @@ -2992,9 +2927,9 @@ checksum = "dc375e1527247fe1a97d8b7156678dfe7c1af2fc075c9a4db3690ecd2a148068" [[package]] name = "proc-macro2" -version = "1.0.56" +version = "1.0.58" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b63bdb0cd06f1f4dedf69b254734f9b45af66e4a031e42a7480257d9898b435" +checksum = "fa1fb82fc0c281dd9671101b66b771ebbe1eaf967b96ac8740dcba4b70005ca8" dependencies = [ "unicode-ident", ] @@ -3009,7 +2944,7 @@ dependencies = [ "byteorder", "hex", "lazy_static", - "rustix 0.36.12", + "rustix 0.36.14", ] [[package]] @@ -3093,7 +3028,7 @@ dependencies = [ "bstr", "bytes", "chrono", - "clap 4.2.2", + "clap 4.3.0", "consumption_metrics", "futures", "git-version", @@ -3131,7 +3066,7 @@ dependencies = [ "serde", "serde_json", "sha2", - "socket2 0.5.2", + "socket2 0.5.3", "sync_wrapper", "thiserror", "tls-listener", @@ -3154,9 +3089,9 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.26" +version = "1.0.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4424af4bf778aae2051a77b60283332f386554255d722233d09fbfc7e30da2fc" +checksum = "8f4f29d145265ec1c483c7c654450edde0bfe043d3938d6972630663356d9500" dependencies = [ "proc-macro2", ] @@ -3245,13 +3180,13 @@ dependencies = [ [[package]] name = "regex" -version = "1.7.3" +version = "1.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b1f693b24f6ac912f4893ef08244d70b6067480d2f1a46e950c9691e6749d1d" +checksum = "d1a59b5d8e97dee33696bf13c5ba8ab85341c002922fba050069326b9c498974" dependencies = [ "aho-corasick", "memchr", - "regex-syntax", + "regex-syntax 0.7.2", ] [[package]] @@ -3260,7 +3195,7 @@ version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6c230d73fb8d8c1b9c0b3135c5142a8acee3a0558fb8db5cf1cb65f8d7862132" dependencies = [ - "regex-syntax", + "regex-syntax 0.6.29", ] [[package]] @@ -3269,6 +3204,12 @@ version = "0.6.29" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1" +[[package]] +name = "regex-syntax" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "436b050e76ed2903236f032a59761c1eb99e1b0aead2c257922771dab1fc8c78" + [[package]] name = "remote_storage" version = "0.1.0" @@ -3298,11 +3239,11 @@ dependencies = [ [[package]] name = "reqwest" -version = "0.11.16" +version = "0.11.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "27b71749df584b7f4cac2c426c127a7c785a5106cc98f7a8feb044115f0fa254" +checksum = "cde824a14b7c14f85caff81225f411faacc04a2013f41670f41443742b1c1c55" dependencies = [ - "base64 0.21.0", + "base64 0.21.1", "bytes", "encoding_rs", "futures-core", @@ -3311,7 +3252,7 @@ dependencies = [ "http", "http-body", "hyper", - "hyper-rustls", + "hyper-rustls 0.24.0", "ipnet", "js-sys", "log", @@ -3320,13 +3261,13 @@ dependencies = [ "once_cell", "percent-encoding", "pin-project-lite", - "rustls 0.20.8", + "rustls 0.21.1", "rustls-pemfile", "serde", "serde_json", "serde_urlencoded", "tokio", - "tokio-rustls 0.23.4", + "tokio-rustls 0.24.0", "tower-service", "url", "wasm-bindgen", @@ -3338,9 +3279,9 @@ dependencies = [ [[package]] name = "reqwest-middleware" -version = "0.2.1" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "99c50db2c7ccd815f976473dd7d0bde296f8c3b77c383acf4fc021cdcf10852b" +checksum = "4531c89d50effe1fac90d095c8b133c20c5c714204feee0bfc3fd158e784209d" dependencies = [ "anyhow", "async-trait", @@ -3353,12 +3294,14 @@ dependencies = [ [[package]] name = "reqwest-tracing" -version = "0.4.1" +version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a71d77945a1c5ae9604f0504901e77a1e2e71f2932b1cb8103078179ca62ff8" +checksum = "783e8130d2427ddd7897dd3f814d4a3aea31b05deb42a4fdf8c18258fe5aefd1" dependencies = [ + "anyhow", "async-trait", "getrandom", + "matchit", "opentelemetry", "reqwest", "reqwest-middleware", @@ -3432,9 +3375,9 @@ dependencies = [ [[package]] name = "rustc-demangle" -version = "0.1.22" +version = "0.1.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d4a36c42d1873f9a77c53bde094f9664d9891bc604a45b4798fd2c389ed12e5b" +checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76" [[package]] name = "rustc-hash" @@ -3462,9 +3405,9 @@ dependencies = [ [[package]] name = "rustix" -version = "0.36.12" +version = "0.36.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0af200a3324fa5bcd922e84e9b55a298ea9f431a489f01961acdebc6e908f25" +checksum = "14e4d67015953998ad0eb82887a0eb0129e18a7e2f3b7b0f6c422fddcd503d62" dependencies = [ "bitflags", "errno", @@ -3476,15 +3419,15 @@ dependencies = [ [[package]] name = "rustix" -version = "0.37.11" +version = "0.37.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85597d61f83914ddeba6a47b3b8ffe7365107221c2e557ed94426489fefb5f77" +checksum = "acf8729d8542766f1b2cf77eb034d52f40d375bb8b615d0b147089946e16613d" dependencies = [ "bitflags", "errno", "io-lifetimes", "libc", - "linux-raw-sys 0.3.1", + "linux-raw-sys 0.3.8", "windows-sys 0.48.0", ] @@ -3502,9 +3445,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.21.0" +version = "0.21.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07180898a28ed6a7f7ba2311594308f595e3dd2e3c3812fa0a80a47b45f17e5d" +checksum = "c911ba11bc8433e811ce56fde130ccf32f5127cab0e0194e9c68c5a5b671791e" dependencies = [ "log", "ring", @@ -3530,7 +3473,7 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d194b56d58803a43635bdc398cd17e383d6f71f9182b9a192c127ca42494a59b" dependencies = [ - "base64 0.21.0", + "base64 0.21.1", ] [[package]] @@ -3565,7 +3508,7 @@ dependencies = [ "byteorder", "bytes", "chrono", - "clap 4.2.2", + "clap 4.3.0", "const_format", "crc32c", "fs2", @@ -3639,12 +3582,6 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" -[[package]] -name = "scratch" -version = "1.0.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1792db035ce95be60c3f8853017b3999209281c24e2ba5bc8e59bf97a0c590c1" - [[package]] name = "sct" version = "0.7.0" @@ -3657,9 +3594,9 @@ dependencies = [ [[package]] name = "security-framework" -version = "2.8.2" +version = "2.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a332be01508d814fed64bf28f798a146d73792121129962fdf335bb3c49a4254" +checksum = "1fc758eb7bffce5b308734e9b0c1468893cae9ff70ebf13e7090be8dcbcc83a8" dependencies = [ "bitflags", "core-foundation", @@ -3670,9 +3607,9 @@ dependencies = [ [[package]] name = "security-framework-sys" -version = "2.8.0" +version = "2.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31c9bb296072e961fcbd8853511dd39c2d8be2deb1e17c6860b1d30732b323b4" +checksum = "f51d0c0d83bec45f16480d0ce0058397a69e48fcdc52d1dc8855fb68acbd31a7" dependencies = [ "core-foundation-sys", "libc", @@ -3770,22 +3707,22 @@ dependencies = [ [[package]] name = "serde" -version = "1.0.160" +version = "1.0.163" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb2f3770c8bce3bcda7e149193a069a0f4365bda1fa5cd88e03bca26afc1216c" +checksum = "2113ab51b87a539ae008b5c6c02dc020ffa39afd2d83cffcb3f4eb2722cebec2" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.160" +version = "1.0.163" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "291a097c63d8497e00160b166a967a4a79c64f3facdd01cbd7502231688d77df" +checksum = "8c805777e3930c8883389c602315a24224bcc738b63905ef87cd1420353ea93e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.15", + "syn 2.0.16", ] [[package]] @@ -3801,9 +3738,9 @@ dependencies = [ [[package]] name = "serde_spanned" -version = "0.6.1" +version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0efd8caf556a6cebd3b285caf480045fcc1ac04f6bd786b09a6f11af30c4fcf4" +checksum = "93107647184f6027e3b7dcb2e11034cf95ffa1e3a682c67951963ac69c1c007d" dependencies = [ "serde", ] @@ -3822,9 +3759,9 @@ dependencies = [ [[package]] name = "serde_with" -version = "2.3.2" +version = "2.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "331bb8c3bf9b92457ab7abecf07078c13f7d270ba490103e84e8b014490cd0b0" +checksum = "07ff71d2c147a7b57362cead5e22f772cd52f6ab31cfcd9edcd7f6aeb2a0afbe" dependencies = [ "base64 0.13.1", "chrono", @@ -3838,14 +3775,14 @@ dependencies = [ [[package]] name = "serde_with_macros" -version = "2.3.2" +version = "2.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "859011bddcc11f289f07f467cc1fe01c7a941daa4d8f6c40d4d1c92eb6d9319c" +checksum = "881b6f881b17d13214e5d494c939ebab463d01264ce1811e9d4ac3a882e7695f" dependencies = [ "darling", "proc-macro2", "quote", - "syn 1.0.109", + "syn 2.0.16", ] [[package]] @@ -3959,9 +3896,9 @@ dependencies = [ [[package]] name = "socket2" -version = "0.5.2" +version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d283f86695ae989d1e18440a943880967156325ba025f05049946bff47bcc2b" +checksum = "2538b18701741680e0322a2302176d3253a35388e2e62f172f64f4f16605f877" dependencies = [ "libc", "windows-sys 0.48.0", @@ -4001,7 +3938,7 @@ dependencies = [ "anyhow", "async-stream", "bytes", - "clap 4.2.2", + "clap 4.3.0", "const_format", "futures", "futures-core", @@ -4015,8 +3952,8 @@ dependencies = [ "prost", "tokio", "tokio-stream", - "tonic 0.9.1", - "tonic-build 0.9.1", + "tonic 0.9.2", + "tonic-build 0.9.2", "tracing", "utils", "workspace_hack", @@ -4059,9 +3996,9 @@ dependencies = [ [[package]] name = "subtle" -version = "2.4.1" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6bdef32e8150c2a081110b42772ffe7d7c9032b606bc226c8260fd97e0976601" +checksum = "81cdd64d312baedb58e21336b31bc043b77e01cc99033ce76ef539f78e965ebc" [[package]] name = "svg_fmt" @@ -4082,9 +4019,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.15" +version = "2.0.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a34fcf3e8b60f57e6a14301a2e916d323af98b0ea63c599441eec8558660c822" +checksum = "a6f671d4b5ffdb8eadec19c0ae67fe2639df8684bd7bc4b83d986b8db549cf01" dependencies = [ "proc-macro2", "quote", @@ -4138,7 +4075,7 @@ dependencies = [ "cfg-if", "fastrand", "redox_syscall 0.3.5", - "rustix 0.37.11", + "rustix 0.37.19", "windows-sys 0.45.0", ] @@ -4205,7 +4142,7 @@ checksum = "f9456a42c5b0d803c8cd86e73dd7cc9edd429499f37a3550d286d5e86720569f" dependencies = [ "proc-macro2", "quote", - "syn 2.0.15", + "syn 2.0.16", ] [[package]] @@ -4220,9 +4157,9 @@ dependencies = [ [[package]] name = "time" -version = "0.3.20" +version = "0.3.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd0cbfecb4d19b5ea75bb31ad904eb5b9fa13f21079c3b92017ebdf4999a5890" +checksum = "8f3403384eaacbca9923fa06940178ac13e4edb725486d70e8e15881d0c836cc" dependencies = [ "itoa", "serde", @@ -4232,15 +4169,15 @@ dependencies = [ [[package]] name = "time-core" -version = "0.1.0" +version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e153e1f1acaef8acc537e68b44906d2db6436e2b35ac2c6b42640fff91f00fd" +checksum = "7300fbefb4dadc1af235a9cef3737cea692a9d97e1b9cbcd4ebdae6f8868e6fb" [[package]] name = "time-macros" -version = "0.2.8" +version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd80a657e71da814b8e5d60d3374fc6d35045062245d80224748ae522dd76f36" +checksum = "372950940a5f07bf38dbe211d7283c9e6d7327df53794992d293e534c733d09b" dependencies = [ "time-core", ] @@ -4320,7 +4257,7 @@ checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.15", + "syn 2.0.16", ] [[package]] @@ -4387,15 +4324,15 @@ version = "0.24.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e0d409377ff5b1e3ca6437aa86c1eb7d40c134bfec254e44c830defa92669db5" dependencies = [ - "rustls 0.21.0", + "rustls 0.21.1", "tokio", ] [[package]] name = "tokio-stream" -version = "0.1.12" +version = "0.1.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fb52b74f05dbf495a8fba459fdc331812b96aa086d9eb78101fa0d4569c3313" +checksum = "397c988d37662c7dda6d2208364a706264bf3d6138b11d436cbac0ad38832842" dependencies = [ "futures-core", "pin-project-lite", @@ -4430,9 +4367,9 @@ dependencies = [ [[package]] name = "tokio-util" -version = "0.7.7" +version = "0.7.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5427d89453009325de0d8f342c9490009f76e999cb7672d77e46267448f7e6b2" +checksum = "806fe8c2c87eccc8b3267cbae29ed3ab2d0bd37fca70ab622e46aaa9375ddb7d" dependencies = [ "bytes", "futures-core", @@ -4444,9 +4381,9 @@ dependencies = [ [[package]] name = "toml" -version = "0.7.3" +version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b403acf6f2bb0859c93c7f0d967cb4a75a7ac552100f9322faf64dc047669b21" +checksum = "d6135d499e69981f9ff0ef2167955a5333c35e36f6937d382974566b3d5b94ec" dependencies = [ "serde", "serde_spanned", @@ -4456,18 +4393,18 @@ dependencies = [ [[package]] name = "toml_datetime" -version = "0.6.1" +version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ab8ed2edee10b50132aed5f331333428b011c99402b5a534154ed15746f9622" +checksum = "5a76a9312f5ba4c2dec6b9161fdf25d87ad8a09256ccea5a556fef03c706a10f" dependencies = [ "serde", ] [[package]] name = "toml_edit" -version = "0.19.8" +version = "0.19.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "239410c8609e8125456927e6707163a3b1fdb40561e4b803bc041f466ccfdc13" +checksum = "2380d56e8670370eee6566b0bfd4265f65b3f432e8c6d85623f728d4fa31f739" dependencies = [ "indexmap", "serde", @@ -4510,14 +4447,14 @@ dependencies = [ [[package]] name = "tonic" -version = "0.9.1" +version = "0.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38bd8e87955eb13c1986671838177d6792cdc52af9bffced0d2c8a9a7f741ab3" +checksum = "3082666a3a6433f7f511c7192923fa1fe07c69332d3c6a2e6bb040b569199d5a" dependencies = [ "async-stream", "async-trait", "axum", - "base64 0.21.0", + "base64 0.21.1", "bytes", "futures-core", "futures-util", @@ -4555,9 +4492,9 @@ dependencies = [ [[package]] name = "tonic-build" -version = "0.9.1" +version = "0.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f60a933bbea70c95d633c04c951197ddf084958abaa2ed502a3743bdd8d8dd7" +checksum = "a6fdaae4c2c638bb70fe42803a26fbd6fc6ac8c72f5c59f67ecc2a2dcabf4b07" dependencies = [ "prettyplease 0.1.25", "proc-macro2", @@ -4603,7 +4540,7 @@ name = "trace" version = "0.1.0" dependencies = [ "anyhow", - "clap 4.2.2", + "clap 4.3.0", "pageserver_api", "utils", "workspace_hack", @@ -4624,20 +4561,20 @@ dependencies = [ [[package]] name = "tracing-attributes" -version = "0.1.23" +version = "0.1.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4017f8f45139870ca7e672686113917c71c7a6e02d4924eda67186083c03081a" +checksum = "0f57e3ca2a01450b1a921183a9c9cbfda207fd822cef4ccb00a65402cbba7a74" dependencies = [ "proc-macro2", "quote", - "syn 1.0.109", + "syn 2.0.16", ] [[package]] name = "tracing-core" -version = "0.1.30" +version = "0.1.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24eb03ba0eab1fd845050058ce5e616558e8f8d8fca633e6b163fe25c797213a" +checksum = "0955b8137a1df6f1a2e9a37d8a6656291ff0297c1a97c24e0d8425fe2312f79a" dependencies = [ "once_cell", "valuable", @@ -4700,9 +4637,9 @@ dependencies = [ [[package]] name = "tracing-subscriber" -version = "0.3.16" +version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6176eae26dd70d0c919749377897b54a9276bd7061339665dd68777926b5a70" +checksum = "30a651bc37f915e81f087d86e62a18eec5f79550c7faff886f7090b4ea757c77" dependencies = [ "matchers", "nu-ansi-term", @@ -4792,9 +4729,9 @@ checksum = "92888ba5573ff080736b3648696b70cafad7d250551175acbaa4e0385b3e1460" [[package]] name = "unicode-ident" -version = "1.0.8" +version = "1.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5464a87b239f13a63a501f2701565754bae92d243d4bb7eb12f6d57d2269bf4" +checksum = "b15811caf2415fb889178633e7724bad2509101cde276048e013b9def5e51fa0" [[package]] name = "unicode-normalization" @@ -4914,9 +4851,9 @@ dependencies = [ [[package]] name = "uuid" -version = "1.3.1" +version = "1.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b55a3fef2a1e3b3a00ce878640918820d3c51081576ac657d23af9fc7928fdb" +checksum = "345444e32442451b267fc254ae85a209c64be56d2890e601a0c37ff0c3c5ecd2" dependencies = [ "getrandom", "serde", @@ -4951,7 +4888,7 @@ name = "wal_craft" version = "0.1.0" dependencies = [ "anyhow", - "clap 4.2.2", + "clap 4.3.0", "env_logger", "log", "once_cell", @@ -4989,9 +4926,9 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "wasm-bindgen" -version = "0.2.84" +version = "0.2.86" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31f8dcbc21f30d9b8f2ea926ecb58f6b91192c17e9d33594b3df58b2007ca53b" +checksum = "5bba0e8cb82ba49ff4e229459ff22a191bbe9a1cb3a341610c9c33efc27ddf73" dependencies = [ "cfg-if", "wasm-bindgen-macro", @@ -4999,24 +4936,24 @@ dependencies = [ [[package]] name = "wasm-bindgen-backend" -version = "0.2.84" +version = "0.2.86" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95ce90fd5bcc06af55a641a86428ee4229e44e07033963a2290a8e241607ccb9" +checksum = "19b04bc93f9d6bdee709f6bd2118f57dd6679cf1176a1af464fca3ab0d66d8fb" dependencies = [ "bumpalo", "log", "once_cell", "proc-macro2", "quote", - "syn 1.0.109", + "syn 2.0.16", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-futures" -version = "0.4.34" +version = "0.4.36" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f219e0d211ba40266969f6dbdd90636da12f75bee4fc9d6c23d1260dadb51454" +checksum = "2d1985d03709c53167ce907ff394f5316aa22cb4e12761295c5dc57dacb6297e" dependencies = [ "cfg-if", "js-sys", @@ -5026,9 +4963,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.84" +version = "0.2.86" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c21f77c0bedc37fd5dc21f897894a5ca01e7bb159884559461862ae90c0b4c5" +checksum = "14d6b024f1a526bb0234f52840389927257beb670610081360e5a03c5df9c258" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -5036,28 +4973,28 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.84" +version = "0.2.86" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2aff81306fcac3c7515ad4e177f521b5c9a15f2b08f4e32d823066102f35a5f6" +checksum = "e128beba882dd1eb6200e1dc92ae6c5dbaa4311aa7bb211ca035779e5efc39f8" dependencies = [ "proc-macro2", "quote", - "syn 1.0.109", + "syn 2.0.16", "wasm-bindgen-backend", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.84" +version = "0.2.86" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0046fef7e28c3804e5e38bfa31ea2a0f73905319b677e57ebe37e49358989b5d" +checksum = "ed9d5b4305409d1fc9482fee2d7f9bcbf24b3972bf59817ef757e23982242a93" [[package]] name = "web-sys" -version = "0.3.61" +version = "0.3.63" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e33b99f4b23ba3eec1a53ac264e35a755f00e966e0065077d6027c0f575b0b97" +checksum = "3bdd9ef4e984da1187bf8110c5cf5b845fbc87a23602cdf912386a76fcd3a7c2" dependencies = [ "js-sys", "wasm-bindgen", @@ -5291,9 +5228,9 @@ checksum = "1a515f5799fe4961cb532f983ce2b23082366b898e52ffbce459c86f67c8378a" [[package]] name = "winnow" -version = "0.4.1" +version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae8970b36c66498d8ff1d66685dc86b91b29db0c7739899012f63a63814b4b28" +checksum = "61de7bac303dc551fe038e2b3cef0f571087a47571ea6e79a87692ac99b99699" dependencies = [ "memchr", ] @@ -5314,7 +5251,7 @@ dependencies = [ "anyhow", "bytes", "chrono", - "clap 4.2.2", + "clap 4.3.0", "clap_builder", "crossbeam-utils", "either", @@ -5325,7 +5262,6 @@ dependencies = [ "futures-executor", "futures-sink", "futures-util", - "hashbrown 0.12.3", "itertools", "libc", "log", @@ -5337,7 +5273,7 @@ dependencies = [ "prost", "rand", "regex", - "regex-syntax", + "regex-syntax 0.7.2", "reqwest", "ring", "rustls 0.20.8", @@ -5346,7 +5282,7 @@ dependencies = [ "serde_json", "socket2 0.4.9", "syn 1.0.109", - "syn 2.0.15", + "syn 2.0.16", "tokio", "tokio-rustls 0.23.4", "tokio-util", diff --git a/Cargo.toml b/Cargo.toml index 19d1783851..1cb8d65948 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -23,7 +23,7 @@ async-stream = "0.3" async-trait = "0.1" atty = "0.2.14" aws-config = { version = "0.55", default-features = false, features=["rustls"] } -aws-sdk-s3 = "0.25" +aws-sdk-s3 = "0.27" aws-smithy-http = "0.55" aws-credential-types = "0.55" aws-types = "0.55" diff --git a/Dockerfile.compute-node b/Dockerfile.compute-node index 3a3dee8a8a..de8a904c02 100644 --- a/Dockerfile.compute-node +++ b/Dockerfile.compute-node @@ -632,6 +632,7 @@ RUN apt update && \ libxml2 \ libxslt1.1 \ libzstd1 \ + libcurl4-openssl-dev \ procps && \ rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* && \ localedef -i en_US -c -f UTF-8 -A /usr/share/locale/locale.alias en_US.UTF-8 diff --git a/compute_tools/src/compute.rs b/compute_tools/src/compute.rs index da5ad00da6..a7746629a8 100644 --- a/compute_tools/src/compute.rs +++ b/compute_tools/src/compute.rs @@ -362,6 +362,8 @@ impl ComputeNode { }; // Proceed with post-startup configuration. Note, that order of operations is important. + // Disable DDL forwarding because control plane already knows about these roles/databases. + client.simple_query("SET neon.forward_ddl = false")?; let spec = &compute_state.pspec.as_ref().expect("spec must be set").spec; handle_roles(spec, &mut client)?; handle_databases(spec, &mut client)?; @@ -403,7 +405,9 @@ impl ComputeNode { self.pg_reload_conf(&mut client)?; // Proceed with post-startup configuration. Note, that order of operations is important. + // Disable DDL forwarding because control plane already knows about these roles/databases. if spec.mode == ComputeMode::Primary { + client.simple_query("SET neon.forward_ddl = false")?; handle_roles(&spec, &mut client)?; handle_databases(&spec, &mut client)?; handle_role_deletions(&spec, self.connstr.as_str(), &mut client)?; diff --git a/compute_tools/src/pg_helpers.rs b/compute_tools/src/pg_helpers.rs index 40dbea6907..ed00485d5a 100644 --- a/compute_tools/src/pg_helpers.rs +++ b/compute_tools/src/pg_helpers.rs @@ -121,9 +121,8 @@ impl RoleExt for Role { /// string of arguments. fn to_pg_options(&self) -> String { // XXX: consider putting LOGIN as a default option somewhere higher, e.g. in control-plane. - // For now, we do not use generic `options` for roles. Once used, add - // `self.options.as_pg_options()` somewhere here. - let mut params: String = "LOGIN".to_string(); + let mut params: String = self.options.as_pg_options(); + params.push_str(" LOGIN"); if let Some(pass) = &self.encrypted_password { // Some time ago we supported only md5 and treated all encrypted_password as md5. diff --git a/compute_tools/src/spec.rs b/compute_tools/src/spec.rs index bf3c407202..a2a19ae0da 100644 --- a/compute_tools/src/spec.rs +++ b/compute_tools/src/spec.rs @@ -62,7 +62,7 @@ fn do_control_plane_request( } } -/// Request spec from the control-plane by compute_id. If `NEON_CONSOLE_JWT` +/// Request spec from the control-plane by compute_id. If `NEON_CONTROL_PLANE_TOKEN` /// env variable is set, it will be used for authorization. pub fn get_spec_from_control_plane( base_uri: &str, diff --git a/compute_tools/tests/pg_helpers_tests.rs b/compute_tools/tests/pg_helpers_tests.rs index a63ee038c7..265556d3b9 100644 --- a/compute_tools/tests/pg_helpers_tests.rs +++ b/compute_tools/tests/pg_helpers_tests.rs @@ -16,7 +16,7 @@ mod pg_helpers_tests { ); assert_eq!( spec.cluster.roles.first().unwrap().to_pg_options(), - "LOGIN PASSWORD 'md56b1d16b78004bbd51fa06af9eda75972'" + " LOGIN PASSWORD 'md56b1d16b78004bbd51fa06af9eda75972'" ); } diff --git a/control_plane/src/bin/neon_local.rs b/control_plane/src/bin/neon_local.rs index 30880565ab..39551642c0 100644 --- a/control_plane/src/bin/neon_local.rs +++ b/control_plane/src/bin/neon_local.rs @@ -41,7 +41,7 @@ const DEFAULT_PAGESERVER_ID: NodeId = NodeId(1); const DEFAULT_BRANCH_NAME: &str = "main"; project_git_version!(GIT_VERSION); -const DEFAULT_PG_VERSION: &str = "14"; +const DEFAULT_PG_VERSION: &str = "15"; fn default_conf() -> String { format!( diff --git a/control_plane/src/local_env.rs b/control_plane/src/local_env.rs index 2b1eec7c4b..9286944412 100644 --- a/control_plane/src/local_env.rs +++ b/control_plane/src/local_env.rs @@ -24,7 +24,7 @@ use utils::{ use crate::safekeeper::SafekeeperNode; -pub const DEFAULT_PG_VERSION: u32 = 14; +pub const DEFAULT_PG_VERSION: u32 = 15; // // This data structures represents neon_local CLI config diff --git a/pageserver/src/http/routes.rs b/pageserver/src/http/routes.rs index 59105f122c..b3ff575aac 100644 --- a/pageserver/src/http/routes.rs +++ b/pageserver/src/http/routes.rs @@ -25,7 +25,9 @@ use crate::metrics::{StorageTimeOperation, STORAGE_TIME_GLOBAL}; use crate::pgdatadir_mapping::LsnForTimestamp; use crate::task_mgr::TaskKind; use crate::tenant::config::TenantConfOpt; -use crate::tenant::mgr::{TenantMapInsertError, TenantStateError}; +use crate::tenant::mgr::{ + GetTenantError, SetNewTenantConfigError, TenantMapInsertError, TenantStateError, +}; use crate::tenant::size::ModelInputs; use crate::tenant::storage_layer::LayerAccessStatsReset; use crate::tenant::{LogicalSizeCalculationCause, PageReconstructError, Timeline}; @@ -144,6 +146,36 @@ impl From for ApiError { } } +impl From for ApiError { + fn from(tse: GetTenantError) -> ApiError { + match tse { + GetTenantError::NotFound(tid) => ApiError::NotFound(anyhow!("tenant {}", tid)), + e @ GetTenantError::NotActive(_) => { + // Why is this not `ApiError::NotFound`? + // Because we must be careful to never return 404 for a tenant if it does + // in fact exist locally. If we did, the caller could draw the conclusion + // that it can attach the tenant to another PS and we'd be in split-brain. + // + // (We can produce this variant only in `mgr::get_tenant(..., active=true)` calls). + ApiError::InternalServerError(anyhow::Error::new(e)) + } + } + } +} + +impl From for ApiError { + fn from(e: SetNewTenantConfigError) -> ApiError { + match e { + SetNewTenantConfigError::GetTenant(tid) => { + ApiError::NotFound(anyhow!("tenant {}", tid)) + } + e @ SetNewTenantConfigError::Persist(_) => { + ApiError::InternalServerError(anyhow::Error::new(e)) + } + } + } +} + impl From for ApiError { fn from(value: crate::tenant::DeleteTimelineError) -> Self { use crate::tenant::DeleteTimelineError::*; @@ -163,7 +195,7 @@ impl From for ApiError { match value { // Report Precondition failed so client can distinguish between // "tenant is missing" case from "timeline is missing" - Tenant(TenantStateError::NotFound(..)) => { + Tenant(GetTenantError::NotFound(..)) => { ApiError::PreconditionFailed("Requested tenant is missing") } Tenant(t) => ApiError::from(t), @@ -525,7 +557,7 @@ async fn tenant_status(request: Request) -> Result, ApiErro } let state = tenant.current_state(); - Ok(TenantInfo { + Result::<_, ApiError>::Ok(TenantInfo { id: tenant_id, state: state.clone(), current_physical_size: Some(current_physical_size), @@ -533,8 +565,7 @@ async fn tenant_status(request: Request) -> Result, ApiErro }) } .instrument(info_span!("tenant_status_handler", tenant = %tenant_id)) - .await - .map_err(ApiError::InternalServerError)?; + .await?; json_response(StatusCode::OK, tenant_info) } diff --git a/pageserver/src/lib.rs b/pageserver/src/lib.rs index 4349f0e2ea..36578ee4e0 100644 --- a/pageserver/src/lib.rs +++ b/pageserver/src/lib.rs @@ -35,7 +35,7 @@ use tracing::info; /// backwards-compatible changes to the metadata format. pub const STORAGE_FORMAT_VERSION: u16 = 3; -pub const DEFAULT_PG_VERSION: u32 = 14; +pub const DEFAULT_PG_VERSION: u32 = 15; // Magic constants used to identify different kinds of files pub const IMAGE_FILE_MAGIC: u16 = 0x5A60; diff --git a/pageserver/src/page_service.rs b/pageserver/src/page_service.rs index 886e0b2c35..9e9285a009 100644 --- a/pageserver/src/page_service.rs +++ b/pageserver/src/page_service.rs @@ -50,7 +50,9 @@ use crate::import_datadir::import_wal_from_tar; use crate::metrics::{LIVE_CONNECTIONS_COUNT, SMGR_QUERY_TIME}; use crate::task_mgr; use crate::task_mgr::TaskKind; +use crate::tenant; use crate::tenant::mgr; +use crate::tenant::mgr::GetTenantError; use crate::tenant::{Tenant, Timeline}; use crate::trace::Tracer; @@ -1148,7 +1150,9 @@ enum GetActiveTenantError { wait_time: Duration, }, #[error(transparent)] - Other(#[from] anyhow::Error), + NotFound(GetTenantError), + #[error(transparent)] + WaitTenantActive(tenant::WaitToBecomeActiveError), } impl From for QueryError { @@ -1157,7 +1161,8 @@ impl From for QueryError { GetActiveTenantError::WaitForActiveTimeout { .. } => QueryError::Disconnected( ConnectionError::Io(io::Error::new(io::ErrorKind::TimedOut, e.to_string())), ), - GetActiveTenantError::Other(e) => QueryError::Other(e), + GetActiveTenantError::WaitTenantActive(e) => QueryError::Other(anyhow::Error::new(e)), + GetActiveTenantError::NotFound(e) => QueryError::Other(anyhow::Error::new(e)), } } } @@ -1173,13 +1178,16 @@ async fn get_active_tenant_with_timeout( ) -> Result, GetActiveTenantError> { let tenant = match mgr::get_tenant(tenant_id, false).await { Ok(tenant) => tenant, - Err(e) => return Err(GetActiveTenantError::Other(e.into())), + Err(e @ GetTenantError::NotFound(_)) => return Err(GetActiveTenantError::NotFound(e)), + Err(GetTenantError::NotActive(_)) => { + unreachable!("we're calling get_tenant with active=false") + } }; let wait_time = Duration::from_secs(30); match tokio::time::timeout(wait_time, tenant.wait_to_become_active()).await { Ok(Ok(())) => Ok(tenant), // no .context(), the error message is good enough and some tests depend on it - Ok(Err(wait_error)) => Err(GetActiveTenantError::Other(wait_error)), + Ok(Err(e)) => Err(GetActiveTenantError::WaitTenantActive(e)), Err(_) => { let latest_state = tenant.current_state(); if latest_state == TenantState::Active { @@ -1194,13 +1202,34 @@ async fn get_active_tenant_with_timeout( } } +#[derive(Debug, thiserror::Error)] +enum GetActiveTimelineError { + #[error(transparent)] + Tenant(GetActiveTenantError), + #[error(transparent)] + Timeline(anyhow::Error), +} + +impl From for QueryError { + fn from(e: GetActiveTimelineError) -> Self { + match e { + GetActiveTimelineError::Tenant(e) => e.into(), + GetActiveTimelineError::Timeline(e) => QueryError::Other(e), + } + } +} + /// Shorthand for getting a reference to a Timeline of an Active tenant. async fn get_active_tenant_timeline( tenant_id: TenantId, timeline_id: TimelineId, ctx: &RequestContext, -) -> Result, GetActiveTenantError> { - let tenant = get_active_tenant_with_timeout(tenant_id, ctx).await?; - let timeline = tenant.get_timeline(timeline_id, true)?; +) -> Result, GetActiveTimelineError> { + let tenant = get_active_tenant_with_timeout(tenant_id, ctx) + .await + .map_err(GetActiveTimelineError::Tenant)?; + let timeline = tenant + .get_timeline(timeline_id, true) + .map_err(GetActiveTimelineError::Timeline)?; Ok(timeline) } diff --git a/pageserver/src/tenant.rs b/pageserver/src/tenant.rs index 304651b6ac..a3a1cb3ca5 100644 --- a/pageserver/src/tenant.rs +++ b/pageserver/src/tenant.rs @@ -452,6 +452,34 @@ struct RemoteStartupData { remote_metadata: TimelineMetadata, } +#[derive(Debug, thiserror::Error)] +pub(crate) enum WaitToBecomeActiveError { + WillNotBecomeActive { + tenant_id: TenantId, + state: TenantState, + }, + TenantDropped { + tenant_id: TenantId, + }, +} + +impl std::fmt::Display for WaitToBecomeActiveError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + WaitToBecomeActiveError::WillNotBecomeActive { tenant_id, state } => { + write!( + f, + "Tenant {} will not become active. Current state: {:?}", + tenant_id, state + ) + } + WaitToBecomeActiveError::TenantDropped { tenant_id } => { + write!(f, "Tenant {tenant_id} will not become active (dropped)") + } + } + } +} + impl Tenant { /// Yet another helper for timeline initialization. /// Contains the common part of `load_local_timeline` and `load_remote_timeline`. @@ -1781,25 +1809,30 @@ impl Tenant { self.state.subscribe() } - pub async fn wait_to_become_active(&self) -> anyhow::Result<()> { + pub(crate) async fn wait_to_become_active(&self) -> Result<(), WaitToBecomeActiveError> { let mut receiver = self.state.subscribe(); loop { let current_state = receiver.borrow_and_update().clone(); match current_state { TenantState::Loading | TenantState::Attaching | TenantState::Activating => { // in these states, there's a chance that we can reach ::Active - receiver.changed().await?; + receiver.changed().await.map_err( + |_e: tokio::sync::watch::error::RecvError| { + WaitToBecomeActiveError::TenantDropped { + tenant_id: self.tenant_id, + } + }, + )?; } TenantState::Active { .. } => { return Ok(()); } TenantState::Broken { .. } | TenantState::Stopping => { // There's no chance the tenant can transition back into ::Active - anyhow::bail!( - "Tenant {} will not become active. Current state: {:?}", - self.tenant_id, - ¤t_state, - ); + return Err(WaitToBecomeActiveError::WillNotBecomeActive { + tenant_id: self.tenant_id, + state: current_state, + }); } } } diff --git a/pageserver/src/tenant/mgr.rs b/pageserver/src/tenant/mgr.rs index 191ecf1008..d40468a5b3 100644 --- a/pageserver/src/tenant/mgr.rs +++ b/pageserver/src/tenant/mgr.rs @@ -302,11 +302,19 @@ pub async fn create_tenant( }).await } +#[derive(Debug, thiserror::Error)] +pub enum SetNewTenantConfigError { + #[error(transparent)] + GetTenant(#[from] GetTenantError), + #[error(transparent)] + Persist(anyhow::Error), +} + pub async fn set_new_tenant_config( conf: &'static PageServerConf, new_tenant_conf: TenantConfOpt, tenant_id: TenantId, -) -> Result<(), TenantStateError> { +) -> Result<(), SetNewTenantConfigError> { info!("configuring tenant {tenant_id}"); let tenant = get_tenant(tenant_id, true).await?; @@ -316,23 +324,32 @@ pub async fn set_new_tenant_config( &tenant_config_path, new_tenant_conf, false, - )?; + ) + .map_err(SetNewTenantConfigError::Persist)?; tenant.set_new_tenant_config(new_tenant_conf); Ok(()) } +#[derive(Debug, thiserror::Error)] +pub enum GetTenantError { + #[error("Tenant {0} not found")] + NotFound(TenantId), + #[error("Tenant {0} is not active")] + NotActive(TenantId), +} + /// Gets the tenant from the in-memory data, erroring if it's absent or is not fitting to the query. /// `active_only = true` allows to query only tenants that are ready for operations, erroring on other kinds of tenants. pub async fn get_tenant( tenant_id: TenantId, active_only: bool, -) -> Result, TenantStateError> { +) -> Result, GetTenantError> { let m = TENANTS.read().await; let tenant = m .get(&tenant_id) - .ok_or(TenantStateError::NotFound(tenant_id))?; + .ok_or(GetTenantError::NotFound(tenant_id))?; if active_only && !tenant.is_active() { - Err(TenantStateError::NotActive(tenant_id)) + Err(GetTenantError::NotActive(tenant_id)) } else { Ok(Arc::clone(tenant)) } @@ -341,7 +358,7 @@ pub async fn get_tenant( #[derive(Debug, thiserror::Error)] pub enum DeleteTimelineError { #[error("Tenant {0}")] - Tenant(#[from] TenantStateError), + Tenant(#[from] GetTenantError), #[error("Timeline {0}")] Timeline(#[from] crate::tenant::DeleteTimelineError), diff --git a/pgxn/neon/Makefile b/pgxn/neon/Makefile index ec377dbb1e..1948023472 100644 --- a/pgxn/neon/Makefile +++ b/pgxn/neon/Makefile @@ -11,10 +11,12 @@ OBJS = \ pagestore_smgr.o \ relsize_cache.o \ walproposer.o \ - walproposer_utils.o + walproposer_utils.o \ + control_plane_connector.o PG_CPPFLAGS = -I$(libpq_srcdir) SHLIB_LINK_INTERNAL = $(libpq) +SHLIB_LINK = -lcurl EXTENSION = neon DATA = neon--1.0.sql diff --git a/pgxn/neon/control_plane_connector.c b/pgxn/neon/control_plane_connector.c new file mode 100644 index 0000000000..82e4af4b4a --- /dev/null +++ b/pgxn/neon/control_plane_connector.c @@ -0,0 +1,830 @@ +/*------------------------------------------------------------------------- + * + * control_plane_connector.c + * Captures updates to roles/databases using ProcessUtility_hook and + * sends them to the control ProcessUtility_hook. The changes are sent + * via HTTP to the URL specified by the GUC neon.console_url when the + * transaction commits. Forwarding may be disabled temporarily by + * setting neon.forward_ddl to false. + * + * Currently, the transaction may abort AFTER + * changes have already been forwarded, and that case is not handled. + * Subtransactions are handled using a stack of hash tables, which + * accumulate changes. On subtransaction commit, the top of the stack + * is merged with the table below it. + * + * IDENTIFICATION + * contrib/neon/control_plane_connector.c + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" +#include "tcop/pquery.h" +#include "tcop/utility.h" +#include "access/xact.h" +#include "utils/hsearch.h" +#include "utils/memutils.h" +#include "commands/defrem.h" +#include "miscadmin.h" +#include "utils/acl.h" +#include "fmgr.h" +#include "utils/guc.h" +#include "port.h" +#include +#include "utils/jsonb.h" + +static ProcessUtility_hook_type PreviousProcessUtilityHook = NULL; + +/* GUCs */ +static char *ConsoleURL = NULL; +static bool ForwardDDL = true; + +/* Curl structures for sending the HTTP requests */ +static CURL * CurlHandle; +static struct curl_slist *ContentHeader = NULL; + +/* + * CURL docs say that this buffer must exist until we call curl_easy_cleanup + * (which we never do), so we make this a static + */ +static char CurlErrorBuf[CURL_ERROR_SIZE]; + +typedef enum +{ + Op_Set, /* An upsert: Either a creation or an alter */ + Op_Delete, +} OpType; + +typedef struct +{ + char name[NAMEDATALEN]; + Oid owner; + char old_name[NAMEDATALEN]; + OpType type; +} DbEntry; + +typedef struct +{ + char name[NAMEDATALEN]; + char old_name[NAMEDATALEN]; + const char *password; + OpType type; +} RoleEntry; + +/* + * We keep one of these for each subtransaction in a stack. When a subtransaction + * commits, we merge the top of the stack into the table below it. It is allocated in the + * subtransaction's context. + */ +typedef struct DdlHashTable +{ + struct DdlHashTable *prev_table; + HTAB *db_table; + HTAB *role_table; +} DdlHashTable; + +static DdlHashTable RootTable; +static DdlHashTable * CurrentDdlTable = &RootTable; + +static void +PushKeyValue(JsonbParseState **state, char *key, char *value) +{ + JsonbValue k, + v; + + k.type = jbvString; + k.val.string.len = strlen(key); + k.val.string.val = key; + v.type = jbvString; + v.val.string.len = strlen(value); + v.val.string.val = value; + pushJsonbValue(state, WJB_KEY, &k); + pushJsonbValue(state, WJB_VALUE, &v); +} + +static char * +ConstructDeltaMessage() +{ + JsonbParseState *state = NULL; + + pushJsonbValue(&state, WJB_BEGIN_OBJECT, NULL); + if (RootTable.db_table) + { + JsonbValue dbs; + + dbs.type = jbvString; + dbs.val.string.val = "dbs"; + dbs.val.string.len = strlen(dbs.val.string.val); + pushJsonbValue(&state, WJB_KEY, &dbs); + pushJsonbValue(&state, WJB_BEGIN_ARRAY, NULL); + + HASH_SEQ_STATUS status; + DbEntry *entry; + + hash_seq_init(&status, RootTable.db_table); + while ((entry = hash_seq_search(&status)) != NULL) + { + pushJsonbValue(&state, WJB_BEGIN_OBJECT, NULL); + PushKeyValue(&state, "op", entry->type == Op_Set ? "set" : "del"); + PushKeyValue(&state, "name", entry->name); + if (entry->owner != InvalidOid) + { + PushKeyValue(&state, "owner", GetUserNameFromId(entry->owner, false)); + } + if (entry->old_name[0] != '\0') + { + PushKeyValue(&state, "old_name", entry->old_name); + } + pushJsonbValue(&state, WJB_END_OBJECT, NULL); + } + pushJsonbValue(&state, WJB_END_ARRAY, NULL); + } + + if (RootTable.role_table) + { + JsonbValue roles; + + roles.type = jbvString; + roles.val.string.val = "roles"; + roles.val.string.len = strlen(roles.val.string.val); + pushJsonbValue(&state, WJB_KEY, &roles); + pushJsonbValue(&state, WJB_BEGIN_ARRAY, NULL); + + HASH_SEQ_STATUS status; + RoleEntry *entry; + + hash_seq_init(&status, RootTable.role_table); + while ((entry = hash_seq_search(&status)) != NULL) + { + pushJsonbValue(&state, WJB_BEGIN_OBJECT, NULL); + PushKeyValue(&state, "op", entry->type == Op_Set ? "set" : "del"); + PushKeyValue(&state, "name", entry->name); + if (entry->password) + { + PushKeyValue(&state, "password", (char *) entry->password); + } + if (entry->old_name[0] != '\0') + { + PushKeyValue(&state, "old_name", entry->old_name); + } + pushJsonbValue(&state, WJB_END_OBJECT, NULL); + } + pushJsonbValue(&state, WJB_END_ARRAY, NULL); + } + JsonbValue *result = pushJsonbValue(&state, WJB_END_OBJECT, NULL); + Jsonb *jsonb = JsonbValueToJsonb(result); + + return JsonbToCString(NULL, &jsonb->root, 0 /* estimated_len */ ); +} + +#define ERROR_SIZE 1024 + +typedef struct +{ + char str[ERROR_SIZE]; + size_t size; +} ErrorString; + +static size_t +ErrorWriteCallback(char *ptr, size_t size, size_t nmemb, void *userdata) +{ + /* Docs say size is always 1 */ + ErrorString *str = userdata; + + size_t to_write = nmemb; + + /* +1 for null terminator */ + if (str->size + nmemb + 1 >= ERROR_SIZE) + to_write = ERROR_SIZE - str->size - 1; + + /* Ignore everyrthing past the first ERROR_SIZE bytes */ + if (to_write == 0) + return nmemb; + memcpy(str->str + str->size, ptr, to_write); + str->size += to_write; + str->str[str->size] = '\0'; + return nmemb; +} + +static void +SendDeltasToControlPlane() +{ + if (!RootTable.db_table && !RootTable.role_table) + return; + if (!ConsoleURL) + { + elog(LOG, "ConsoleURL not set, skipping forwarding"); + return; + } + if (!ForwardDDL) + return; + + char *message = ConstructDeltaMessage(); + ErrorString str = {}; + + curl_easy_setopt(CurlHandle, CURLOPT_CUSTOMREQUEST, "PATCH"); + curl_easy_setopt(CurlHandle, CURLOPT_HTTPHEADER, ContentHeader); + curl_easy_setopt(CurlHandle, CURLOPT_POSTFIELDS, message); + curl_easy_setopt(CurlHandle, CURLOPT_URL, ConsoleURL); + curl_easy_setopt(CurlHandle, CURLOPT_ERRORBUFFER, CurlErrorBuf); + curl_easy_setopt(CurlHandle, CURLOPT_TIMEOUT, 3L /* seconds */ ); + curl_easy_setopt(CurlHandle, CURLOPT_WRITEDATA, &str); + curl_easy_setopt(CurlHandle, CURLOPT_WRITEFUNCTION, ErrorWriteCallback); + + const int num_retries = 5; + int curl_status; + + for (int i = 0; i < num_retries; i++) + { + if ((curl_status = curl_easy_perform(CurlHandle)) == 0) + break; + elog(LOG, "Curl request failed on attempt %d: %s", i, CurlErrorBuf); + pg_usleep(1000 * 1000); + } + if (curl_status != 0) + { + elog(ERROR, "Failed to perform curl request: %s", CurlErrorBuf); + } + else + { + long response_code; + + if (curl_easy_getinfo(CurlHandle, CURLINFO_RESPONSE_CODE, &response_code) != CURLE_UNKNOWN_OPTION) + { + bool error_exists = str.size != 0; + + if (response_code != 200) + { + if (error_exists) + { + elog(ERROR, + "Received HTTP code %ld from control plane: %s", + response_code, + str.str); + } + else + { + elog(ERROR, + "Received HTTP code %ld from control plane", + response_code); + } + } + } + } +} + +static void +InitDbTableIfNeeded() +{ + if (!CurrentDdlTable->db_table) + { + HASHCTL db_ctl = {}; + + db_ctl.keysize = NAMEDATALEN; + db_ctl.entrysize = sizeof(DbEntry); + db_ctl.hcxt = CurTransactionContext; + CurrentDdlTable->db_table = hash_create( + "Dbs Created", + 4, + &db_ctl, + HASH_ELEM | HASH_STRINGS | HASH_CONTEXT); + } +} + +static void +InitRoleTableIfNeeded() +{ + if (!CurrentDdlTable->role_table) + { + HASHCTL role_ctl = {}; + + role_ctl.keysize = NAMEDATALEN; + role_ctl.entrysize = sizeof(RoleEntry); + role_ctl.hcxt = CurTransactionContext; + CurrentDdlTable->role_table = hash_create( + "Roles Created", + 4, + &role_ctl, + HASH_ELEM | HASH_STRINGS | HASH_CONTEXT); + } +} + +static void +PushTable() +{ + DdlHashTable *new_table = MemoryContextAlloc(CurTransactionContext, sizeof(DdlHashTable)); + + new_table->prev_table = CurrentDdlTable; + new_table->role_table = NULL; + new_table->db_table = NULL; + CurrentDdlTable = new_table; +} + +static void +MergeTable() +{ + DdlHashTable *old_table = CurrentDdlTable; + + CurrentDdlTable = old_table->prev_table; + + if (old_table->db_table) + { + InitDbTableIfNeeded(); + DbEntry *entry; + HASH_SEQ_STATUS status; + + hash_seq_init(&status, old_table->db_table); + while ((entry = hash_seq_search(&status)) != NULL) + { + DbEntry *to_write = hash_search( + CurrentDdlTable->db_table, + entry->name, + HASH_ENTER, + NULL); + + to_write->type = entry->type; + if (entry->owner != InvalidOid) + to_write->owner = entry->owner; + strlcpy(to_write->old_name, entry->old_name, NAMEDATALEN); + if (entry->old_name[0] != '\0') + { + bool found_old = false; + DbEntry *old = hash_search( + CurrentDdlTable->db_table, + entry->old_name, + HASH_FIND, + &found_old); + + if (found_old) + { + if (old->old_name[0] != '\0') + strlcpy(to_write->old_name, old->old_name, NAMEDATALEN); + else + strlcpy(to_write->old_name, entry->old_name, NAMEDATALEN); + hash_search( + CurrentDdlTable->db_table, + entry->old_name, + HASH_REMOVE, + NULL); + } + } + } + hash_destroy(old_table->db_table); + } + + if (old_table->role_table) + { + InitRoleTableIfNeeded(); + RoleEntry *entry; + HASH_SEQ_STATUS status; + + hash_seq_init(&status, old_table->role_table); + while ((entry = hash_seq_search(&status)) != NULL) + { + RoleEntry *to_write = hash_search( + CurrentDdlTable->role_table, + entry->name, + HASH_ENTER, + NULL); + + to_write->type = entry->type; + if (entry->password) + to_write->password = entry->password; + strlcpy(to_write->old_name, entry->old_name, NAMEDATALEN); + if (entry->old_name[0] != '\0') + { + bool found_old = false; + RoleEntry *old = hash_search( + CurrentDdlTable->role_table, + entry->old_name, + HASH_FIND, + &found_old); + + if (found_old) + { + if (old->old_name[0] != '\0') + strlcpy(to_write->old_name, old->old_name, NAMEDATALEN); + else + strlcpy(to_write->old_name, entry->old_name, NAMEDATALEN); + hash_search(CurrentDdlTable->role_table, + entry->old_name, + HASH_REMOVE, + NULL); + } + } + } + hash_destroy(old_table->role_table); + } +} + +static void +PopTable() +{ + /* + * Current table gets freed because it is allocated in aborted + * subtransaction's memory context. + */ + CurrentDdlTable = CurrentDdlTable->prev_table; +} + +static void +NeonSubXactCallback( + SubXactEvent event, + SubTransactionId mySubid, + SubTransactionId parentSubid, + void *arg) +{ + switch (event) + { + case SUBXACT_EVENT_START_SUB: + return PushTable(); + case SUBXACT_EVENT_COMMIT_SUB: + return MergeTable(); + case SUBXACT_EVENT_ABORT_SUB: + return PopTable(); + default: + return; + } +} + +static void +NeonXactCallback(XactEvent event, void *arg) +{ + if (event == XACT_EVENT_PRE_COMMIT || event == XACT_EVENT_PARALLEL_PRE_COMMIT) + { + SendDeltasToControlPlane(); + } + RootTable.role_table = NULL; + RootTable.db_table = NULL; + Assert(CurrentDdlTable == &RootTable); +} + +static void +HandleCreateDb(CreatedbStmt *stmt) +{ + InitDbTableIfNeeded(); + DefElem *downer = NULL; + ListCell *option; + + foreach(option, stmt->options) + { + DefElem *defel = lfirst(option); + + if (strcmp(defel->defname, "owner") == 0) + downer = defel; + } + bool found = false; + DbEntry *entry = hash_search( + CurrentDdlTable->db_table, + stmt->dbname, + HASH_ENTER, + &found); + + if (!found) + memset(entry->old_name, 0, sizeof(entry->old_name)); + + entry->type = Op_Set; + if (downer && downer->arg) + entry->owner = get_role_oid(defGetString(downer), false); + else + entry->owner = GetUserId(); +} + +static void +HandleAlterOwner(AlterOwnerStmt *stmt) +{ + if (stmt->objectType != OBJECT_DATABASE) + return; + InitDbTableIfNeeded(); + const char *name = strVal(stmt->object); + bool found = false; + DbEntry *entry = hash_search( + CurrentDdlTable->db_table, + name, + HASH_ENTER, + &found); + + if (!found) + memset(entry->old_name, 0, sizeof(entry->old_name)); + + entry->owner = get_role_oid(get_rolespec_name(stmt->newowner), false); + entry->type = Op_Set; +} + +static void +HandleDbRename(RenameStmt *stmt) +{ + Assert(stmt->renameType == OBJECT_DATABASE); + InitDbTableIfNeeded(); + bool found = false; + DbEntry *entry = hash_search( + CurrentDdlTable->db_table, + stmt->subname, + HASH_FIND, + &found); + DbEntry *entry_for_new_name = hash_search( + CurrentDdlTable->db_table, + stmt->newname, + HASH_ENTER, + NULL); + + entry_for_new_name->type = Op_Set; + if (found) + { + if (entry->old_name[0] != '\0') + strlcpy(entry_for_new_name->old_name, entry->old_name, NAMEDATALEN); + else + strlcpy(entry_for_new_name->old_name, entry->name, NAMEDATALEN); + entry_for_new_name->owner = entry->owner; + hash_search( + CurrentDdlTable->db_table, + stmt->subname, + HASH_REMOVE, + NULL); + } + else + { + strlcpy(entry_for_new_name->old_name, stmt->subname, NAMEDATALEN); + entry_for_new_name->owner = InvalidOid; + } +} + +static void +HandleDropDb(DropdbStmt *stmt) +{ + InitDbTableIfNeeded(); + bool found = false; + DbEntry *entry = hash_search( + CurrentDdlTable->db_table, + stmt->dbname, + HASH_ENTER, + &found); + + entry->type = Op_Delete; + entry->owner = InvalidOid; + if (!found) + memset(entry->old_name, 0, sizeof(entry->old_name)); +} + +static void +HandleCreateRole(CreateRoleStmt *stmt) +{ + InitRoleTableIfNeeded(); + bool found = false; + RoleEntry *entry = hash_search( + CurrentDdlTable->role_table, + stmt->role, + HASH_ENTER, + &found); + DefElem *dpass = NULL; + ListCell *option; + + foreach(option, stmt->options) + { + DefElem *defel = lfirst(option); + + if (strcmp(defel->defname, "password") == 0) + dpass = defel; + } + if (!found) + memset(entry->old_name, 0, sizeof(entry->old_name)); + if (dpass && dpass->arg) + entry->password = MemoryContextStrdup(CurTransactionContext, strVal(dpass->arg)); + else + entry->password = NULL; + entry->type = Op_Set; +} + +static void +HandleAlterRole(AlterRoleStmt *stmt) +{ + InitRoleTableIfNeeded(); + DefElem *dpass = NULL; + ListCell *option; + + foreach(option, stmt->options) + { + DefElem *defel = lfirst(option); + + if (strcmp(defel->defname, "password") == 0) + dpass = defel; + } + /* We only care about updates to the password */ + if (!dpass) + return; + bool found = false; + RoleEntry *entry = hash_search( + CurrentDdlTable->role_table, + stmt->role->rolename, + HASH_ENTER, + &found); + + if (!found) + memset(entry->old_name, 0, sizeof(entry->old_name)); + if (dpass->arg) + entry->password = MemoryContextStrdup(CurTransactionContext, strVal(dpass->arg)); + else + entry->password = NULL; + entry->type = Op_Set; +} + +static void +HandleRoleRename(RenameStmt *stmt) +{ + InitRoleTableIfNeeded(); + Assert(stmt->renameType == OBJECT_ROLE); + bool found = false; + RoleEntry *entry = hash_search( + CurrentDdlTable->role_table, + stmt->subname, + HASH_FIND, + &found); + + RoleEntry *entry_for_new_name = hash_search( + CurrentDdlTable->role_table, + stmt->newname, + HASH_ENTER, + NULL); + + entry_for_new_name->type = Op_Set; + if (found) + { + if (entry->old_name[0] != '\0') + strlcpy(entry_for_new_name->old_name, entry->old_name, NAMEDATALEN); + else + strlcpy(entry_for_new_name->old_name, entry->name, NAMEDATALEN); + entry_for_new_name->password = entry->password; + hash_search( + CurrentDdlTable->role_table, + entry->name, + HASH_REMOVE, + NULL); + } + else + { + strlcpy(entry_for_new_name->old_name, stmt->subname, NAMEDATALEN); + entry_for_new_name->password = NULL; + } +} + +static void +HandleDropRole(DropRoleStmt *stmt) +{ + InitRoleTableIfNeeded(); + ListCell *item; + + foreach(item, stmt->roles) + { + RoleSpec *spec = lfirst(item); + bool found = false; + RoleEntry *entry = hash_search( + CurrentDdlTable->role_table, + spec->rolename, + HASH_ENTER, + &found); + + entry->type = Op_Delete; + entry->password = NULL; + if (!found) + memset(entry->old_name, 0, sizeof(entry)); + } +} + +static void +HandleRename(RenameStmt *stmt) +{ + if (stmt->renameType == OBJECT_DATABASE) + return HandleDbRename(stmt); + else if (stmt->renameType == OBJECT_ROLE) + return HandleRoleRename(stmt); +} + +static void +NeonProcessUtility( + PlannedStmt *pstmt, + const char *queryString, + bool readOnlyTree, + ProcessUtilityContext context, + ParamListInfo params, + QueryEnvironment *queryEnv, + DestReceiver *dest, + QueryCompletion *qc) +{ + Node *parseTree = pstmt->utilityStmt; + + switch (nodeTag(parseTree)) + { + case T_CreatedbStmt: + HandleCreateDb(castNode(CreatedbStmt, parseTree)); + break; + case T_AlterOwnerStmt: + HandleAlterOwner(castNode(AlterOwnerStmt, parseTree)); + break; + case T_RenameStmt: + HandleRename(castNode(RenameStmt, parseTree)); + break; + case T_DropdbStmt: + HandleDropDb(castNode(DropdbStmt, parseTree)); + break; + case T_CreateRoleStmt: + HandleCreateRole(castNode(CreateRoleStmt, parseTree)); + break; + case T_AlterRoleStmt: + HandleAlterRole(castNode(AlterRoleStmt, parseTree)); + break; + case T_DropRoleStmt: + HandleDropRole(castNode(DropRoleStmt, parseTree)); + break; + default: + break; + } + + if (PreviousProcessUtilityHook) + { + PreviousProcessUtilityHook( + pstmt, + queryString, + readOnlyTree, + context, + params, + queryEnv, + dest, + qc); + } + else + { + standard_ProcessUtility( + pstmt, + queryString, + readOnlyTree, + context, + params, + queryEnv, + dest, + qc); + } +} + +extern void +InitControlPlaneConnector() +{ + PreviousProcessUtilityHook = ProcessUtility_hook; + ProcessUtility_hook = NeonProcessUtility; + RegisterXactCallback(NeonXactCallback, NULL); + RegisterSubXactCallback(NeonSubXactCallback, NULL); + + DefineCustomStringVariable( + "neon.console_url", + "URL of the Neon Console, which will be forwarded changes to dbs and roles", + NULL, + &ConsoleURL, + NULL, + PGC_POSTMASTER, + 0, + NULL, + NULL, + NULL); + + DefineCustomBoolVariable( + "neon.forward_ddl", + "Controls whether to forward DDL to the control plane", + NULL, + &ForwardDDL, + true, + PGC_SUSET, + 0, + NULL, + NULL, + NULL); + + const char *jwt_token = getenv("NEON_CONTROL_PLANE_TOKEN"); + + if (!jwt_token) + { + elog(LOG, "Missing NEON_CONTROL_PLANE_TOKEN environment variable, forwarding will not be authenticated"); + } + + if (curl_global_init(CURL_GLOBAL_DEFAULT)) + { + elog(ERROR, "Failed to initialize curl"); + } + if ((CurlHandle = curl_easy_init()) == NULL) + { + elog(ERROR, "Failed to initialize curl handle"); + } + if ((ContentHeader = curl_slist_append(ContentHeader, "Content-Type: application/json")) == NULL) + { + elog(ERROR, "Failed to initialize content header"); + } + + if (jwt_token) + { + char auth_header[8192]; + + snprintf(auth_header, sizeof(auth_header), "Authorization: Bearer %s", jwt_token); + if ((ContentHeader = curl_slist_append(ContentHeader, auth_header)) == NULL) + { + elog(ERROR, "Failed to initialize authorization header"); + } + } +} diff --git a/pgxn/neon/control_plane_connector.h b/pgxn/neon/control_plane_connector.h new file mode 100644 index 0000000000..12d6a97562 --- /dev/null +++ b/pgxn/neon/control_plane_connector.h @@ -0,0 +1,6 @@ +#ifndef CONTROL_PLANE_CONNECTOR_H +#define CONTROL_PLANE_CONNECTOR_H + +void InitControlPlaneConnector(); + +#endif diff --git a/pgxn/neon/neon.c b/pgxn/neon/neon.c index 217c1974a0..b45d7cfc32 100644 --- a/pgxn/neon/neon.c +++ b/pgxn/neon/neon.c @@ -25,6 +25,7 @@ #include "neon.h" #include "walproposer.h" #include "pagestore_client.h" +#include "control_plane_connector.h" PG_MODULE_MAGIC; void _PG_init(void); @@ -34,7 +35,11 @@ _PG_init(void) { pg_init_libpagestore(); pg_init_walproposer(); + InitControlPlaneConnector(); + // Important: This must happen after other parts of the extension + // are loaded, otherwise any settings to GUCs that were set before + // the extension was loaded will be removed. EmitWarningsOnPlaceholders("neon"); } diff --git a/scripts/export_import_between_pageservers.py b/scripts/export_import_between_pageservers.py index 4b599ce9b6..d95878b341 100755 --- a/scripts/export_import_between_pageservers.py +++ b/scripts/export_import_between_pageservers.py @@ -535,8 +535,8 @@ def export_timeline( def main(args: argparse.Namespace): - # any psql version will do here. use current DEFAULT_PG_VERSION = 14 - psql_path = str(Path(args.pg_distrib_dir) / "v14" / "bin" / "psql") + # any psql version will do here. use current DEFAULT_PG_VERSION = 15 + psql_path = str(Path(args.pg_distrib_dir) / "v15" / "bin" / "psql") old_pageserver_host = args.old_pageserver_host new_pageserver_host = args.new_pageserver_host diff --git a/test_runner/fixtures/neon_fixtures.py b/test_runner/fixtures/neon_fixtures.py index 3ff5429616..6b97c33ae4 100644 --- a/test_runner/fixtures/neon_fixtures.py +++ b/test_runner/fixtures/neon_fixtures.py @@ -1619,6 +1619,8 @@ class NeonPageserver(PgProtocol): ".*task iteration took longer than the configured period.*", # this is until #3501 ".*Compaction failed, retrying in [^:]+: Cannot run compaction iteration on inactive tenant", + # these can happen anytime we do compactions from background task and shutdown pageserver + r".*ERROR.*ancestor timeline \S+ is being stopped", ] def start( diff --git a/test_runner/fixtures/pg_version.py b/test_runner/fixtures/pg_version.py index d67f088365..14ae88cc2c 100644 --- a/test_runner/fixtures/pg_version.py +++ b/test_runner/fixtures/pg_version.py @@ -27,6 +27,10 @@ class PgVersion(str, enum.Enum): def __repr__(self) -> str: return f"'{self.value}'" + # Make this explicit for Python 3.11 compatibility, which changes the behavior of enums + def __str__(self) -> str: + return self.value + # In GitHub workflows we use Postgres version with v-prefix (e.g. v14 instead of just 14), # sometime we need to do so in tests. @property @@ -78,11 +82,11 @@ def pytest_addoption(parser: Parser): @pytest.fixture(scope="session") def pg_version(request: FixtureRequest) -> Iterator[PgVersion]: if v := request.config.getoption("--pg-version"): - version, source = v, "from --pg-version commad-line argument" + version, source = v, "from --pg-version command-line argument" elif v := os.environ.get("DEFAULT_PG_VERSION"): version, source = PgVersion(v), "from DEFAULT_PG_VERSION environment variable" else: - version, source = DEFAULT_VERSION, "default verson" + version, source = DEFAULT_VERSION, "default version" log.info(f"pg_version is {version} ({source})") yield version diff --git a/test_runner/regress/test_ddl_forwarding.py b/test_runner/regress/test_ddl_forwarding.py new file mode 100644 index 0000000000..27ebd3c181 --- /dev/null +++ b/test_runner/regress/test_ddl_forwarding.py @@ -0,0 +1,219 @@ +from types import TracebackType +from typing import Any, Dict, List, Optional, Tuple, Type + +import psycopg2 +import pytest +from fixtures.log_helper import log +from fixtures.neon_fixtures import ( + PortDistributor, + VanillaPostgres, +) +from pytest_httpserver import HTTPServer +from werkzeug.wrappers.request import Request +from werkzeug.wrappers.response import Response + + +@pytest.fixture(scope="session") +def httpserver_listen_address(port_distributor: PortDistributor): + port = port_distributor.get_port() + return ("localhost", port) + + +def handle_db(dbs, roles, operation): + if operation["op"] == "set": + if "old_name" in operation and operation["old_name"] in dbs: + dbs[operation["name"]] = dbs[operation["old_name"]] + dbs.pop(operation["old_name"]) + if "owner" in operation: + dbs[operation["name"]] = operation["owner"] + elif operation["op"] == "del": + dbs.pop(operation["name"]) + else: + raise ValueError("Invalid op") + + +def handle_role(dbs, roles, operation): + if operation["op"] == "set": + if "old_name" in operation and operation["old_name"] in roles: + roles[operation["name"]] = roles[operation["old_name"]] + roles.pop(operation["old_name"]) + for db, owner in dbs.items(): + if owner == operation["old_name"]: + dbs[db] = operation["name"] + if "password" in operation: + roles[operation["name"]] = operation["password"] + elif operation["op"] == "del": + if "old_name" in operation: + roles.pop(operation["old_name"]) + roles.pop(operation["name"]) + else: + raise ValueError("Invalid op") + + +fail = False + + +def ddl_forward_handler(request: Request, dbs: Dict[str, str], roles: Dict[str, str]) -> Response: + log.info(f"Received request with data {request.get_data(as_text=True)}") + if fail: + log.info("FAILING") + return Response(status=500, response="Failed just cuz") + if request.json is None: + log.info("Received invalid JSON") + return Response(status=400) + json = request.json + # Handle roles first + if "roles" in json: + for operation in json["roles"]: + handle_role(dbs, roles, operation) + if "dbs" in json: + for operation in json["dbs"]: + handle_db(dbs, roles, operation) + return Response(status=200) + + +class DdlForwardingContext: + def __init__(self, httpserver: HTTPServer, vanilla_pg: VanillaPostgres, host: str, port: int): + self.server = httpserver + self.pg = vanilla_pg + self.host = host + self.port = port + self.dbs: Dict[str, str] = {} + self.roles: Dict[str, str] = {} + endpoint = "/management/api/v2/roles_and_databases" + ddl_url = f"http://{host}:{port}{endpoint}" + self.pg.configure( + [ + f"neon.console_url={ddl_url}", + "shared_preload_libraries = 'neon'", + ] + ) + log.info(f"Listening on {ddl_url}") + self.server.expect_request(endpoint, method="PATCH").respond_with_handler( + lambda request: ddl_forward_handler(request, self.dbs, self.roles) + ) + + def __enter__(self): + self.pg.start() + return self + + def __exit__( + self, + exc_type: Optional[Type[BaseException]], + exc: Optional[BaseException], + tb: Optional[TracebackType], + ): + self.pg.stop() + + def send(self, query: str) -> List[Tuple[Any, ...]]: + return self.pg.safe_psql(query) + + def wait(self, timeout=3): + self.server.wait(timeout=timeout) + + def send_and_wait(self, query: str, timeout=3) -> List[Tuple[Any, ...]]: + res = self.send(query) + self.wait(timeout=timeout) + return res + + +@pytest.fixture(scope="function") +def ddl( + httpserver: HTTPServer, vanilla_pg: VanillaPostgres, httpserver_listen_address: tuple[str, int] +): + (host, port) = httpserver_listen_address + with DdlForwardingContext(httpserver, vanilla_pg, host, port) as ddl: + yield ddl + + +def test_ddl_forwarding(ddl: DdlForwardingContext): + curr_user = ddl.send("SELECT current_user")[0][0] + log.info(f"Current user is {curr_user}") + ddl.send_and_wait("CREATE DATABASE bork") + assert ddl.dbs == {"bork": curr_user} + ddl.send_and_wait("CREATE ROLE volk WITH PASSWORD 'nu_zayats'") + ddl.send_and_wait("ALTER DATABASE bork RENAME TO nu_pogodi") + assert ddl.dbs == {"nu_pogodi": curr_user} + ddl.send_and_wait("ALTER DATABASE nu_pogodi OWNER TO volk") + assert ddl.dbs == {"nu_pogodi": "volk"} + ddl.send_and_wait("DROP DATABASE nu_pogodi") + assert ddl.dbs == {} + ddl.send_and_wait("DROP ROLE volk") + assert ddl.roles == {} + + ddl.send_and_wait("CREATE ROLE tarzan WITH PASSWORD 'of_the_apes'") + assert ddl.roles == {"tarzan": "of_the_apes"} + ddl.send_and_wait("DROP ROLE tarzan") + assert ddl.roles == {} + ddl.send_and_wait("CREATE ROLE tarzan WITH PASSWORD 'of_the_apes'") + assert ddl.roles == {"tarzan": "of_the_apes"} + ddl.send_and_wait("ALTER ROLE tarzan WITH PASSWORD 'jungle_man'") + assert ddl.roles == {"tarzan": "jungle_man"} + ddl.send_and_wait("ALTER ROLE tarzan RENAME TO mowgli") + assert ddl.roles == {"mowgli": "jungle_man"} + ddl.send_and_wait("DROP ROLE mowgli") + assert ddl.roles == {} + + conn = ddl.pg.connect() + cur = conn.cursor() + + cur.execute("BEGIN") + cur.execute("CREATE ROLE bork WITH PASSWORD 'cork'") + cur.execute("COMMIT") + ddl.wait() + assert ddl.roles == {"bork": "cork"} + cur.execute("BEGIN") + cur.execute("CREATE ROLE stork WITH PASSWORD 'pork'") + cur.execute("ABORT") + ddl.wait() + assert ("stork", "pork") not in ddl.roles.items() + cur.execute("BEGIN") + cur.execute("ALTER ROLE bork WITH PASSWORD 'pork'") + cur.execute("ALTER ROLE bork RENAME TO stork") + cur.execute("COMMIT") + ddl.wait() + assert ddl.roles == {"stork": "pork"} + cur.execute("BEGIN") + cur.execute("CREATE ROLE dork WITH PASSWORD 'york'") + cur.execute("SAVEPOINT point") + cur.execute("ALTER ROLE dork WITH PASSWORD 'zork'") + cur.execute("ALTER ROLE dork RENAME TO fork") + cur.execute("ROLLBACK TO SAVEPOINT point") + cur.execute("ALTER ROLE dork WITH PASSWORD 'fork'") + cur.execute("ALTER ROLE dork RENAME TO zork") + cur.execute("RELEASE SAVEPOINT point") + cur.execute("COMMIT") + ddl.wait() + assert ddl.roles == {"stork": "pork", "zork": "fork"} + + cur.execute("DROP ROLE stork") + cur.execute("DROP ROLE zork") + ddl.wait() + assert ddl.roles == {} + + cur.execute("CREATE ROLE bork WITH PASSWORD 'dork'") + cur.execute("CREATE ROLE stork WITH PASSWORD 'cork'") + cur.execute("BEGIN") + cur.execute("DROP ROLE bork") + cur.execute("ALTER ROLE stork RENAME TO bork") + cur.execute("COMMIT") + ddl.wait() + assert ddl.roles == {"bork": "cork"} + + cur.execute("DROP ROLE bork") + ddl.wait() + assert ddl.roles == {} + + cur.execute("CREATE ROLE bork WITH PASSWORD 'dork'") + cur.execute("CREATE DATABASE stork WITH OWNER=bork") + cur.execute("ALTER ROLE bork RENAME TO cork") + ddl.wait() + assert ddl.dbs == {"stork": "cork"} + + with pytest.raises(psycopg2.InternalError): + global fail + fail = True + cur.execute("CREATE DATABASE failure WITH OWNER=cork") + ddl.wait() + + conn.close() diff --git a/workspace_hack/Cargo.toml b/workspace_hack/Cargo.toml index 3d40f5dede..677b59f453 100644 --- a/workspace_hack/Cargo.toml +++ b/workspace_hack/Cargo.toml @@ -27,7 +27,6 @@ futures-core = { version = "0.3" } futures-executor = { version = "0.3" } futures-sink = { version = "0.3" } futures-util = { version = "0.3", features = ["channel", "io", "sink"] } -hashbrown = { version = "0.12", features = ["raw"] } itertools = { version = "0.10" } libc = { version = "0.2", features = ["extra_traits"] } log = { version = "0.4", default-features = false, features = ["std"] } @@ -39,7 +38,7 @@ num-traits = { version = "0.2", features = ["i128"] } prost = { version = "0.11" } rand = { version = "0.8", features = ["small_rng"] } regex = { version = "1" } -regex-syntax = { version = "0.6" } +regex-syntax = { version = "0.7" } reqwest = { version = "0.11", default-features = false, features = ["blocking", "json", "multipart", "rustls-tls"] } ring = { version = "0.16", features = ["std"] } rustls = { version = "0.20", features = ["dangerous_configuration"] } @@ -62,7 +61,6 @@ url = { version = "2", features = ["serde"] } anyhow = { version = "1", features = ["backtrace"] } bytes = { version = "1", features = ["serde"] } either = { version = "1" } -hashbrown = { version = "0.12", features = ["raw"] } itertools = { version = "0.10" } libc = { version = "0.2", features = ["extra_traits"] } log = { version = "0.4", default-features = false, features = ["std"] } @@ -70,7 +68,7 @@ memchr = { version = "2" } nom = { version = "7" } prost = { version = "0.11" } regex = { version = "1" } -regex-syntax = { version = "0.6" } +regex-syntax = { version = "0.7" } serde = { version = "1", features = ["alloc", "derive"] } syn-dff4ba8e3ae991db = { package = "syn", version = "1", features = ["extra-traits", "full", "visit", "visit-mut"] } syn-f595c2ba2a3f28df = { package = "syn", version = "2", features = ["extra-traits", "full", "visit-mut"] }