From fd440e7d7926733813985c5babab079136230126 Mon Sep 17 00:00:00 2001 From: Alexander Bayandin Date: Tue, 28 Nov 2023 23:43:24 +0000 Subject: [PATCH 01/38] neonvm: add pgbouncer patch to support DEALLOCATD/DISCARD ALL (#5958) pgbouncer 1.21.0 doesn't play nicely with DEALLOCATD/DISCARD ALL if prepared statement support is enabled (max_prepared_statements > 0). There's a patch[0] that improves this (it will be included in the next release of pgbouncer). This PR applies this patch on top of 1.21.0 release tarball. For some reason, the tarball doesn't include `test/test_prepared.py` (which is modified by the patch as well), so the patch can't be applied clearly. I use `filterdiff` (from `patchutils` package) to apply the required changes. [0] https://github.com/pgbouncer/pgbouncer/commit/a7b3c0a5f4caa9dbe92743d04cf1e28c4c05806c --- vm-image-spec.yaml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/vm-image-spec.yaml b/vm-image-spec.yaml index 2aa935fac6..d0ba742a63 100644 --- a/vm-image-spec.yaml +++ b/vm-image-spec.yaml @@ -88,11 +88,12 @@ build: | RUN set -e \ && apt-get update \ && apt-get install -y \ - curl \ build-essential \ - pkg-config \ + curl \ libevent-dev \ - libssl-dev + libssl-dev \ + patchutils \ + pkg-config ENV PGBOUNCER_VERSION 1.21.0 ENV PGBOUNCER_GITPATH 1_21_0 @@ -100,6 +101,7 @@ build: | && curl -sfSL https://github.com/pgbouncer/pgbouncer/releases/download/pgbouncer_${PGBOUNCER_GITPATH}/pgbouncer-${PGBOUNCER_VERSION}.tar.gz -o pgbouncer-${PGBOUNCER_VERSION}.tar.gz \ && tar xzvf pgbouncer-${PGBOUNCER_VERSION}.tar.gz \ && cd pgbouncer-${PGBOUNCER_VERSION} \ + && curl https://github.com/pgbouncer/pgbouncer/commit/a7b3c0a5f4caa9dbe92743d04cf1e28c4c05806c.patch | filterdiff --include a/src/server.c | patch -p1 \ && LDFLAGS=-static ./configure --prefix=/usr/local/pgbouncer --without-openssl \ && make -j $(nproc) \ && make install From 0d16874960f30a62b3cf716661013f794531a6e3 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 29 Nov 2023 01:24:02 +0100 Subject: [PATCH 02/38] build(deps): bump openssl from 0.10.55 to 0.10.60 (#5965) Bumps [openssl](https://github.com/sfackler/rust-openssl) from 0.10.55 to 0.10.60. Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 48 +++++++++++++++++++++++++++--------------------- 1 file changed, 27 insertions(+), 21 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 551866e34a..65c21c10aa 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -651,7 +651,7 @@ dependencies = [ "async-trait", "axum-core", "base64 0.21.1", - "bitflags", + "bitflags 1.3.2", "bytes", "futures-util", "http", @@ -848,7 +848,7 @@ version = "0.65.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cfdf7b466f9a4903edc73f95d6d2bcd5baf8ae620638762244d3f60143643cc5" dependencies = [ - "bitflags", + "bitflags 1.3.2", "cexpr", "clang-sys", "lazy_static", @@ -871,6 +871,12 @@ version = "1.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" +[[package]] +name = "bitflags" +version = "2.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "327762f6e5a765692301e5bb513e0d9fef63be86bbc14528052b1cd3e6f03e07" + [[package]] name = "block-buffer" version = "0.10.4" @@ -1054,7 +1060,7 @@ checksum = "4f423e341edefb78c9caba2d9c7f7687d0e72e89df3ce3394554754393ac3990" dependencies = [ "anstream", "anstyle", - "bitflags", + "bitflags 1.3.2", "clap_lex", "strsim", ] @@ -1376,7 +1382,7 @@ version = "0.25.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e64e6c0fbe2c17357405f7c758c1ef960fce08bdfb2c03d88d2a18d7e09c4b67" dependencies = [ - "bitflags", + "bitflags 1.3.2", "crossterm_winapi", "libc", "mio", @@ -2236,7 +2242,7 @@ version = "0.9.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8069d3ec154eb856955c1c0fbffefbf5f3c40a104ec912d4797314c1801abff" dependencies = [ - "bitflags", + "bitflags 1.3.2", "inotify-sys", "libc", ] @@ -2247,7 +2253,7 @@ version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fdd168d97690d0b8c412d6b6c10360277f4d7ee495c5d0d5d5fe0854923255cc" dependencies = [ - "bitflags", + "bitflags 1.3.2", "futures-core", "inotify-sys", "libc", @@ -2367,7 +2373,7 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8367585489f01bc55dd27404dcf56b95e6da061a256a666ab23be9ba96a2e587" dependencies = [ - "bitflags", + "bitflags 1.3.2", "libc", ] @@ -2581,7 +2587,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f346ff70e7dbfd675fe90590b92d59ef2de15a8779ae305ebcbfd3f0caf59be4" dependencies = [ "autocfg", - "bitflags", + "bitflags 1.3.2", "cfg-if", "libc", ] @@ -2592,7 +2598,7 @@ version = "0.26.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bfdda3d196821d6af13126e40375cdf7da646a96114af134d5f417a9a1dc8e1a" dependencies = [ - "bitflags", + "bitflags 1.3.2", "cfg-if", "libc", "memoffset 0.7.1", @@ -2616,7 +2622,7 @@ version = "5.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "729f63e1ca555a43fe3efa4f3efdf4801c479da85b432242a7b726f353c88486" dependencies = [ - "bitflags", + "bitflags 1.3.2", "crossbeam-channel", "filetime", "fsevent-sys", @@ -2737,11 +2743,11 @@ checksum = "0ab1bc2a289d34bd04a330323ac98a1b4bc82c9d9fcb1e66b63caa84da26b575" [[package]] name = "openssl" -version = "0.10.55" +version = "0.10.60" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "345df152bc43501c5eb9e4654ff05f794effb78d4efe3d53abc158baddc0703d" +checksum = "79a4c6c3a2b158f7f8f2a2fc5a969fa3a068df6fc9dbb4a43845436e3af7c800" dependencies = [ - "bitflags", + "bitflags 2.4.1", "cfg-if", "foreign-types", "libc", @@ -2769,9 +2775,9 @@ checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" [[package]] name = "openssl-sys" -version = "0.9.90" +version = "0.9.96" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "374533b0e45f3a7ced10fcaeccca020e66656bc03dac384f852e4e5a7a8104a6" +checksum = "3812c071ba60da8b5677cc12bcb1d42989a65553772897a7e0355545a819838f" dependencies = [ "cc", "libc", @@ -3396,7 +3402,7 @@ version = "0.14.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b1de8dacb0873f77e6aefc6d71e044761fcc68060290f5b1089fcdf84626bb69" dependencies = [ - "bitflags", + "bitflags 1.3.2", "byteorder", "hex", "lazy_static", @@ -3678,7 +3684,7 @@ version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a" dependencies = [ - "bitflags", + "bitflags 1.3.2", ] [[package]] @@ -3687,7 +3693,7 @@ version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "567664f262709473930a4bf9e51bf2ebf3348f2e748ccc50dea20646858f8f29" dependencies = [ - "bitflags", + "bitflags 1.3.2", ] [[package]] @@ -3981,7 +3987,7 @@ version = "0.36.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6da3636faa25820d8648e0e31c5d519bbb01f72fdf57131f0f5f7da5fed36eab" dependencies = [ - "bitflags", + "bitflags 1.3.2", "errno", "io-lifetimes", "libc", @@ -3995,7 +4001,7 @@ version = "0.37.25" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d4eb579851244c2c03e7c24f501c3432bed80b8f720af1d6e5b0e0f01555a035" dependencies = [ - "bitflags", + "bitflags 1.3.2", "errno", "io-lifetimes", "libc", @@ -4208,7 +4214,7 @@ version = "2.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1fc758eb7bffce5b308734e9b0c1468893cae9ff70ebf13e7090be8dcbcc83a8" dependencies = [ - "bitflags", + "bitflags 1.3.2", "core-foundation", "core-foundation-sys", "libc", From 8c195d82140ffe2361cb3eccef048bb29ad28a49 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 29 Nov 2023 02:16:35 +0100 Subject: [PATCH 03/38] build(deps): bump cryptography from 41.0.4 to 41.0.6 (#5970) Bumps [cryptography](https://github.com/pyca/cryptography) from 41.0.4 to 41.0.6. Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- poetry.lock | 50 +++++++++++++++++++++++++------------------------- 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/poetry.lock b/poetry.lock index f8838b92df..830f80dc97 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.6.1 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.7.1 and should not be changed by hand. [[package]] name = "aiohttp" @@ -875,34 +875,34 @@ files = [ [[package]] name = "cryptography" -version = "41.0.4" +version = "41.0.6" description = "cryptography is a package which provides cryptographic recipes and primitives to Python developers." optional = false python-versions = ">=3.7" files = [ - {file = "cryptography-41.0.4-cp37-abi3-macosx_10_12_universal2.whl", hash = "sha256:80907d3faa55dc5434a16579952ac6da800935cd98d14dbd62f6f042c7f5e839"}, - {file = "cryptography-41.0.4-cp37-abi3-macosx_10_12_x86_64.whl", hash = "sha256:35c00f637cd0b9d5b6c6bd11b6c3359194a8eba9c46d4e875a3660e3b400005f"}, - {file = "cryptography-41.0.4-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cecfefa17042941f94ab54f769c8ce0fe14beff2694e9ac684176a2535bf9714"}, - {file = "cryptography-41.0.4-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e40211b4923ba5a6dc9769eab704bdb3fbb58d56c5b336d30996c24fcf12aadb"}, - {file = "cryptography-41.0.4-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:23a25c09dfd0d9f28da2352503b23e086f8e78096b9fd585d1d14eca01613e13"}, - {file = "cryptography-41.0.4-cp37-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:2ed09183922d66c4ec5fdaa59b4d14e105c084dd0febd27452de8f6f74704143"}, - {file = "cryptography-41.0.4-cp37-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:5a0f09cefded00e648a127048119f77bc2b2ec61e736660b5789e638f43cc397"}, - {file = "cryptography-41.0.4-cp37-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:9eeb77214afae972a00dee47382d2591abe77bdae166bda672fb1e24702a3860"}, - {file = "cryptography-41.0.4-cp37-abi3-win32.whl", hash = "sha256:3b224890962a2d7b57cf5eeb16ccaafba6083f7b811829f00476309bce2fe0fd"}, - {file = "cryptography-41.0.4-cp37-abi3-win_amd64.whl", hash = "sha256:c880eba5175f4307129784eca96f4e70b88e57aa3f680aeba3bab0e980b0f37d"}, - {file = "cryptography-41.0.4-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:004b6ccc95943f6a9ad3142cfabcc769d7ee38a3f60fb0dddbfb431f818c3a67"}, - {file = "cryptography-41.0.4-pp310-pypy310_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:86defa8d248c3fa029da68ce61fe735432b047e32179883bdb1e79ed9bb8195e"}, - {file = "cryptography-41.0.4-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:37480760ae08065437e6573d14be973112c9e6dcaf5f11d00147ee74f37a3829"}, - {file = "cryptography-41.0.4-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:b5f4dfe950ff0479f1f00eda09c18798d4f49b98f4e2006d644b3301682ebdca"}, - {file = "cryptography-41.0.4-pp38-pypy38_pp73-macosx_10_12_x86_64.whl", hash = "sha256:7e53db173370dea832190870e975a1e09c86a879b613948f09eb49324218c14d"}, - {file = "cryptography-41.0.4-pp38-pypy38_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:5b72205a360f3b6176485a333256b9bcd48700fc755fef51c8e7e67c4b63e3ac"}, - {file = "cryptography-41.0.4-pp38-pypy38_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:93530900d14c37a46ce3d6c9e6fd35dbe5f5601bf6b3a5c325c7bffc030344d9"}, - {file = "cryptography-41.0.4-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:efc8ad4e6fc4f1752ebfb58aefece8b4e3c4cae940b0994d43649bdfce8d0d4f"}, - {file = "cryptography-41.0.4-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:c3391bd8e6de35f6f1140e50aaeb3e2b3d6a9012536ca23ab0d9c35ec18c8a91"}, - {file = "cryptography-41.0.4-pp39-pypy39_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:0d9409894f495d465fe6fda92cb70e8323e9648af912d5b9141d616df40a87b8"}, - {file = "cryptography-41.0.4-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:8ac4f9ead4bbd0bc8ab2d318f97d85147167a488be0e08814a37eb2f439d5cf6"}, - {file = "cryptography-41.0.4-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:047c4603aeb4bbd8db2756e38f5b8bd7e94318c047cfe4efeb5d715e08b49311"}, - {file = "cryptography-41.0.4.tar.gz", hash = "sha256:7febc3094125fc126a7f6fb1f420d0da639f3f32cb15c8ff0dc3997c4549f51a"}, + {file = "cryptography-41.0.6-cp37-abi3-macosx_10_12_universal2.whl", hash = "sha256:0f27acb55a4e77b9be8d550d762b0513ef3fc658cd3eb15110ebbcbd626db12c"}, + {file = "cryptography-41.0.6-cp37-abi3-macosx_10_12_x86_64.whl", hash = "sha256:ae236bb8760c1e55b7a39b6d4d32d2279bc6c7c8500b7d5a13b6fb9fc97be35b"}, + {file = "cryptography-41.0.6-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:afda76d84b053923c27ede5edc1ed7d53e3c9f475ebaf63c68e69f1403c405a8"}, + {file = "cryptography-41.0.6-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:da46e2b5df770070412c46f87bac0849b8d685c5f2679771de277a422c7d0b86"}, + {file = "cryptography-41.0.6-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:ff369dd19e8fe0528b02e8df9f2aeb2479f89b1270d90f96a63500afe9af5cae"}, + {file = "cryptography-41.0.6-cp37-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:b648fe2a45e426aaee684ddca2632f62ec4613ef362f4d681a9a6283d10e079d"}, + {file = "cryptography-41.0.6-cp37-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:5daeb18e7886a358064a68dbcaf441c036cbdb7da52ae744e7b9207b04d3908c"}, + {file = "cryptography-41.0.6-cp37-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:068bc551698c234742c40049e46840843f3d98ad7ce265fd2bd4ec0d11306596"}, + {file = "cryptography-41.0.6-cp37-abi3-win32.whl", hash = "sha256:2132d5865eea673fe6712c2ed5fb4fa49dba10768bb4cc798345748380ee3660"}, + {file = "cryptography-41.0.6-cp37-abi3-win_amd64.whl", hash = "sha256:48783b7e2bef51224020efb61b42704207dde583d7e371ef8fc2a5fb6c0aabc7"}, + {file = "cryptography-41.0.6-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:8efb2af8d4ba9dbc9c9dd8f04d19a7abb5b49eab1f3694e7b5a16a5fc2856f5c"}, + {file = "cryptography-41.0.6-pp310-pypy310_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:c5a550dc7a3b50b116323e3d376241829fd326ac47bc195e04eb33a8170902a9"}, + {file = "cryptography-41.0.6-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:85abd057699b98fce40b41737afb234fef05c67e116f6f3650782c10862c43da"}, + {file = "cryptography-41.0.6-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:f39812f70fc5c71a15aa3c97b2bbe213c3f2a460b79bd21c40d033bb34a9bf36"}, + {file = "cryptography-41.0.6-pp38-pypy38_pp73-macosx_10_12_x86_64.whl", hash = "sha256:742ae5e9a2310e9dade7932f9576606836ed174da3c7d26bc3d3ab4bd49b9f65"}, + {file = "cryptography-41.0.6-pp38-pypy38_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:35f3f288e83c3f6f10752467c48919a7a94b7d88cc00b0668372a0d2ad4f8ead"}, + {file = "cryptography-41.0.6-pp38-pypy38_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:4d03186af98b1c01a4eda396b137f29e4e3fb0173e30f885e27acec8823c1b09"}, + {file = "cryptography-41.0.6-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:b27a7fd4229abef715e064269d98a7e2909ebf92eb6912a9603c7e14c181928c"}, + {file = "cryptography-41.0.6-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:398ae1fc711b5eb78e977daa3cbf47cec20f2c08c5da129b7a296055fbb22aed"}, + {file = "cryptography-41.0.6-pp39-pypy39_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:7e00fb556bda398b99b0da289ce7053639d33b572847181d6483ad89835115f6"}, + {file = "cryptography-41.0.6-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:60e746b11b937911dc70d164060d28d273e31853bb359e2b2033c9e93e6f3c43"}, + {file = "cryptography-41.0.6-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:3288acccef021e3c3c10d58933f44e8602cf04dba96d9796d70d537bb2f4bbc4"}, + {file = "cryptography-41.0.6.tar.gz", hash = "sha256:422e3e31d63743855e43e5a6fcc8b4acab860f560f9321b0ee6269cc7ed70cc3"}, ] [package.dependencies] From a15969714cbce744956abf06b2164cf1a06e5ac0 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 29 Nov 2023 02:17:15 +0100 Subject: [PATCH 04/38] build(deps): bump openssl from 0.10.57 to 0.10.60 in /test_runner/pg_clients/rust/tokio-postgres (#5966) Bumps [openssl](https://github.com/sfackler/rust-openssl) from 0.10.57 to 0.10.60. Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- test_runner/pg_clients/rust/tokio-postgres/Cargo.lock | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test_runner/pg_clients/rust/tokio-postgres/Cargo.lock b/test_runner/pg_clients/rust/tokio-postgres/Cargo.lock index 4ea21eb378..3ac0f16e4b 100644 --- a/test_runner/pg_clients/rust/tokio-postgres/Cargo.lock +++ b/test_runner/pg_clients/rust/tokio-postgres/Cargo.lock @@ -437,9 +437,9 @@ checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d" [[package]] name = "openssl" -version = "0.10.57" +version = "0.10.60" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bac25ee399abb46215765b1cb35bc0212377e58a061560d8b29b024fd0430e7c" +checksum = "79a4c6c3a2b158f7f8f2a2fc5a969fa3a068df6fc9dbb4a43845436e3af7c800" dependencies = [ "bitflags 2.4.1", "cfg-if", @@ -469,9 +469,9 @@ checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" [[package]] name = "openssl-sys" -version = "0.9.93" +version = "0.9.96" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db4d56a4c0478783083cfafcc42493dd4a981d41669da64b4572a2a089b51b1d" +checksum = "3812c071ba60da8b5677cc12bcb1d42989a65553772897a7e0355545a819838f" dependencies = [ "cc", "libc", From c48cc020bdbcd8b0a7b9e88eda90eeacdd2397e4 Mon Sep 17 00:00:00 2001 From: John Spray Date: Wed, 29 Nov 2023 09:32:26 +0000 Subject: [PATCH 05/38] pageserver: fix race between deletion completion and incoming requests (#5941) ## Problem This is a narrow race that can leave a stuck Stopping tenant behind, while emitting a log error "Missing InProgress marker during tenant upsert, this is a bug" - Deletion request 1 puts tenant into Stopping state, and fires off background part of DeleteTenantFlow - Deletion request 2 acquires a SlotGuard for the same tenant ID, leaves a TenantSlot::InProgress in place while it checks if the tenant's state is accept able. - DeleteTenantFlow finishes, calls TenantsMap::remove, which removes the InProgress marker. - Deletion request 2 calls SlotGuard::revert, which upserts the old value (the Tenant in Stopping state), and emits the telltale log message. Closes: #5936 ## Summary of changes - Add a regression test which uses pausable failpoints to reproduce this scenario. - TenantsMap::remove is only called by DeleteTenantFlow. Its behavior is tweaked to express the different possible states, especially `InProgress` which carriers a barrier. - In DeleteTenantFlow, if we see such a barrier result from remove(), wait for the barrier and then try removing again. --------- Co-authored-by: Joonas Koivunen --- pageserver/src/tenant/delete.rs | 75 +++++++++++++++--- pageserver/src/tenant/mgr.rs | 28 ++++++- test_runner/fixtures/pageserver/http.py | 1 + test_runner/regress/test_tenant_delete.py | 94 ++++++++++++++++++++++- 4 files changed, 184 insertions(+), 14 deletions(-) diff --git a/pageserver/src/tenant/delete.rs b/pageserver/src/tenant/delete.rs index 066f239ff0..6289d40b93 100644 --- a/pageserver/src/tenant/delete.rs +++ b/pageserver/src/tenant/delete.rs @@ -6,7 +6,7 @@ use pageserver_api::models::TenantState; use remote_storage::{GenericRemoteStorage, RemotePath}; use tokio::sync::OwnedMutexGuard; use tokio_util::sync::CancellationToken; -use tracing::{error, instrument, warn, Instrument, Span}; +use tracing::{error, instrument, Instrument, Span}; use utils::{ backoff, completion, crashsafe, fs_ext, @@ -17,6 +17,7 @@ use crate::{ config::PageServerConf, context::RequestContext, task_mgr::{self, TaskKind}, + tenant::mgr::{TenantSlot, TenantsMapRemoveResult}, InitializationOrder, }; @@ -287,6 +288,8 @@ impl DeleteTenantFlow { ) -> Result<(), DeleteTenantError> { span::debug_assert_current_span_has_tenant_id(); + pausable_failpoint!("tenant-delete-before-run"); + let mut guard = Self::prepare(&tenant).await?; if let Err(e) = Self::run_inner(&mut guard, conf, remote_storage.as_ref(), &tenant).await { @@ -538,16 +541,68 @@ impl DeleteTenantFlow { .context("cleanup_remaining_fs_traces")?; { - let mut locked = tenants.write().unwrap(); - if locked.remove(&tenant.tenant_id).is_none() { - warn!("Tenant got removed from tenants map during deletion"); - }; + pausable_failpoint!("tenant-delete-before-map-remove"); - // FIXME: we should not be modifying this from outside of mgr.rs. - // This will go away when we simplify deletion (https://github.com/neondatabase/neon/issues/5080) - crate::metrics::TENANT_MANAGER - .tenant_slots - .set(locked.len() as u64); + // This block is simply removing the TenantSlot for this tenant. It requires a loop because + // we might conflict with a TenantSlot::InProgress marker and need to wait for it. + // + // This complexity will go away when we simplify how deletion works: + // https://github.com/neondatabase/neon/issues/5080 + loop { + // Under the TenantMap lock, try to remove the tenant. We usually succeed, but if + // we encounter an InProgress marker, yield the barrier it contains and wait on it. + let barrier = { + let mut locked = tenants.write().unwrap(); + let removed = locked.remove(&tenant.tenant_id); + + // FIXME: we should not be modifying this from outside of mgr.rs. + // This will go away when we simplify deletion (https://github.com/neondatabase/neon/issues/5080) + crate::metrics::TENANT_MANAGER + .tenant_slots + .set(locked.len() as u64); + + match removed { + TenantsMapRemoveResult::Occupied(TenantSlot::Attached(tenant)) => { + match tenant.current_state() { + TenantState::Stopping { .. } | TenantState::Broken { .. } => { + // Expected: we put the tenant into stopping state before we start deleting it + } + state => { + // Unexpected state + tracing::warn!( + "Tenant in unexpected state {state} after deletion" + ); + } + } + break; + } + TenantsMapRemoveResult::Occupied(TenantSlot::Secondary) => { + // This is unexpected: this secondary tenants should not have been created, and we + // are not in a position to shut it down from here. + tracing::warn!("Tenant transitioned to secondary mode while deleting!"); + break; + } + TenantsMapRemoveResult::Occupied(TenantSlot::InProgress(_)) => { + unreachable!("TenantsMap::remove handles InProgress separately, should never return it here"); + } + TenantsMapRemoveResult::Vacant => { + tracing::warn!( + "Tenant removed from TenantsMap before deletion completed" + ); + break; + } + TenantsMapRemoveResult::InProgress(barrier) => { + // An InProgress entry was found, we must wait on its barrier + barrier + } + } + }; + + tracing::info!( + "Waiting for competing operation to complete before deleting state for tenant" + ); + barrier.wait().await; + } } *guard = Self::Finished; diff --git a/pageserver/src/tenant/mgr.rs b/pageserver/src/tenant/mgr.rs index 5988dcd45f..52d697a878 100644 --- a/pageserver/src/tenant/mgr.rs +++ b/pageserver/src/tenant/mgr.rs @@ -122,6 +122,12 @@ fn exactly_one_or_none<'a>( } } +pub(crate) enum TenantsMapRemoveResult { + Occupied(TenantSlot), + Vacant, + InProgress(utils::completion::Barrier), +} + impl TenantsMap { /// Convenience function for typical usage, where we want to get a `Tenant` object, for /// working with attached tenants. If the TenantId is in the map but in Secondary state, @@ -136,12 +142,28 @@ impl TenantsMap { } } - pub(crate) fn remove(&mut self, tenant_id: &TenantId) -> Option { + /// Only for use from DeleteTenantFlow. This method directly removes a TenantSlot from the map. + /// + /// The normal way to remove a tenant is using a SlotGuard, which will gracefully remove the guarded + /// slot if the enclosed tenant is shutdown. + pub(crate) fn remove(&mut self, tenant_id: &TenantId) -> TenantsMapRemoveResult { + use std::collections::btree_map::Entry; match self { - TenantsMap::Initializing => None, + TenantsMap::Initializing => TenantsMapRemoveResult::Vacant, TenantsMap::Open(m) | TenantsMap::ShuttingDown(m) => { let key = exactly_one_or_none(m, tenant_id).map(|(k, _)| *k); - key.and_then(|key| m.remove(&key)) + match key { + Some(key) => match m.entry(key) { + Entry::Occupied(entry) => match entry.get() { + TenantSlot::InProgress(barrier) => { + TenantsMapRemoveResult::InProgress(barrier.clone()) + } + _ => TenantsMapRemoveResult::Occupied(entry.remove()), + }, + Entry::Vacant(_entry) => TenantsMapRemoveResult::Vacant, + }, + None => TenantsMapRemoveResult::Vacant, + } } } } diff --git a/test_runner/fixtures/pageserver/http.py b/test_runner/fixtures/pageserver/http.py index 76edf45496..0e00d4a7de 100644 --- a/test_runner/fixtures/pageserver/http.py +++ b/test_runner/fixtures/pageserver/http.py @@ -263,6 +263,7 @@ class PageserverHttpClient(requests.Session): def tenant_delete(self, tenant_id: TenantId): res = self.delete(f"http://localhost:{self.port}/v1/tenant/{tenant_id}") self.verbose_error(res) + return res def tenant_load(self, tenant_id: TenantId): res = self.post(f"http://localhost:{self.port}/v1/tenant/{tenant_id}/load") diff --git a/test_runner/regress/test_tenant_delete.py b/test_runner/regress/test_tenant_delete.py index 0dd1f9a295..167c8355cc 100644 --- a/test_runner/regress/test_tenant_delete.py +++ b/test_runner/regress/test_tenant_delete.py @@ -1,3 +1,4 @@ +import concurrent.futures import enum import os import shutil @@ -474,4 +475,95 @@ def test_long_timeline_create_cancelled_by_tenant_delete(neon_env_builder: NeonE deletion.join() -# TODO test concurrent deletions with "hang" failpoint +def test_tenant_delete_concurrent( + neon_env_builder: NeonEnvBuilder, + pg_bin: PgBin, +): + """ + Validate that concurrent delete requests to the same tenant behave correctly: + exactly one should succeed. + + This is a reproducer for https://github.com/neondatabase/neon/issues/5936 + """ + neon_env_builder.enable_pageserver_remote_storage(RemoteStorageKind.MOCK_S3) + env = neon_env_builder.init_start(initial_tenant_conf=MANY_SMALL_LAYERS_TENANT_CONFIG) + ps_http = env.pageserver.http_client() + tenant_id = env.initial_tenant + timeline_id = env.initial_timeline + + # Populate some data + with env.endpoints.create_start("main", tenant_id=tenant_id) as endpoint: + run_pg_bench_small(pg_bin, endpoint.connstr()) + last_flush_lsn_upload(env, endpoint, tenant_id, timeline_id) + + CONFLICT_MESSAGE = "Precondition failed: Invalid state Stopping. Expected Active or Broken" + + env.pageserver.allowed_errors.extend( + [ + # lucky race with stopping from flushing a layer we fail to schedule any uploads + ".*layer flush task.+: could not flush frozen layer: update_metadata_file", + # Errors logged from our 4xx requests + f".*{CONFLICT_MESSAGE}.*", + ] + ) + + BEFORE_REMOVE_FAILPOINT = "tenant-delete-before-map-remove" + BEFORE_RUN_FAILPOINT = "tenant-delete-before-run" + + # We will let the initial delete run until right before it would remove + # the tenant's TenantSlot. This pauses it in a state where the tenant + # is visible in Stopping state, and concurrent requests should fail with 4xx. + ps_http.configure_failpoints((BEFORE_REMOVE_FAILPOINT, "pause")) + + def delete_tenant(): + return ps_http.tenant_delete(tenant_id) + + def hit_remove_failpoint(): + assert env.pageserver.log_contains(f"at failpoint {BEFORE_REMOVE_FAILPOINT}") + + def hit_run_failpoint(): + assert env.pageserver.log_contains(f"at failpoint {BEFORE_RUN_FAILPOINT}") + + with concurrent.futures.ThreadPoolExecutor() as executor: + background_200_req = executor.submit(delete_tenant) + assert background_200_req.result(timeout=10).status_code == 202 + + # Wait until the first request completes its work and is blocked on removing + # the TenantSlot from tenant manager. + wait_until(100, 0.1, hit_remove_failpoint) + + # Start another request: this should fail when it sees a tenant in Stopping state + with pytest.raises(PageserverApiException, match=CONFLICT_MESSAGE): + ps_http.tenant_delete(tenant_id) + + # Start another background request, which will pause after acquiring a TenantSlotGuard + # but before completing. + ps_http.configure_failpoints((BEFORE_RUN_FAILPOINT, "pause")) + background_4xx_req = executor.submit(delete_tenant) + wait_until(100, 0.1, hit_run_failpoint) + + # The TenantSlot is still present while the original request is hung before + # final removal + assert ps_http.get_metric_value("pageserver_tenant_manager_slots") == 1 + + # Permit the original request to run to success + ps_http.configure_failpoints((BEFORE_REMOVE_FAILPOINT, "off")) + + # Permit the duplicate background request to run to completion and fail. + ps_http.configure_failpoints((BEFORE_RUN_FAILPOINT, "off")) + with pytest.raises(PageserverApiException, match=CONFLICT_MESSAGE): + background_4xx_req.result(timeout=10) + + # Physical deletion should have happened + assert_prefix_empty( + neon_env_builder, + prefix="/".join( + ( + "tenants", + str(tenant_id), + ) + ), + ) + + # Zero tenants remain (we deleted the default tenant) + assert ps_http.get_metric_value("pageserver_tenant_manager_slots") == 0 From 78e73b20e1c5f8cf712809e59d5c13f508543b43 Mon Sep 17 00:00:00 2001 From: Arseny Sher Date: Wed, 29 Nov 2023 11:36:23 +0300 Subject: [PATCH 06/38] Notify safekeeper readiness with systemd. To avoid downtime during deploy, as in busy regions initial load can currently take ~30s. --- Cargo.lock | 7 +++++++ Cargo.toml | 1 + safekeeper/Cargo.toml | 1 + safekeeper/src/bin/safekeeper.rs | 7 +++++++ 4 files changed, 16 insertions(+) diff --git a/Cargo.lock b/Cargo.lock index 65c21c10aa..dc463942ed 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4146,6 +4146,7 @@ dependencies = [ "reqwest", "safekeeper_api", "scopeguard", + "sd-notify", "serde", "serde_json", "serde_with", @@ -4208,6 +4209,12 @@ dependencies = [ "untrusted", ] +[[package]] +name = "sd-notify" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "621e3680f3e07db4c9c2c3fb07c6223ab2fab2e54bd3c04c3ae037990f428c32" + [[package]] name = "security-framework" version = "2.9.1" diff --git a/Cargo.toml b/Cargo.toml index c50fb7be42..6df48ffc55 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -122,6 +122,7 @@ rustls-pemfile = "1" rustls-split = "0.3" scopeguard = "1.1" sysinfo = "0.29.2" +sd-notify = "0.4.1" sentry = { version = "0.31", default-features = false, features = ["backtrace", "contexts", "panic", "rustls", "reqwest" ] } serde = { version = "1.0", features = ["derive"] } serde_json = "1" diff --git a/safekeeper/Cargo.toml b/safekeeper/Cargo.toml index 30516c0763..53fcd5ff07 100644 --- a/safekeeper/Cargo.toml +++ b/safekeeper/Cargo.toml @@ -46,6 +46,7 @@ postgres_ffi.workspace = true pq_proto.workspace = true remote_storage.workspace = true safekeeper_api.workspace = true +sd-notify.workspace = true storage_broker.workspace = true tokio-stream.workspace = true utils.workspace = true diff --git a/safekeeper/src/bin/safekeeper.rs b/safekeeper/src/bin/safekeeper.rs index 0b5bb22c8b..e59deb9fda 100644 --- a/safekeeper/src/bin/safekeeper.rs +++ b/safekeeper/src/bin/safekeeper.rs @@ -8,6 +8,7 @@ use futures::future::BoxFuture; use futures::stream::FuturesUnordered; use futures::{FutureExt, StreamExt}; use remote_storage::RemoteStorageConfig; +use sd_notify::NotifyState; use tokio::runtime::Handle; use tokio::signal::unix::{signal, SignalKind}; use tokio::task::JoinError; @@ -434,6 +435,12 @@ async fn start_safekeeper(conf: SafeKeeperConf) -> Result<()> { let mut sigint_stream = signal(SignalKind::interrupt())?; let mut sigterm_stream = signal(SignalKind::terminate())?; + // Notify systemd that we are ready. This is important as currently loading + // timelines takes significant time (~30s in busy regions). + if let Err(e) = sd_notify::notify(true, &[NotifyState::Ready]) { + warn!("systemd notify failed: {:?}", e); + } + tokio::select! { Some((task_name, res)) = tasks_handles.next()=> { error!("{} task failed: {:?}, exiting", task_name, res); From 64890594a5ddcb4027d0ad365a8681ff6c4585db Mon Sep 17 00:00:00 2001 From: Konstantin Knizhnik Date: Wed, 29 Nov 2023 12:08:20 +0200 Subject: [PATCH 07/38] Optimize storing of null page in WAL (#5910) ## Problem PG16 (https://github.com/neondatabase/postgres/pull/327) adds new function to SMGR: zeroextend It's implementation in Neon actually wal-log zero pages of extended relation. This zero page is wal-logged using XLOG_FPI. As far as page is zero, the hole optimization (excluding from the image everything between pg_upper and pd_lower) doesn't work. ## Summary of changes In case of zero page (`PageIsNull()` returns true) assume `hole_size=BLCKSZ` ## Checklist before requesting a review - [ ] I have performed a self-review of my code. - [ ] If it is a core feature, I have added thorough tests. - [ ] Do we need to implement analytics? if so did you add the relevant metrics to the dashboard? - [ ] If this PR requires public announcement, mark it with /release-notes label and add several sentences in this section. ## Checklist before merging - [ ] Do not forget to reformat commit message to not include the above checklist --------- Co-authored-by: Konstantin Knizhnik --- vendor/postgres-v16 | 2 +- vendor/revisions.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/vendor/postgres-v16 b/vendor/postgres-v16 index 763000f1d0..e3a22b7292 160000 --- a/vendor/postgres-v16 +++ b/vendor/postgres-v16 @@ -1 +1 @@ -Subproject commit 763000f1d0873b827829c41f2f6f799ffc0de55c +Subproject commit e3a22b72922055f9212eca12700190f118578362 diff --git a/vendor/revisions.json b/vendor/revisions.json index 377357e131..c4cea208ee 100644 --- a/vendor/revisions.json +++ b/vendor/revisions.json @@ -1,5 +1,5 @@ { - "postgres-v16": "763000f1d0873b827829c41f2f6f799ffc0de55c", + "postgres-v16": "e3a22b72922055f9212eca12700190f118578362", "postgres-v15": "bc88f539312fcc4bb292ce94ae9db09ab6656e8a", "postgres-v14": "dd067cf656f6810a25aca6025633d32d02c5085a" } From 70b5646fbad095b0ce9721f2088e05498cbdad98 Mon Sep 17 00:00:00 2001 From: John Spray Date: Wed, 29 Nov 2023 10:39:12 +0000 Subject: [PATCH 08/38] pageserver: remove redundant serialization helpers on DeletionList (#5960) Precursor for https://github.com/neondatabase/neon/pull/5957 ## Problem When DeletionList was written, TenantId/TimelineId didn't have human-friendly modes in their serde. #5335 added those, such that the helpers used in serialization of HashMaps are no longer necessary. ## Summary of changes - Add a unit test to ensure that this change isn't changing anything about the serialized form - Remove the serialization helpers for maps of Id --- pageserver/src/deletion_queue.rs | 69 +++++++++++++++----------------- 1 file changed, 32 insertions(+), 37 deletions(-) diff --git a/pageserver/src/deletion_queue.rs b/pageserver/src/deletion_queue.rs index ad95254a65..4bc99eb94f 100644 --- a/pageserver/src/deletion_queue.rs +++ b/pageserver/src/deletion_queue.rs @@ -15,7 +15,6 @@ use crate::virtual_file::MaybeFatalIo; use crate::virtual_file::VirtualFile; use anyhow::Context; use camino::Utf8PathBuf; -use hex::FromHex; use remote_storage::{GenericRemoteStorage, RemotePath}; use serde::Deserialize; use serde::Serialize; @@ -160,11 +159,10 @@ pub struct DeletionQueueClient { lsn_table: Arc>, } -#[derive(Debug, Serialize, Deserialize)] +#[derive(Debug, Serialize, Deserialize, PartialEq, Eq)] struct TenantDeletionList { /// For each Timeline, a list of key fragments to append to the timeline remote path /// when reconstructing a full key - #[serde(serialize_with = "to_hex_map", deserialize_with = "from_hex_map")] timelines: HashMap>, /// The generation in which this deletion was emitted: note that this may not be the @@ -179,43 +177,11 @@ impl TenantDeletionList { } } -/// For HashMaps using a `hex` compatible key, where we would like to encode the key as a string -fn to_hex_map(input: &HashMap, serializer: S) -> Result -where - S: serde::Serializer, - V: Serialize, - I: AsRef<[u8]>, -{ - let transformed = input.iter().map(|(k, v)| (hex::encode(k), v)); - - transformed - .collect::>() - .serialize(serializer) -} - -/// For HashMaps using a FromHex key, where we would like to decode the key -fn from_hex_map<'de, D, V, I>(deserializer: D) -> Result, D::Error> -where - D: serde::de::Deserializer<'de>, - V: Deserialize<'de>, - I: FromHex + std::hash::Hash + Eq, -{ - let hex_map = HashMap::::deserialize(deserializer)?; - hex_map - .into_iter() - .map(|(k, v)| { - I::from_hex(k) - .map(|k| (k, v)) - .map_err(|_| serde::de::Error::custom("Invalid hex ID")) - }) - .collect() -} - /// Files ending with this suffix will be ignored and erased /// during recovery as startup. const TEMP_SUFFIX: &str = "tmp"; -#[derive(Debug, Serialize, Deserialize)] +#[derive(Debug, Serialize, Deserialize, PartialEq, Eq)] struct DeletionList { /// Serialization version, for future use version: u8, @@ -227,7 +193,6 @@ struct DeletionList { /// nested HashMaps by TenantTimelineID. Each Tenant only appears once /// with one unique generation ID: if someone tries to push a second generation /// ID for the same tenant, we will start a new DeletionList. - #[serde(serialize_with = "to_hex_map", deserialize_with = "from_hex_map")] tenants: HashMap, /// Avoid having to walk `tenants` to calculate the number of keys in @@ -1321,4 +1286,34 @@ pub(crate) mod mock { } } } + + /// Test round-trip serialization/deserialization, and test stability of the format + /// vs. a static expected string for the serialized version. + #[test] + fn deletion_list_serialization() -> anyhow::Result<()> { + let tenant_id = "ad6c1a56f5680419d3a16ff55d97ec3c" + .to_string() + .parse::()?; + let timeline_id = "be322c834ed9e709e63b5c9698691910" + .to_string() + .parse::()?; + let generation = Generation::new(123); + + let object = + RemotePath::from_string(&format!("tenants/{tenant_id}/timelines/{timeline_id}/foo"))?; + let mut objects = [object].to_vec(); + + let mut example = DeletionList::new(1); + example.push(&tenant_id, &timeline_id, generation, &mut objects); + + let encoded = serde_json::to_string(&example)?; + + let expected = "{\"version\":1,\"sequence\":1,\"tenants\":{\"ad6c1a56f5680419d3a16ff55d97ec3c\":{\"timelines\":{\"be322c834ed9e709e63b5c9698691910\":[\"foo\"]},\"generation\":123}},\"size\":1}".to_string(); + assert_eq!(encoded, expected); + + let decoded = serde_json::from_str::(&encoded)?; + assert_eq!(example, decoded); + + Ok(()) + } } From 9e55ad47966ce458091a87cce3acb7642ae9f836 Mon Sep 17 00:00:00 2001 From: John Spray Date: Wed, 29 Nov 2023 14:52:35 +0000 Subject: [PATCH 09/38] pageserver: refactor TenantId to TenantShardId in Tenant & Timeline (#5957) (includes two preparatory commits from https://github.com/neondatabase/neon/pull/5960) ## Problem To accommodate multiple shards in the same tenant on the same pageserver, we must include the full TenantShardId in local paths. That means that all code touching local storage needs to see the TenantShardId. ## Summary of changes - Replace `tenant_id: TenantId` with `tenant_shard_id: TenantShardId` on Tenant, Timeline and RemoteTimelineClient. - Use TenantShardId in helpers for building local paths. - Update all the relevant call sites. This doesn't update absolutely everything: things like PageCache, TaskMgr, WalRedo are still shard-naive. The purpose of this PR is to update the core types so that others code can be added/updated incrementally without churning the most central shared types. --- control_plane/src/bin/attachment_service.rs | 7 +- libs/pageserver_api/src/control_api.rs | 10 +- libs/pageserver_api/src/shard.rs | 6 +- pageserver/benches/bench_layer_map.rs | 3 +- pageserver/src/config.rs | 58 ++-- pageserver/src/consumption_metrics/metrics.rs | 2 +- pageserver/src/control_plane_client.rs | 25 +- pageserver/src/deletion_queue.rs | 84 +++--- pageserver/src/deletion_queue/list_writer.rs | 20 +- pageserver/src/disk_usage_eviction_task.rs | 8 +- pageserver/src/http/routes.rs | 3 +- pageserver/src/metrics.rs | 5 +- pageserver/src/page_service.rs | 10 +- pageserver/src/tenant.rs | 262 ++++++++++-------- pageserver/src/tenant/delete.rs | 64 ++--- pageserver/src/tenant/ephemeral_file.rs | 19 +- pageserver/src/tenant/metadata.rs | 17 +- pageserver/src/tenant/mgr.rs | 129 ++++----- .../src/tenant/remote_timeline_client.rs | 122 ++++---- .../tenant/remote_timeline_client/download.rs | 42 ++- .../tenant/remote_timeline_client/upload.rs | 9 +- pageserver/src/tenant/storage_layer.rs | 11 +- .../src/tenant/storage_layer/delta_layer.rs | 53 ++-- .../src/tenant/storage_layer/image_layer.rs | 31 ++- .../tenant/storage_layer/inmemory_layer.rs | 18 +- pageserver/src/tenant/storage_layer/layer.rs | 16 +- .../src/tenant/storage_layer/layer_desc.rs | 43 +-- pageserver/src/tenant/tasks.rs | 2 +- pageserver/src/tenant/timeline.rs | 125 ++++++--- pageserver/src/tenant/timeline/delete.rs | 48 ++-- .../src/tenant/timeline/eviction_task.rs | 11 +- .../src/tenant/timeline/layer_manager.rs | 8 +- pageserver/src/tenant/timeline/uninit.rs | 14 +- pageserver/src/tenant/timeline/walreceiver.rs | 2 +- .../walreceiver/connection_manager.rs | 4 +- .../walreceiver/walreceiver_connection.rs | 2 +- pageserver/src/walredo.rs | 9 +- 37 files changed, 690 insertions(+), 612 deletions(-) diff --git a/control_plane/src/bin/attachment_service.rs b/control_plane/src/bin/attachment_service.rs index 16577e27d6..be7cff352c 100644 --- a/control_plane/src/bin/attachment_service.rs +++ b/control_plane/src/bin/attachment_service.rs @@ -9,6 +9,7 @@ use clap::Parser; use hex::FromHex; use hyper::StatusCode; use hyper::{Body, Request, Response}; +use pageserver_api::shard::TenantShardId; use serde::{Deserialize, Serialize}; use std::path::{Path, PathBuf}; use std::{collections::HashMap, sync::Arc}; @@ -173,7 +174,8 @@ async fn handle_re_attach(mut req: Request) -> Result, ApiE if state.pageserver == Some(reattach_req.node_id) { state.generation += 1; response.tenants.push(ReAttachResponseTenant { - id: *t, + // TODO(sharding): make this shard-aware + id: TenantShardId::unsharded(*t), gen: state.generation, }); } @@ -196,7 +198,8 @@ async fn handle_validate(mut req: Request) -> Result, ApiEr }; for req_tenant in validate_req.tenants { - if let Some(tenant_state) = locked.tenants.get(&req_tenant.id) { + // TODO(sharding): make this shard-aware + if let Some(tenant_state) = locked.tenants.get(&req_tenant.id.tenant_id) { let valid = tenant_state.generation == req_tenant.gen; response.tenants.push(ValidateResponseTenant { id: req_tenant.id, diff --git a/libs/pageserver_api/src/control_api.rs b/libs/pageserver_api/src/control_api.rs index 8232e81b98..0acc3a7bb0 100644 --- a/libs/pageserver_api/src/control_api.rs +++ b/libs/pageserver_api/src/control_api.rs @@ -4,7 +4,9 @@ //! See docs/rfcs/025-generation-numbers.md use serde::{Deserialize, Serialize}; -use utils::id::{NodeId, TenantId}; +use utils::id::NodeId; + +use crate::shard::TenantShardId; #[derive(Serialize, Deserialize)] pub struct ReAttachRequest { @@ -13,7 +15,7 @@ pub struct ReAttachRequest { #[derive(Serialize, Deserialize)] pub struct ReAttachResponseTenant { - pub id: TenantId, + pub id: TenantShardId, pub gen: u32, } @@ -24,7 +26,7 @@ pub struct ReAttachResponse { #[derive(Serialize, Deserialize)] pub struct ValidateRequestTenant { - pub id: TenantId, + pub id: TenantShardId, pub gen: u32, } @@ -40,6 +42,6 @@ pub struct ValidateResponse { #[derive(Serialize, Deserialize)] pub struct ValidateResponseTenant { - pub id: TenantId, + pub id: TenantShardId, pub valid: bool, } diff --git a/libs/pageserver_api/src/shard.rs b/libs/pageserver_api/src/shard.rs index 688b911425..3510b4dbca 100644 --- a/libs/pageserver_api/src/shard.rs +++ b/libs/pageserver_api/src/shard.rs @@ -5,10 +5,10 @@ use serde::{Deserialize, Serialize}; use thiserror; use utils::id::TenantId; -#[derive(Ord, PartialOrd, Eq, PartialEq, Clone, Copy, Serialize, Deserialize, Debug)] +#[derive(Ord, PartialOrd, Eq, PartialEq, Clone, Copy, Serialize, Deserialize, Debug, Hash)] pub struct ShardNumber(pub u8); -#[derive(Ord, PartialOrd, Eq, PartialEq, Clone, Copy, Serialize, Deserialize, Debug)] +#[derive(Ord, PartialOrd, Eq, PartialEq, Clone, Copy, Serialize, Deserialize, Debug, Hash)] pub struct ShardCount(pub u8); impl ShardCount { @@ -39,7 +39,7 @@ impl ShardNumber { /// Note that the binary encoding is _not_ backward compatible, because /// at the time sharding is introduced, there are no existing binary structures /// containing TenantId that we need to handle. -#[derive(Eq, PartialEq, PartialOrd, Ord, Clone, Copy)] +#[derive(Eq, PartialEq, PartialOrd, Ord, Clone, Copy, Hash)] pub struct TenantShardId { pub tenant_id: TenantId, pub shard_number: ShardNumber, diff --git a/pageserver/benches/bench_layer_map.rs b/pageserver/benches/bench_layer_map.rs index 735f358d8b..5d05af0c00 100644 --- a/pageserver/benches/bench_layer_map.rs +++ b/pageserver/benches/bench_layer_map.rs @@ -3,6 +3,7 @@ use pageserver::repository::Key; use pageserver::tenant::layer_map::LayerMap; use pageserver::tenant::storage_layer::LayerFileName; use pageserver::tenant::storage_layer::PersistentLayerDesc; +use pageserver_api::shard::TenantShardId; use rand::prelude::{SeedableRng, SliceRandom, StdRng}; use std::cmp::{max, min}; use std::fs::File; @@ -211,7 +212,7 @@ fn bench_sequential(c: &mut Criterion) { let i32 = (i as u32) % 100; let zero = Key::from_hex("000000000000000000000000000000000000").unwrap(); let layer = PersistentLayerDesc::new_img( - TenantId::generate(), + TenantShardId::unsharded(TenantId::generate()), TimelineId::generate(), zero.add(10 * i32)..zero.add(10 * i32 + 1), Lsn(i), diff --git a/pageserver/src/config.rs b/pageserver/src/config.rs index 87d9cc522e..5b170af4ef 100644 --- a/pageserver/src/config.rs +++ b/pageserver/src/config.rs @@ -5,6 +5,7 @@ //! See also `settings.md` for better description on every parameter. use anyhow::{anyhow, bail, ensure, Context, Result}; +use pageserver_api::shard::TenantShardId; use remote_storage::{RemotePath, RemoteStorageConfig}; use serde::de::IntoDeserializer; use std::env; @@ -25,7 +26,7 @@ use toml_edit::{Document, Item}; use camino::{Utf8Path, Utf8PathBuf}; use postgres_backend::AuthType; use utils::{ - id::{NodeId, TenantId, TimelineId}, + id::{NodeId, TimelineId}, logging::LogFormat, }; @@ -628,12 +629,13 @@ impl PageServerConf { self.deletion_prefix().join(format!("header-{VERSION:02x}")) } - pub fn tenant_path(&self, tenant_id: &TenantId) -> Utf8PathBuf { - self.tenants_path().join(tenant_id.to_string()) + pub fn tenant_path(&self, tenant_shard_id: &TenantShardId) -> Utf8PathBuf { + self.tenants_path().join(tenant_shard_id.to_string()) } - pub fn tenant_ignore_mark_file_path(&self, tenant_id: &TenantId) -> Utf8PathBuf { - self.tenant_path(tenant_id).join(IGNORED_TENANT_FILE_NAME) + pub fn tenant_ignore_mark_file_path(&self, tenant_shard_id: &TenantShardId) -> Utf8PathBuf { + self.tenant_path(tenant_shard_id) + .join(IGNORED_TENANT_FILE_NAME) } /// Points to a place in pageserver's local directory, @@ -641,47 +643,53 @@ impl PageServerConf { /// /// Legacy: superseded by tenant_location_config_path. Eventually /// remove this function. - pub fn tenant_config_path(&self, tenant_id: &TenantId) -> Utf8PathBuf { - self.tenant_path(tenant_id).join(TENANT_CONFIG_NAME) + pub fn tenant_config_path(&self, tenant_shard_id: &TenantShardId) -> Utf8PathBuf { + self.tenant_path(tenant_shard_id).join(TENANT_CONFIG_NAME) } - pub fn tenant_location_config_path(&self, tenant_id: &TenantId) -> Utf8PathBuf { - self.tenant_path(tenant_id) + pub fn tenant_location_config_path(&self, tenant_shard_id: &TenantShardId) -> Utf8PathBuf { + self.tenant_path(tenant_shard_id) .join(TENANT_LOCATION_CONFIG_NAME) } - pub fn timelines_path(&self, tenant_id: &TenantId) -> Utf8PathBuf { - self.tenant_path(tenant_id).join(TIMELINES_SEGMENT_NAME) + pub fn timelines_path(&self, tenant_shard_id: &TenantShardId) -> Utf8PathBuf { + self.tenant_path(tenant_shard_id) + .join(TIMELINES_SEGMENT_NAME) } - pub fn timeline_path(&self, tenant_id: &TenantId, timeline_id: &TimelineId) -> Utf8PathBuf { - self.timelines_path(tenant_id).join(timeline_id.to_string()) + pub fn timeline_path( + &self, + tenant_shard_id: &TenantShardId, + timeline_id: &TimelineId, + ) -> Utf8PathBuf { + self.timelines_path(tenant_shard_id) + .join(timeline_id.to_string()) } pub fn timeline_uninit_mark_file_path( &self, - tenant_id: TenantId, + tenant_shard_id: TenantShardId, timeline_id: TimelineId, ) -> Utf8PathBuf { path_with_suffix_extension( - self.timeline_path(&tenant_id, &timeline_id), + self.timeline_path(&tenant_shard_id, &timeline_id), TIMELINE_UNINIT_MARK_SUFFIX, ) } pub fn timeline_delete_mark_file_path( &self, - tenant_id: TenantId, + tenant_shard_id: TenantShardId, timeline_id: TimelineId, ) -> Utf8PathBuf { path_with_suffix_extension( - self.timeline_path(&tenant_id, &timeline_id), + self.timeline_path(&tenant_shard_id, &timeline_id), TIMELINE_DELETE_MARK_SUFFIX, ) } - pub fn tenant_deleted_mark_file_path(&self, tenant_id: &TenantId) -> Utf8PathBuf { - self.tenant_path(tenant_id) + pub fn tenant_deleted_mark_file_path(&self, tenant_shard_id: &TenantShardId) -> Utf8PathBuf { + self.tenant_path(tenant_shard_id) .join(TENANT_DELETED_MARKER_FILE_NAME) } @@ -691,20 +699,24 @@ impl PageServerConf { pub fn trace_path( &self, - tenant_id: &TenantId, + tenant_shard_id: &TenantShardId, timeline_id: &TimelineId, connection_id: &ConnectionId, ) -> Utf8PathBuf { self.traces_path() - .join(tenant_id.to_string()) + .join(tenant_shard_id.to_string()) .join(timeline_id.to_string()) .join(connection_id.to_string()) } /// Points to a place in pageserver's local directory, /// where certain timeline's metadata file should be located. - pub fn metadata_path(&self, tenant_id: &TenantId, timeline_id: &TimelineId) -> Utf8PathBuf { - self.timeline_path(tenant_id, timeline_id) + pub fn metadata_path( + &self, + tenant_shard_id: &TenantShardId, + timeline_id: &TimelineId, + ) -> Utf8PathBuf { + self.timeline_path(tenant_shard_id, timeline_id) .join(METADATA_FILE_NAME) } diff --git a/pageserver/src/consumption_metrics/metrics.rs b/pageserver/src/consumption_metrics/metrics.rs index 2989e15e8e..213f08484c 100644 --- a/pageserver/src/consumption_metrics/metrics.rs +++ b/pageserver/src/consumption_metrics/metrics.rs @@ -351,7 +351,7 @@ impl TimelineSnapshot { let last_record_lsn = t.get_last_record_lsn(); let current_exact_logical_size = { - let span = tracing::info_span!("collect_metrics_iteration", tenant_id = %t.tenant_id, timeline_id = %t.timeline_id); + let span = tracing::info_span!("collect_metrics_iteration", tenant_id = %t.tenant_shard_id.tenant_id, timeline_id = %t.timeline_id); let res = span .in_scope(|| t.get_current_logical_size(ctx)) .context("get_current_logical_size"); diff --git a/pageserver/src/control_plane_client.rs b/pageserver/src/control_plane_client.rs index f50c19a629..25ae3d1b01 100644 --- a/pageserver/src/control_plane_client.rs +++ b/pageserver/src/control_plane_client.rs @@ -1,16 +1,15 @@ use std::collections::HashMap; -use pageserver_api::control_api::{ - ReAttachRequest, ReAttachResponse, ValidateRequest, ValidateRequestTenant, ValidateResponse, +use pageserver_api::{ + control_api::{ + ReAttachRequest, ReAttachResponse, ValidateRequest, ValidateRequestTenant, ValidateResponse, + }, + shard::TenantShardId, }; use serde::{de::DeserializeOwned, Serialize}; use tokio_util::sync::CancellationToken; use url::Url; -use utils::{ - backoff, - generation::Generation, - id::{NodeId, TenantId}, -}; +use utils::{backoff, generation::Generation, id::NodeId}; use crate::config::PageServerConf; @@ -31,11 +30,11 @@ pub enum RetryForeverError { #[async_trait::async_trait] pub trait ControlPlaneGenerationsApi { - async fn re_attach(&self) -> Result, RetryForeverError>; + async fn re_attach(&self) -> Result, RetryForeverError>; async fn validate( &self, - tenants: Vec<(TenantId, Generation)>, - ) -> Result, RetryForeverError>; + tenants: Vec<(TenantShardId, Generation)>, + ) -> Result, RetryForeverError>; } impl ControlPlaneClient { @@ -127,7 +126,7 @@ impl ControlPlaneClient { #[async_trait::async_trait] impl ControlPlaneGenerationsApi for ControlPlaneClient { /// Block until we get a successful response, or error out if we are shut down - async fn re_attach(&self) -> Result, RetryForeverError> { + async fn re_attach(&self) -> Result, RetryForeverError> { let re_attach_path = self .base_url .join("re-attach") @@ -154,8 +153,8 @@ impl ControlPlaneGenerationsApi for ControlPlaneClient { /// Block until we get a successful response, or error out if we are shut down async fn validate( &self, - tenants: Vec<(TenantId, Generation)>, - ) -> Result, RetryForeverError> { + tenants: Vec<(TenantShardId, Generation)>, + ) -> Result, RetryForeverError> { let re_attach_path = self .base_url .join("validate") diff --git a/pageserver/src/deletion_queue.rs b/pageserver/src/deletion_queue.rs index 4bc99eb94f..7b05745483 100644 --- a/pageserver/src/deletion_queue.rs +++ b/pageserver/src/deletion_queue.rs @@ -15,6 +15,7 @@ use crate::virtual_file::MaybeFatalIo; use crate::virtual_file::VirtualFile; use anyhow::Context; use camino::Utf8PathBuf; +use pageserver_api::shard::TenantShardId; use remote_storage::{GenericRemoteStorage, RemotePath}; use serde::Deserialize; use serde::Serialize; @@ -25,7 +26,7 @@ use tracing::Instrument; use tracing::{self, debug, error}; use utils::crashsafe::path_with_suffix_extension; use utils::generation::Generation; -use utils::id::{TenantId, TimelineId}; +use utils::id::TimelineId; use utils::lsn::AtomicLsn; use utils::lsn::Lsn; @@ -193,7 +194,7 @@ struct DeletionList { /// nested HashMaps by TenantTimelineID. Each Tenant only appears once /// with one unique generation ID: if someone tries to push a second generation /// ID for the same tenant, we will start a new DeletionList. - tenants: HashMap, + tenants: HashMap, /// Avoid having to walk `tenants` to calculate the number of keys in /// the nested deletion lists @@ -265,7 +266,7 @@ impl DeletionList { /// deletion list. fn push( &mut self, - tenant: &TenantId, + tenant: &TenantShardId, timeline: &TimelineId, generation: Generation, objects: &mut Vec, @@ -357,7 +358,7 @@ struct TenantLsnState { #[derive(Default)] struct VisibleLsnUpdates { - tenants: HashMap, + tenants: HashMap, } impl VisibleLsnUpdates { @@ -414,7 +415,7 @@ impl DeletionQueueClient { pub(crate) fn recover( &self, - attached_tenants: HashMap, + attached_tenants: HashMap, ) -> Result<(), DeletionQueueError> { self.do_push( &self.tx, @@ -431,7 +432,7 @@ impl DeletionQueueClient { /// backend will later wake up and notice that the tenant's generation requires validation. pub(crate) async fn update_remote_consistent_lsn( &self, - tenant_id: TenantId, + tenant_shard_id: TenantShardId, timeline_id: TimelineId, current_generation: Generation, lsn: Lsn, @@ -442,10 +443,13 @@ impl DeletionQueueClient { .write() .expect("Lock should never be poisoned"); - let tenant_entry = locked.tenants.entry(tenant_id).or_insert(TenantLsnState { - timelines: HashMap::new(), - generation: current_generation, - }); + let tenant_entry = locked + .tenants + .entry(tenant_shard_id) + .or_insert(TenantLsnState { + timelines: HashMap::new(), + generation: current_generation, + }); if tenant_entry.generation != current_generation { // Generation might have changed if we were detached and then re-attached: in this case, @@ -472,7 +476,7 @@ impl DeletionQueueClient { /// generations in `layers` are the generations in which those layers were written. pub(crate) async fn push_layers( &self, - tenant_id: TenantId, + tenant_shard_id: TenantShardId, timeline_id: TimelineId, current_generation: Generation, layers: Vec<(LayerFileName, LayerFileMetadata)>, @@ -483,7 +487,7 @@ impl DeletionQueueClient { let mut layer_paths = Vec::new(); for (layer, meta) in layers { layer_paths.push(remote_layer_path( - &tenant_id, + &tenant_shard_id.tenant_id, &timeline_id, meta.shard, &layer, @@ -494,7 +498,7 @@ impl DeletionQueueClient { return self.flush_immediate().await; } - self.push_layers_sync(tenant_id, timeline_id, current_generation, layers) + self.push_layers_sync(tenant_shard_id, timeline_id, current_generation, layers) } /// When a Tenant has a generation, push_layers is always synchronous because @@ -504,7 +508,7 @@ impl DeletionQueueClient { /// support (``) pub(crate) fn push_layers_sync( &self, - tenant_id: TenantId, + tenant_shard_id: TenantShardId, timeline_id: TimelineId, current_generation: Generation, layers: Vec<(LayerFileName, LayerFileMetadata)>, @@ -515,7 +519,7 @@ impl DeletionQueueClient { self.do_push( &self.tx, ListWriterQueueMessage::Delete(DeletionOp { - tenant_id, + tenant_shard_id, timeline_id, layers, generation: current_generation, @@ -783,12 +787,12 @@ mod test { } fn set_latest_generation(&self, gen: Generation) { - let tenant_id = self.harness.tenant_id; + let tenant_shard_id = self.harness.tenant_shard_id; self.mock_control_plane .latest_generation .lock() .unwrap() - .insert(tenant_id, gen); + .insert(tenant_shard_id, gen); } /// Returns remote layer file name, suitable for use in assert_remote_files @@ -797,8 +801,8 @@ mod test { file_name: LayerFileName, gen: Generation, ) -> anyhow::Result { - let tenant_id = self.harness.tenant_id; - let relative_remote_path = remote_timeline_path(&tenant_id, &TIMELINE_ID); + let tenant_shard_id = self.harness.tenant_shard_id; + let relative_remote_path = remote_timeline_path(&tenant_shard_id, &TIMELINE_ID); let remote_timeline_path = self.remote_fs_dir.join(relative_remote_path.get_path()); std::fs::create_dir_all(&remote_timeline_path)?; let remote_layer_file_name = format!("{}{}", file_name, gen.get_suffix()); @@ -816,7 +820,7 @@ mod test { #[derive(Debug, Clone)] struct MockControlPlane { - pub latest_generation: std::sync::Arc>>, + pub latest_generation: std::sync::Arc>>, } impl MockControlPlane { @@ -830,20 +834,20 @@ mod test { #[async_trait::async_trait] impl ControlPlaneGenerationsApi for MockControlPlane { #[allow(clippy::diverging_sub_expression)] // False positive via async_trait - async fn re_attach(&self) -> Result, RetryForeverError> { + async fn re_attach(&self) -> Result, RetryForeverError> { unimplemented!() } async fn validate( &self, - tenants: Vec<(TenantId, Generation)>, - ) -> Result, RetryForeverError> { + tenants: Vec<(TenantShardId, Generation)>, + ) -> Result, RetryForeverError> { let mut result = HashMap::new(); let latest_generation = self.latest_generation.lock().unwrap(); - for (tenant_id, generation) in tenants { - if let Some(latest) = latest_generation.get(&tenant_id) { - result.insert(tenant_id, *latest == generation); + for (tenant_shard_id, generation) in tenants { + if let Some(latest) = latest_generation.get(&tenant_shard_id) { + result.insert(tenant_shard_id, *latest == generation); } } @@ -947,10 +951,10 @@ mod test { client.recover(HashMap::new())?; let layer_file_name_1: LayerFileName = "000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__00000000016B59D8-00000000016B5A51".parse().unwrap(); - let tenant_id = ctx.harness.tenant_id; + let tenant_shard_id = ctx.harness.tenant_shard_id; let content: Vec = "victim1 contents".into(); - let relative_remote_path = remote_timeline_path(&tenant_id, &TIMELINE_ID); + let relative_remote_path = remote_timeline_path(&tenant_shard_id, &TIMELINE_ID); let remote_timeline_path = ctx.remote_fs_dir.join(relative_remote_path.get_path()); let deletion_prefix = ctx.harness.conf.deletion_prefix(); @@ -980,7 +984,7 @@ mod test { info!("Pushing"); client .push_layers( - tenant_id, + tenant_shard_id, TIMELINE_ID, now_generation, [(layer_file_name_1.clone(), layer_metadata)].to_vec(), @@ -1027,8 +1031,8 @@ mod test { ctx.set_latest_generation(latest_generation); - let tenant_id = ctx.harness.tenant_id; - let relative_remote_path = remote_timeline_path(&tenant_id, &TIMELINE_ID); + let tenant_shard_id = ctx.harness.tenant_shard_id; + let relative_remote_path = remote_timeline_path(&tenant_shard_id, &TIMELINE_ID); let remote_timeline_path = ctx.remote_fs_dir.join(relative_remote_path.get_path()); // Initial state: a remote layer exists @@ -1038,7 +1042,7 @@ mod test { tracing::debug!("Pushing..."); client .push_layers( - tenant_id, + tenant_shard_id, TIMELINE_ID, stale_generation, [(EXAMPLE_LAYER_NAME.clone(), layer_metadata.clone())].to_vec(), @@ -1053,7 +1057,7 @@ mod test { tracing::debug!("Pushing..."); client .push_layers( - tenant_id, + tenant_shard_id, TIMELINE_ID, latest_generation, [(EXAMPLE_LAYER_NAME.clone(), layer_metadata.clone())].to_vec(), @@ -1075,9 +1079,9 @@ mod test { let client = ctx.deletion_queue.new_client(); client.recover(HashMap::new())?; - let tenant_id = ctx.harness.tenant_id; + let tenant_shard_id = ctx.harness.tenant_shard_id; - let relative_remote_path = remote_timeline_path(&tenant_id, &TIMELINE_ID); + let relative_remote_path = remote_timeline_path(&tenant_shard_id, &TIMELINE_ID); let remote_timeline_path = ctx.remote_fs_dir.join(relative_remote_path.get_path()); let deletion_prefix = ctx.harness.conf.deletion_prefix(); @@ -1093,7 +1097,7 @@ mod test { ctx.write_remote_layer(EXAMPLE_LAYER_NAME, layer_generation)?; client .push_layers( - tenant_id, + tenant_shard_id, TIMELINE_ID, now_generation.previous(), [(EXAMPLE_LAYER_NAME.clone(), layer_metadata.clone())].to_vec(), @@ -1107,7 +1111,7 @@ mod test { ctx.write_remote_layer(EXAMPLE_LAYER_NAME_ALT, layer_generation)?; client .push_layers( - tenant_id, + tenant_shard_id, TIMELINE_ID, now_generation, [(EXAMPLE_LAYER_NAME_ALT.clone(), layer_metadata.clone())].to_vec(), @@ -1138,7 +1142,7 @@ mod test { drop(client); ctx.restart().await; let client = ctx.deletion_queue.new_client(); - client.recover(HashMap::from([(tenant_id, now_generation)]))?; + client.recover(HashMap::from([(tenant_shard_id, now_generation)]))?; info!("Flush-executing"); client.flush_execute().await?; @@ -1202,7 +1206,7 @@ pub(crate) mod mock { let mut objects = op.objects; for (layer, meta) in op.layers { objects.push(remote_layer_path( - &op.tenant_id, + &op.tenant_shard_id.tenant_id, &op.timeline_id, meta.shard, &layer, @@ -1293,7 +1297,7 @@ pub(crate) mod mock { fn deletion_list_serialization() -> anyhow::Result<()> { let tenant_id = "ad6c1a56f5680419d3a16ff55d97ec3c" .to_string() - .parse::()?; + .parse::()?; let timeline_id = "be322c834ed9e709e63b5c9698691910" .to_string() .parse::()?; diff --git a/pageserver/src/deletion_queue/list_writer.rs b/pageserver/src/deletion_queue/list_writer.rs index 5d52b680e4..7ff27ceb44 100644 --- a/pageserver/src/deletion_queue/list_writer.rs +++ b/pageserver/src/deletion_queue/list_writer.rs @@ -19,6 +19,7 @@ use std::collections::HashMap; use std::fs::create_dir_all; use std::time::Duration; +use pageserver_api::shard::TenantShardId; use regex::Regex; use remote_storage::RemotePath; use tokio_util::sync::CancellationToken; @@ -26,7 +27,6 @@ use tracing::debug; use tracing::info; use tracing::warn; use utils::generation::Generation; -use utils::id::TenantId; use utils::id::TimelineId; use crate::config::PageServerConf; @@ -54,7 +54,7 @@ const FRONTEND_FLUSHING_TIMEOUT: Duration = Duration::from_millis(100); #[derive(Debug)] pub(super) struct DeletionOp { - pub(super) tenant_id: TenantId, + pub(super) tenant_shard_id: TenantShardId, pub(super) timeline_id: TimelineId, // `layers` and `objects` are both just lists of objects. `layers` is used if you do not // have a config object handy to project it to a remote key, and need the consuming worker @@ -62,14 +62,14 @@ pub(super) struct DeletionOp { pub(super) layers: Vec<(LayerFileName, LayerFileMetadata)>, pub(super) objects: Vec, - /// The _current_ generation of the Tenant attachment in which we are enqueuing + /// The _current_ generation of the Tenant shard attachment in which we are enqueuing /// this deletion. pub(super) generation: Generation, } #[derive(Debug)] pub(super) struct RecoverOp { - pub(super) attached_tenants: HashMap, + pub(super) attached_tenants: HashMap, } #[derive(Debug)] @@ -206,7 +206,7 @@ impl ListWriter { async fn recover( &mut self, - attached_tenants: HashMap, + attached_tenants: HashMap, ) -> Result<(), anyhow::Error> { debug!( "recovering with {} attached tenants", @@ -309,8 +309,8 @@ impl ListWriter { // generation was issued to another node in the interval while we restarted, // then we may treat deletion lists from the previous generation as if they // belong to our currently attached generation, and proceed to validate & execute. - for (tenant_id, tenant_list) in &mut deletion_list.tenants { - if let Some(attached_gen) = attached_tenants.get(tenant_id) { + for (tenant_shard_id, tenant_list) in &mut deletion_list.tenants { + if let Some(attached_gen) = attached_tenants.get(tenant_shard_id) { if attached_gen.previous() == tenant_list.generation { tenant_list.generation = *attached_gen; } @@ -390,7 +390,7 @@ impl ListWriter { let mut layer_paths = Vec::new(); for (layer, meta) in op.layers { layer_paths.push(remote_layer_path( - &op.tenant_id, + &op.tenant_shard_id.tenant_id, &op.timeline_id, meta.shard, &layer, @@ -400,14 +400,14 @@ impl ListWriter { layer_paths.extend(op.objects); if !self.pending.push( - &op.tenant_id, + &op.tenant_shard_id, &op.timeline_id, op.generation, &mut layer_paths, ) { self.flush().await; let retry_succeeded = self.pending.push( - &op.tenant_id, + &op.tenant_shard_id, &op.timeline_id, op.generation, &mut layer_paths, diff --git a/pageserver/src/disk_usage_eviction_task.rs b/pageserver/src/disk_usage_eviction_task.rs index 642cafad28..f01cd1cf8c 100644 --- a/pageserver/src/disk_usage_eviction_task.rs +++ b/pageserver/src/disk_usage_eviction_task.rs @@ -310,7 +310,7 @@ pub async fn disk_usage_eviction_task_iteration_impl( .unwrap() .as_micros(), partition, - desc.tenant_id, + desc.tenant_shard_id, desc.timeline_id, candidate.layer, ); @@ -380,7 +380,7 @@ pub async fn disk_usage_eviction_task_iteration_impl( let limit = Arc::new(tokio::sync::Semaphore::new(1000.max(max_batch_size))); for (timeline, batch) in batched { - let tenant_id = timeline.tenant_id; + let tenant_shard_id = timeline.tenant_shard_id; let timeline_id = timeline.timeline_id; let batch_size = u32::try_from(batch.len()).expect("batch size limited to u32::MAX during partitioning"); @@ -431,7 +431,7 @@ pub async fn disk_usage_eviction_task_iteration_impl( (evicted_bytes, evictions_failed) } } - .instrument(tracing::info_span!("evict_batch", %tenant_id, %timeline_id, batch_size)); + .instrument(tracing::info_span!("evict_batch", tenant_id=%tenant_shard_id.tenant_id, shard_id=%tenant_shard_id.shard_slug(), %timeline_id, batch_size)); js.spawn(evict); @@ -572,7 +572,7 @@ async fn collect_eviction_candidates( continue; } let info = tl.get_local_layers_for_disk_usage_eviction().await; - debug!(tenant_id=%tl.tenant_id, timeline_id=%tl.timeline_id, "timeline resident layers count: {}", info.resident_layers.len()); + debug!(tenant_id=%tl.tenant_shard_id.tenant_id, shard_id=%tl.tenant_shard_id.shard_slug(), timeline_id=%tl.timeline_id, "timeline resident layers count: {}", info.resident_layers.len()); tenant_candidates.extend( info.resident_layers .into_iter() diff --git a/pageserver/src/http/routes.rs b/pageserver/src/http/routes.rs index 5ce09500ee..9cb411c95c 100644 --- a/pageserver/src/http/routes.rs +++ b/pageserver/src/http/routes.rs @@ -356,7 +356,8 @@ async fn build_timeline_info_common( let walreceiver_status = timeline.walreceiver_status(); let info = TimelineInfo { - tenant_id: timeline.tenant_id, + // TODO(sharding): add a shard_id field, or make tenant_id into a tenant_shard_id + tenant_id: timeline.tenant_shard_id.tenant_id, timeline_id: timeline.timeline_id, ancestor_timeline_id, ancestor_lsn, diff --git a/pageserver/src/metrics.rs b/pageserver/src/metrics.rs index d5915f4c98..b178024b9a 100644 --- a/pageserver/src/metrics.rs +++ b/pageserver/src/metrics.rs @@ -7,6 +7,7 @@ use metrics::{ HistogramVec, IntCounter, IntCounterVec, IntGauge, IntGaugeVec, UIntGauge, UIntGaugeVec, }; use once_cell::sync::Lazy; +use pageserver_api::shard::TenantShardId; use strum::{EnumCount, IntoEnumIterator, VariantNames}; use strum_macros::{EnumVariantNames, IntoStaticStr}; use utils::id::{TenantId, TimelineId}; @@ -1571,9 +1572,9 @@ pub struct RemoteTimelineClientMetrics { } impl RemoteTimelineClientMetrics { - pub fn new(tenant_id: &TenantId, timeline_id: &TimelineId) -> Self { + pub fn new(tenant_shard_id: &TenantShardId, timeline_id: &TimelineId) -> Self { RemoteTimelineClientMetrics { - tenant_id: tenant_id.to_string(), + tenant_id: tenant_shard_id.tenant_id.to_string(), timeline_id: timeline_id.to_string(), calls_unfinished_gauge: Mutex::new(HashMap::default()), bytes_started_counter: Mutex::new(HashMap::default()), diff --git a/pageserver/src/page_service.rs b/pageserver/src/page_service.rs index ee5f1732e4..82c16eb9bd 100644 --- a/pageserver/src/page_service.rs +++ b/pageserver/src/page_service.rs @@ -399,6 +399,9 @@ impl PageServerHandler { { debug_assert_current_span_has_tenant_and_timeline_id(); + // TODO(sharding): enumerate local tenant shards for this tenant, and select the one + // that should serve this request. + // Make request tracer if needed let tenant = mgr::get_active_tenant_with_timeout( tenant_id, @@ -408,9 +411,10 @@ impl PageServerHandler { .await?; let mut tracer = if tenant.get_trace_read_requests() { let connection_id = ConnectionId::generate(); - let path = tenant - .conf - .trace_path(&tenant_id, &timeline_id, &connection_id); + let path = + tenant + .conf + .trace_path(&tenant.tenant_shard_id(), &timeline_id, &connection_id); Some(Tracer::new(path)) } else { None diff --git a/pageserver/src/tenant.rs b/pageserver/src/tenant.rs index bc8779b26f..fe3a5bfb79 100644 --- a/pageserver/src/tenant.rs +++ b/pageserver/src/tenant.rs @@ -17,6 +17,7 @@ use camino::{Utf8Path, Utf8PathBuf}; use enumset::EnumSet; use futures::FutureExt; use pageserver_api::models::TimelineState; +use pageserver_api::shard::TenantShardId; use remote_storage::DownloadError; use remote_storage::GenericRemoteStorage; use std::fmt; @@ -228,7 +229,7 @@ pub struct Tenant { // This is necessary to allow global config updates. tenant_conf: Arc>, - tenant_id: TenantId, + tenant_shard_id: TenantShardId, /// The remote storage generation, used to protect S3 objects from split-brain. /// Does not change over the lifetime of the [`Tenant`] object. @@ -272,7 +273,7 @@ pub struct Tenant { impl std::fmt::Debug for Tenant { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "{} ({})", self.tenant_id, self.current_state()) + write!(f, "{} ({})", self.tenant_shard_id, self.current_state()) } } @@ -466,7 +467,7 @@ impl Tenant { init_order: Option<&InitializationOrder>, _ctx: &RequestContext, ) -> anyhow::Result<()> { - let tenant_id = self.tenant_id; + let tenant_id = self.tenant_shard_id; let timeline = self.create_timeline_struct( timeline_id, @@ -558,7 +559,7 @@ impl Tenant { #[allow(clippy::too_many_arguments)] pub(crate) fn spawn( conf: &'static PageServerConf, - tenant_id: TenantId, + tenant_shard_id: TenantShardId, resources: TenantSharedResources, attached_conf: AttachedTenantConf, init_order: Option, @@ -566,8 +567,10 @@ impl Tenant { mode: SpawnMode, ctx: &RequestContext, ) -> anyhow::Result> { + // TODO(sharding): make WalRedoManager shard-aware let wal_redo_manager = Arc::new(WalRedoManager::from(PostgresRedoManager::new( - conf, tenant_id, + conf, + tenant_shard_id.tenant_id, ))); let TenantSharedResources { @@ -581,7 +584,7 @@ impl Tenant { conf, attached_conf, wal_redo_manager, - tenant_id, + tenant_shard_id, remote_storage.clone(), deletion_queue_client, )); @@ -593,7 +596,7 @@ impl Tenant { task_mgr::spawn( &tokio::runtime::Handle::current(), TaskKind::Attach, - Some(tenant_id), + Some(tenant_shard_id.tenant_id), None, "attach tenant", false, @@ -632,7 +635,7 @@ impl Tenant { match tenant_clone .preload(remote_storage, task_mgr::shutdown_token()) .instrument( - tracing::info_span!(parent: None, "attach_preload", tenant_id=%tenant_id), + tracing::info_span!(parent: None, "attach_preload", tenant_id=%tenant_shard_id.tenant_id, shard_id=%tenant_shard_id.shard_slug()), ) .await { Ok(p) => p, @@ -714,7 +717,7 @@ impl Tenant { Ok(()) } .instrument({ - let span = tracing::info_span!(parent: None, "attach", tenant_id=%tenant_id); + let span = tracing::info_span!(parent: None, "attach", tenant_id=%tenant_shard_id.tenant_id, shard_id=%tenant_shard_id.shard_slug()); span.follows_from(Span::current()); span }), @@ -732,7 +735,7 @@ impl Tenant { info!("listing remote timelines"); let (remote_timeline_ids, other_keys) = remote_timeline_client::list_remote_timelines( remote_storage, - self.tenant_id, + self.tenant_shard_id, cancel.clone(), ) .await?; @@ -844,7 +847,7 @@ impl Tenant { .with_context(|| { format!( "failed to load remote timeline {} for tenant {}", - timeline_id, self.tenant_id + timeline_id, self.tenant_shard_id ) })?; } @@ -884,7 +887,7 @@ impl Tenant { /// timeline that still exists: this can happen if we crashed during a deletion/creation, or /// if a timeline was deleted while the tenant was attached to a different pageserver. fn clean_up_timelines(&self, existent_timelines: &HashSet) -> anyhow::Result<()> { - let timelines_dir = self.conf.timelines_path(&self.tenant_id); + let timelines_dir = self.conf.timelines_path(&self.tenant_shard_id); let entries = match timelines_dir.read_dir_utf8() { Ok(d) => d, @@ -970,7 +973,7 @@ impl Tenant { span::debug_assert_current_span_has_tenant_id(); info!("downloading index file for timeline {}", timeline_id); - tokio::fs::create_dir_all(self.conf.timeline_path(&self.tenant_id, &timeline_id)) + tokio::fs::create_dir_all(self.conf.timeline_path(&self.tenant_shard_id, &timeline_id)) .await .context("Failed to create new timeline directory")?; @@ -992,10 +995,15 @@ impl Tenant { let init_order = None; // timeline loading after attach expects to find metadata file for each metadata - save_metadata(self.conf, &self.tenant_id, &timeline_id, &remote_metadata) - .await - .context("save_metadata") - .map_err(LoadLocalTimelineError::Load)?; + save_metadata( + self.conf, + &self.tenant_shard_id, + &timeline_id, + &remote_metadata, + ) + .await + .context("save_metadata") + .map_err(LoadLocalTimelineError::Load)?; self.timeline_init_and_sync( timeline_id, @@ -1012,11 +1020,13 @@ impl Tenant { /// Create a placeholder Tenant object for a broken tenant pub fn create_broken_tenant( conf: &'static PageServerConf, - tenant_id: TenantId, + tenant_shard_id: TenantShardId, reason: String, ) -> Arc { + // TODO(sharding): make WalRedoManager shard-aware let wal_redo_manager = Arc::new(WalRedoManager::from(PostgresRedoManager::new( - conf, tenant_id, + conf, + tenant_shard_id.tenant_id, ))); Arc::new(Tenant::new( TenantState::Broken { @@ -1026,7 +1036,7 @@ impl Tenant { conf, AttachedTenantConf::try_from(LocationConf::default()).unwrap(), wal_redo_manager, - tenant_id, + tenant_shard_id, None, DeletionQueueClient::broken(), )) @@ -1039,7 +1049,7 @@ impl Tenant { // completed in non topological order (for example because parent has smaller number of layer files in it) let mut timelines_to_resume_deletion: Vec<(TimelineId, Option)> = vec![]; - let timelines_dir = self.conf.timelines_path(&self.tenant_id); + let timelines_dir = self.conf.timelines_path(&self.tenant_shard_id); for entry in timelines_dir .read_dir_utf8() @@ -1070,7 +1080,7 @@ impl Tenant { "Could not parse timeline id out of the timeline uninit mark name {timeline_uninit_mark_file}", ) })?; - let timeline_dir = self.conf.timeline_path(&self.tenant_id, &timeline_id); + let timeline_dir = self.conf.timeline_path(&self.tenant_shard_id, &timeline_id); if let Err(e) = remove_timeline_and_uninit_mark(&timeline_dir, timeline_uninit_mark_file) { @@ -1087,7 +1097,7 @@ impl Tenant { info!("Found deletion mark for timeline {}", timeline_id); - match load_metadata(self.conf, &self.tenant_id, &timeline_id) { + match load_metadata(self.conf, &self.tenant_shard_id, &timeline_id) { Ok(metadata) => { timelines_to_resume_deletion.push((timeline_id, Some(metadata))) } @@ -1131,7 +1141,7 @@ impl Tenant { })?; let timeline_uninit_mark_file = self .conf - .timeline_uninit_mark_file_path(self.tenant_id, timeline_id); + .timeline_uninit_mark_file_path(self.tenant_shard_id, timeline_id); if timeline_uninit_mark_file.exists() { info!( %timeline_id, @@ -1147,7 +1157,7 @@ impl Tenant { let timeline_delete_mark_file = self .conf - .timeline_delete_mark_file_path(self.tenant_id, timeline_id); + .timeline_delete_mark_file_path(self.tenant_shard_id, timeline_id); if timeline_delete_mark_file.exists() { // Cleanup should be done in `is_delete_mark` branch above continue; @@ -1155,7 +1165,7 @@ impl Tenant { let file_name = entry.file_name(); if let Ok(timeline_id) = file_name.parse::() { - let metadata = load_metadata(self.conf, &self.tenant_id, &timeline_id) + let metadata = load_metadata(self.conf, &self.tenant_shard_id, &timeline_id) .context("failed to load metadata")?; timelines_to_load.insert(timeline_id, metadata); } else { @@ -1187,7 +1197,7 @@ impl Tenant { remote_storage.clone(), self.deletion_queue_client.clone(), self.conf, - self.tenant_id, + self.tenant_shard_id, timeline_id, self.generation, ); @@ -1393,8 +1403,12 @@ impl Tenant { .map_err(LoadLocalTimelineError::Load) } - pub fn tenant_id(&self) -> TenantId { - self.tenant_id + pub(crate) fn tenant_id(&self) -> TenantId { + self.tenant_shard_id.tenant_id + } + + pub(crate) fn tenant_shard_id(&self) -> TenantShardId { + self.tenant_shard_id } /// Get Timeline handle for given Neon timeline ID. @@ -1408,13 +1422,13 @@ impl Tenant { let timeline = timelines_accessor .get(&timeline_id) .ok_or(GetTimelineError::NotFound { - tenant_id: self.tenant_id, + tenant_id: self.tenant_shard_id.tenant_id, timeline_id, })?; if active_only && !timeline.is_active() { Err(GetTimelineError::NotActive { - tenant_id: self.tenant_id, + tenant_id: self.tenant_shard_id.tenant_id, timeline_id, state: timeline.current_state(), }) @@ -1772,7 +1786,7 @@ impl Tenant { *current_state = TenantState::Activating(ActivatingFrom::Attaching); } } - debug!(tenant_id = %self.tenant_id, "Activating tenant"); + debug!(tenant_id = %self.tenant_shard_id.tenant_id, shard_id = %self.tenant_shard_id.shard_slug(), "Activating tenant"); activating = true; // Continue outside the closure. We need to grab timelines.lock() // and we plan to turn it into a tokio::sync::Mutex in a future patch. @@ -1809,7 +1823,8 @@ impl Tenant { // times to activate. see https://github.com/neondatabase/neon/issues/4025 info!( since_creation_millis = elapsed.as_millis(), - tenant_id = %self.tenant_id, + tenant_id = %self.tenant_shard_id.tenant_id, + shard_id = %self.tenant_shard_id.shard_slug(), activated_timelines, total_timelines, post_state = <&'static str>::from(&*current_state), @@ -1906,7 +1921,7 @@ impl Tenant { // // this will additionally shutdown and await all timeline tasks. tracing::debug!("Waiting for tasks..."); - task_mgr::shutdown_tasks(None, Some(self.tenant_id), None).await; + task_mgr::shutdown_tasks(None, Some(self.tenant_shard_id.tenant_id), None).await; // Wait for any in-flight operations to complete self.gate.close().await; @@ -2081,7 +2096,7 @@ impl Tenant { receiver.changed().await.map_err( |_e: tokio::sync::watch::error::RecvError| // Tenant existed but was dropped: report it as non-existent - GetActiveTenantError::NotFound(GetTenantError::NotFound(self.tenant_id)) + GetActiveTenantError::NotFound(GetTenantError::NotFound(self.tenant_shard_id.tenant_id)) )?; } TenantState::Active { .. } => { @@ -2155,9 +2170,6 @@ where } impl Tenant { - pub fn get_tenant_id(&self) -> TenantId { - self.tenant_id - } pub fn tenant_specific_overrides(&self) -> TenantConfOpt { self.tenant_conf.read().unwrap().tenant_conf } @@ -2307,7 +2319,7 @@ impl Tenant { new_metadata, ancestor, new_timeline_id, - self.tenant_id, + self.tenant_shard_id, self.generation, Arc::clone(&self.walredo_mgr), resources, @@ -2329,14 +2341,14 @@ impl Tenant { conf: &'static PageServerConf, attached_conf: AttachedTenantConf, walredo_mgr: Arc, - tenant_id: TenantId, + tenant_shard_id: TenantShardId, remote_storage: Option, deletion_queue_client: DeletionQueueClient, ) -> Tenant { let (state, mut rx) = watch::channel(state); tokio::spawn(async move { - let tid = tenant_id.to_string(); + let tid = tenant_shard_id.to_string(); fn inspect_state(state: &TenantState) -> ([&'static str; 1], bool) { ([state.into()], matches!(state, TenantState::Broken { .. })) @@ -2388,7 +2400,7 @@ impl Tenant { }); Tenant { - tenant_id, + tenant_shard_id, generation: attached_conf.location.generation, conf, // using now here is good enough approximation to catch tenants with really long @@ -2406,17 +2418,17 @@ impl Tenant { eviction_task_tenant_state: tokio::sync::Mutex::new(EvictionTaskTenantState::default()), delete_progress: Arc::new(tokio::sync::Mutex::new(DeleteTenantFlow::default())), cancel: CancellationToken::default(), - gate: Gate::new(format!("Tenant<{tenant_id}>")), + gate: Gate::new(format!("Tenant<{tenant_shard_id}>")), } } /// Locate and load config pub(super) fn load_tenant_config( conf: &'static PageServerConf, - tenant_id: &TenantId, + tenant_shard_id: &TenantShardId, ) -> anyhow::Result { - let legacy_config_path = conf.tenant_config_path(tenant_id); - let config_path = conf.tenant_location_config_path(tenant_id); + let legacy_config_path = conf.tenant_config_path(tenant_shard_id); + let config_path = conf.tenant_location_config_path(tenant_shard_id); if config_path.exists() { // New-style config takes precedence @@ -2470,29 +2482,34 @@ impl Tenant { .with_context(|| format!("Failed to parse config from file '{path}' as toml file")) } - #[tracing::instrument(skip_all, fields(%tenant_id))] + #[tracing::instrument(skip_all, fields(tenant_id=%tenant_shard_id.tenant_id, shard_id=%tenant_shard_id.shard_slug()))] pub(super) async fn persist_tenant_config( conf: &'static PageServerConf, - tenant_id: &TenantId, + tenant_shard_id: &TenantShardId, location_conf: &LocationConf, ) -> anyhow::Result<()> { - let legacy_config_path = conf.tenant_config_path(tenant_id); - let config_path = conf.tenant_location_config_path(tenant_id); + let legacy_config_path = conf.tenant_config_path(tenant_shard_id); + let config_path = conf.tenant_location_config_path(tenant_shard_id); - Self::persist_tenant_config_at(tenant_id, &config_path, &legacy_config_path, location_conf) - .await + Self::persist_tenant_config_at( + tenant_shard_id, + &config_path, + &legacy_config_path, + location_conf, + ) + .await } - #[tracing::instrument(skip_all, fields(%tenant_id))] + #[tracing::instrument(skip_all, fields(tenant_id=%tenant_shard_id.tenant_id, shard_id=%tenant_shard_id.shard_slug()))] pub(super) async fn persist_tenant_config_at( - tenant_id: &TenantId, + tenant_shard_id: &TenantShardId, config_path: &Utf8Path, legacy_config_path: &Utf8Path, location_conf: &LocationConf, ) -> anyhow::Result<()> { // Forward compat: write out an old-style configuration that old versions can read, in case we roll back Self::persist_tenant_config_legacy( - tenant_id, + tenant_shard_id, legacy_config_path, &location_conf.tenant_conf, ) @@ -2519,14 +2536,16 @@ impl Tenant { let temp_path = path_with_suffix_extension(config_path, TEMP_FILE_SUFFIX); - let tenant_id = *tenant_id; + let tenant_shard_id = *tenant_shard_id; let config_path = config_path.to_owned(); tokio::task::spawn_blocking(move || { Handle::current().block_on(async move { let conf_content = conf_content.as_bytes(); VirtualFile::crashsafe_overwrite(&config_path, &temp_path, conf_content) .await - .with_context(|| format!("write tenant {tenant_id} config to {config_path}")) + .with_context(|| { + format!("write tenant {tenant_shard_id} config to {config_path}") + }) }) }) .await??; @@ -2534,9 +2553,9 @@ impl Tenant { Ok(()) } - #[tracing::instrument(skip_all, fields(%tenant_id))] + #[tracing::instrument(skip_all, fields(tenant_id=%tenant_shard_id.tenant_id, shard_id=%tenant_shard_id.shard_slug()))] async fn persist_tenant_config_legacy( - tenant_id: &TenantId, + tenant_shard_id: &TenantShardId, target_config_path: &Utf8Path, tenant_conf: &TenantConfOpt, ) -> anyhow::Result<()> { @@ -2554,7 +2573,7 @@ impl Tenant { let temp_path = path_with_suffix_extension(target_config_path, TEMP_FILE_SUFFIX); - let tenant_id = *tenant_id; + let tenant_shard_id = *tenant_shard_id; let target_config_path = target_config_path.to_owned(); tokio::task::spawn_blocking(move || { Handle::current().block_on(async move { @@ -2562,7 +2581,7 @@ impl Tenant { VirtualFile::crashsafe_overwrite(&target_config_path, &temp_path, conf_content) .await .with_context(|| { - format!("write tenant {tenant_id} config to {target_config_path}") + format!("write tenant {tenant_shard_id} config to {target_config_path}") }) }) }) @@ -2940,7 +2959,7 @@ impl Tenant { // temporary directory for basebackup files for the given timeline. let pgdata_path = path_with_suffix_extension( self.conf - .timelines_path(&self.tenant_id) + .timelines_path(&self.tenant_shard_id) .join(format!("basebackup-{timeline_id}")), TEMP_FILE_SUFFIX, ); @@ -2971,7 +2990,7 @@ impl Tenant { || async { self::remote_timeline_client::upload_initdb_dir( storage, - &self.tenant_id, + &self.tenant_shard_id.tenant_id, &timeline_id, pgdata_zstd.clone(), ) @@ -3010,7 +3029,7 @@ impl Tenant { ) .await?; - let tenant_id = raw_timeline.owning_tenant.tenant_id; + let tenant_shard_id = raw_timeline.owning_tenant.tenant_shard_id; let unfinished_timeline = raw_timeline.raw_timeline()?; import_datadir::import_timeline_from_postgres_datadir( @@ -3021,7 +3040,7 @@ impl Tenant { ) .await .with_context(|| { - format!("Failed to import pgdatadir for timeline {tenant_id}/{timeline_id}") + format!("Failed to import pgdatadir for timeline {tenant_shard_id}/{timeline_id}") })?; // Flush the new layer files to disk, before we make the timeline as available to @@ -3039,7 +3058,7 @@ impl Tenant { .await .with_context(|| { format!( - "Failed to flush after pgdatadir import for timeline {tenant_id}/{timeline_id}" + "Failed to flush after pgdatadir import for timeline {tenant_shard_id}/{timeline_id}" ) })?; @@ -3062,7 +3081,7 @@ impl Tenant { remote_storage.clone(), self.deletion_queue_client.clone(), self.conf, - self.tenant_id, + self.tenant_shard_id, timeline_id, self.generation, ); @@ -3091,7 +3110,7 @@ impl Tenant { start_lsn: Lsn, ancestor: Option>, ) -> anyhow::Result { - let tenant_id = self.tenant_id; + let tenant_shard_id = self.tenant_shard_id; let resources = self.build_timeline_resources(new_timeline_id); if let Some(remote_client) = &resources.remote_client { @@ -3115,12 +3134,14 @@ impl Tenant { .create_timeline_files(&uninit_mark.timeline_path, &new_timeline_id, new_metadata) .await { - error!("Failed to create initial files for timeline {tenant_id}/{new_timeline_id}, cleaning up: {e:?}"); + error!("Failed to create initial files for timeline {tenant_shard_id}/{new_timeline_id}, cleaning up: {e:?}"); cleanup_timeline_directory(uninit_mark); return Err(e); } - debug!("Successfully created initial files for timeline {tenant_id}/{new_timeline_id}"); + debug!( + "Successfully created initial files for timeline {tenant_shard_id}/{new_timeline_id}" + ); Ok(UninitializedTimeline::new( self, @@ -3141,9 +3162,14 @@ impl Tenant { anyhow::bail!("failpoint after-timeline-uninit-mark-creation"); }); - save_metadata(self.conf, &self.tenant_id, new_timeline_id, new_metadata) - .await - .context("Failed to create timeline metadata")?; + save_metadata( + self.conf, + &self.tenant_shard_id, + new_timeline_id, + new_metadata, + ) + .await + .context("Failed to create timeline metadata")?; Ok(()) } @@ -3156,13 +3182,13 @@ impl Tenant { timeline_id: TimelineId, timelines: &MutexGuard>>, ) -> anyhow::Result { - let tenant_id = self.tenant_id; + let tenant_shard_id = self.tenant_shard_id; anyhow::ensure!( timelines.get(&timeline_id).is_none(), - "Timeline {tenant_id}/{timeline_id} already exists in pageserver's memory" + "Timeline {tenant_shard_id}/{timeline_id} already exists in pageserver's memory" ); - let timeline_path = self.conf.timeline_path(&tenant_id, &timeline_id); + let timeline_path = self.conf.timeline_path(&tenant_shard_id, &timeline_id); anyhow::ensure!( !timeline_path.exists(), "Timeline {timeline_path} already exists, cannot create its uninit mark file", @@ -3170,7 +3196,7 @@ impl Tenant { let uninit_mark_path = self .conf - .timeline_uninit_mark_file_path(tenant_id, timeline_id); + .timeline_uninit_mark_file_path(tenant_shard_id, timeline_id); fs::File::create(&uninit_mark_path) .context("Failed to create uninit mark file") .and_then(|_| { @@ -3178,7 +3204,7 @@ impl Tenant { .context("Failed to fsync uninit mark file") }) .with_context(|| { - format!("Failed to crate uninit mark for timeline {tenant_id}/{timeline_id}") + format!("Failed to crate uninit mark for timeline {tenant_shard_id}/{timeline_id}") })?; let uninit_mark = TimelineUninitMark::new(uninit_mark_path, timeline_path); @@ -3189,7 +3215,7 @@ impl Tenant { /// Gathers inputs from all of the timelines to produce a sizing model input. /// /// Future is cancellation safe. Only one calculation can be running at once per tenant. - #[instrument(skip_all, fields(tenant_id=%self.tenant_id))] + #[instrument(skip_all, fields(tenant_id=%self.tenant_shard_id.tenant_id, shard_id=%self.tenant_shard_id.shard_slug()))] pub async fn gather_size_inputs( &self, // `max_retention_period` overrides the cutoff that is used to calculate the size @@ -3228,7 +3254,7 @@ impl Tenant { /// Calculate synthetic tenant size and cache the result. /// This is periodically called by background worker. /// result is cached in tenant struct - #[instrument(skip_all, fields(tenant_id=%self.tenant_id))] + #[instrument(skip_all, fields(tenant_id=%self.tenant_shard_id.tenant_id, shard_id=%self.tenant_shard_id.shard_slug()))] pub async fn calculate_synthetic_size( &self, cause: LogicalSizeCalculationCause, @@ -3250,7 +3276,7 @@ impl Tenant { .store(size, Ordering::Relaxed); TENANT_SYNTHETIC_SIZE_METRIC - .get_metric_with_label_values(&[&self.tenant_id.to_string()]) + .get_metric_with_label_values(&[&self.tenant_shard_id.tenant_id.to_string()]) .unwrap() .set(size); } @@ -3286,9 +3312,9 @@ fn remove_timeline_and_uninit_mark( pub(crate) async fn create_tenant_files( conf: &'static PageServerConf, location_conf: &LocationConf, - tenant_id: &TenantId, + tenant_shard_id: &TenantShardId, ) -> anyhow::Result { - let target_tenant_directory = conf.tenant_path(tenant_id); + let target_tenant_directory = conf.tenant_path(tenant_shard_id); anyhow::ensure!( !target_tenant_directory .try_exists() @@ -3308,14 +3334,16 @@ pub(crate) async fn create_tenant_files( let creation_result = try_create_target_tenant_dir( conf, location_conf, - tenant_id, + tenant_shard_id, &temporary_tenant_dir, &target_tenant_directory, ) .await; if creation_result.is_err() { - error!("Failed to create directory structure for tenant {tenant_id}, cleaning tmp data"); + error!( + "Failed to create directory structure for tenant {tenant_shard_id}, cleaning tmp data" + ); if let Err(e) = fs::remove_dir_all(&temporary_tenant_dir) { error!("Failed to remove temporary tenant directory {temporary_tenant_dir:?}: {e}") } else if let Err(e) = crashsafe::fsync(&temporary_tenant_dir) { @@ -3333,31 +3361,31 @@ pub(crate) async fn create_tenant_files( async fn try_create_target_tenant_dir( conf: &'static PageServerConf, location_conf: &LocationConf, - tenant_id: &TenantId, + tenant_shard_id: &TenantShardId, temporary_tenant_dir: &Utf8Path, target_tenant_directory: &Utf8Path, ) -> Result<(), anyhow::Error> { let temporary_tenant_timelines_dir = rebase_directory( - &conf.timelines_path(tenant_id), + &conf.timelines_path(tenant_shard_id), target_tenant_directory, temporary_tenant_dir, ) - .with_context(|| format!("resolve tenant {tenant_id} temporary timelines dir"))?; + .with_context(|| format!("resolve tenant {tenant_shard_id} temporary timelines dir"))?; let temporary_legacy_tenant_config_path = rebase_directory( - &conf.tenant_config_path(tenant_id), + &conf.tenant_config_path(tenant_shard_id), target_tenant_directory, temporary_tenant_dir, ) - .with_context(|| format!("resolve tenant {tenant_id} temporary config path"))?; + .with_context(|| format!("resolve tenant {tenant_shard_id} temporary config path"))?; let temporary_tenant_config_path = rebase_directory( - &conf.tenant_location_config_path(tenant_id), + &conf.tenant_location_config_path(tenant_shard_id), target_tenant_directory, temporary_tenant_dir, ) - .with_context(|| format!("resolve tenant {tenant_id} temporary config path"))?; + .with_context(|| format!("resolve tenant {tenant_shard_id} temporary config path"))?; Tenant::persist_tenant_config_at( - tenant_id, + tenant_shard_id, &temporary_tenant_config_path, &temporary_legacy_tenant_config_path, location_conf, @@ -3367,7 +3395,7 @@ async fn try_create_target_tenant_dir( crashsafe::create_dir(&temporary_tenant_timelines_dir).with_context(|| { format!( "create tenant {} temporary timelines directory {}", - tenant_id, temporary_tenant_timelines_dir, + tenant_shard_id, temporary_tenant_timelines_dir, ) })?; fail::fail_point!("tenant-creation-before-tmp-rename", |_| { @@ -3382,19 +3410,19 @@ async fn try_create_target_tenant_dir( fs::rename(temporary_tenant_dir, target_tenant_directory).with_context(|| { format!( "move tenant {} temporary directory {} into the permanent one {}", - tenant_id, temporary_tenant_dir, target_tenant_directory + tenant_shard_id, temporary_tenant_dir, target_tenant_directory ) })?; let target_dir_parent = target_tenant_directory.parent().with_context(|| { format!( "get tenant {} dir parent for {}", - tenant_id, target_tenant_directory, + tenant_shard_id, target_tenant_directory, ) })?; crashsafe::fsync(target_dir_parent).with_context(|| { format!( "fsync renamed directory's parent {} for tenant {}", - target_dir_parent, tenant_id, + target_dir_parent, tenant_shard_id, ) })?; @@ -3472,7 +3500,7 @@ async fn run_initdb( impl Drop for Tenant { fn drop(&mut self) { - remove_tenant_metrics(&self.tenant_id); + remove_tenant_metrics(&self.tenant_shard_id.tenant_id); } } /// Dump contents of a layer file to stdout. @@ -3575,7 +3603,9 @@ pub(crate) mod harness { pub struct TenantHarness { pub conf: &'static PageServerConf, pub tenant_conf: TenantConf, - pub tenant_id: TenantId, + // TODO(sharding): remove duplicative `tenant_id` in favor of access to tenant_shard_id + pub(crate) tenant_id: TenantId, + pub tenant_shard_id: TenantShardId, pub generation: Generation, pub shard: ShardIndex, pub remote_storage: GenericRemoteStorage, @@ -3620,8 +3650,9 @@ pub(crate) mod harness { }; let tenant_id = TenantId::generate(); - fs::create_dir_all(conf.tenant_path(&tenant_id))?; - fs::create_dir_all(conf.timelines_path(&tenant_id))?; + let tenant_shard_id = TenantShardId::unsharded(tenant_id); + fs::create_dir_all(conf.tenant_path(&tenant_shard_id))?; + fs::create_dir_all(conf.timelines_path(&tenant_shard_id))?; use remote_storage::{RemoteStorageConfig, RemoteStorageKind}; let remote_fs_dir = conf.workdir.join("localfs"); @@ -3636,6 +3667,7 @@ pub(crate) mod harness { conf, tenant_conf, tenant_id, + tenant_shard_id, generation: Generation::new(0xdeadbeef), shard: ShardIndex::unsharded(), remote_storage, @@ -3655,7 +3687,7 @@ pub(crate) mod harness { } fn remote_empty(&self) -> bool { - let tenant_path = self.conf.tenant_path(&self.tenant_id); + let tenant_path = self.conf.tenant_path(&self.tenant_shard_id); let remote_tenant_dir = self .remote_fs_dir .join(tenant_path.strip_prefix(&self.conf.workdir).unwrap()); @@ -3695,7 +3727,7 @@ pub(crate) mod harness { )) .unwrap(), walredo_mgr, - self.tenant_id, + self.tenant_shard_id, Some(self.remote_storage.clone()), self.deletion_queue.new_client(), )); @@ -3704,17 +3736,17 @@ pub(crate) mod harness { LoadMode::Local => { tenant .load_local(None, ctx) - .instrument(info_span!("try_load", tenant_id=%self.tenant_id)) + .instrument(info_span!("try_load", tenant_id=%self.tenant_shard_id.tenant_id, shard_id=%self.tenant_shard_id.shard_slug())) .await?; } LoadMode::Remote => { let preload = tenant .preload(&self.remote_storage, CancellationToken::new()) - .instrument(info_span!("try_load_preload", tenant_id=%self.tenant_id)) + .instrument(info_span!("try_load_preload", tenant_id=%self.tenant_shard_id.tenant_id, shard_id=%self.tenant_shard_id.shard_slug())) .await?; tenant .attach(None, Some(preload), ctx) - .instrument(info_span!("try_load", tenant_id=%self.tenant_id)) + .instrument(info_span!("try_load", tenant_id=%self.tenant_shard_id.tenant_id, shard_id=%self.tenant_shard_id.shard_slug())) .await?; } } @@ -3748,7 +3780,7 @@ pub(crate) mod harness { } pub fn timeline_path(&self, timeline_id: &TimelineId) -> Utf8PathBuf { - self.conf.timeline_path(&self.tenant_id, timeline_id) + self.conf.timeline_path(&self.tenant_shard_id, timeline_id) } } @@ -3864,7 +3896,7 @@ mod tests { e.to_string(), format!( "Timeline {}/{} already exists in pageserver's memory", - tenant.tenant_id, TIMELINE_ID + tenant.tenant_shard_id, TIMELINE_ID ) ), } @@ -4248,7 +4280,7 @@ mod tests { // so that all uploads finish & we can call harness.load() below again tenant .shutdown(Default::default(), true) - .instrument(info_span!("test_shutdown", tenant_id=%tenant.tenant_id)) + .instrument(info_span!("test_shutdown", tenant_id=%tenant.tenant_shard_id)) .await .ok() .unwrap(); @@ -4289,7 +4321,7 @@ mod tests { // so that all uploads finish & we can call harness.load() below again tenant .shutdown(Default::default(), true) - .instrument(info_span!("test_shutdown", tenant_id=%tenant.tenant_id)) + .instrument(info_span!("test_shutdown", tenant_id=%tenant.tenant_shard_id)) .await .ok() .unwrap(); @@ -4351,7 +4383,7 @@ mod tests { // so that all uploads finish & we can call harness.try_load() below again tenant .shutdown(Default::default(), true) - .instrument(info_span!("test_shutdown", tenant_id=%tenant.tenant_id)) + .instrument(info_span!("test_shutdown", tenant_id=%tenant.tenant_shard_id)) .await .ok() .unwrap(); @@ -4884,7 +4916,7 @@ mod tests { let raw_tline = tline.raw_timeline().unwrap(); raw_tline .shutdown() - .instrument(info_span!("test_shutdown", tenant_id=%raw_tline.tenant_id)) + .instrument(info_span!("test_shutdown", tenant_id=%raw_tline.tenant_shard_id)) .await; std::mem::forget(tline); } @@ -4896,7 +4928,7 @@ mod tests { assert_eq!( e, GetTimelineError::NotFound { - tenant_id: tenant.tenant_id, + tenant_id: tenant.tenant_shard_id.tenant_id, timeline_id: TIMELINE_ID, } ) @@ -4905,12 +4937,12 @@ mod tests { assert!(!harness .conf - .timeline_path(&tenant.tenant_id, &TIMELINE_ID) + .timeline_path(&tenant.tenant_shard_id, &TIMELINE_ID) .exists()); assert!(!harness .conf - .timeline_uninit_mark_file_path(tenant.tenant_id, TIMELINE_ID) + .timeline_uninit_mark_file_path(tenant.tenant_shard_id, TIMELINE_ID) .exists()); Ok(()) diff --git a/pageserver/src/tenant/delete.rs b/pageserver/src/tenant/delete.rs index 6289d40b93..b7b2ef9c79 100644 --- a/pageserver/src/tenant/delete.rs +++ b/pageserver/src/tenant/delete.rs @@ -2,16 +2,13 @@ use std::sync::Arc; use anyhow::Context; use camino::{Utf8Path, Utf8PathBuf}; -use pageserver_api::models::TenantState; +use pageserver_api::{models::TenantState, shard::TenantShardId}; use remote_storage::{GenericRemoteStorage, RemotePath}; use tokio::sync::OwnedMutexGuard; use tokio_util::sync::CancellationToken; use tracing::{error, instrument, Instrument, Span}; -use utils::{ - backoff, completion, crashsafe, fs_ext, - id::{TenantId, TimelineId}, -}; +use utils::{backoff, completion, crashsafe, fs_ext, id::TimelineId}; use crate::{ config::PageServerConf, @@ -60,10 +57,10 @@ type DeletionGuard = tokio::sync::OwnedMutexGuard; fn remote_tenant_delete_mark_path( conf: &PageServerConf, - tenant_id: &TenantId, + tenant_shard_id: &TenantShardId, ) -> anyhow::Result { let tenant_remote_path = conf - .tenant_path(tenant_id) + .tenant_path(tenant_shard_id) .strip_prefix(&conf.workdir) .context("Failed to strip workdir prefix") .and_then(RemotePath::new) @@ -74,9 +71,9 @@ fn remote_tenant_delete_mark_path( async fn create_remote_delete_mark( conf: &PageServerConf, remote_storage: &GenericRemoteStorage, - tenant_id: &TenantId, + tenant_shard_id: &TenantShardId, ) -> Result<(), DeleteTenantError> { - let remote_mark_path = remote_tenant_delete_mark_path(conf, tenant_id)?; + let remote_mark_path = remote_tenant_delete_mark_path(conf, tenant_shard_id)?; let data: &[u8] = &[]; backoff::retry( @@ -100,9 +97,9 @@ async fn create_remote_delete_mark( async fn create_local_delete_mark( conf: &PageServerConf, - tenant_id: &TenantId, + tenant_shard_id: &TenantShardId, ) -> Result<(), DeleteTenantError> { - let marker_path = conf.tenant_deleted_mark_file_path(tenant_id); + let marker_path = conf.tenant_deleted_mark_file_path(tenant_shard_id); // Note: we're ok to replace existing file. let _ = std::fs::OpenOptions::new() @@ -171,10 +168,10 @@ async fn ensure_timelines_dir_empty(timelines_path: &Utf8Path) -> Result<(), Del async fn remove_tenant_remote_delete_mark( conf: &PageServerConf, remote_storage: Option<&GenericRemoteStorage>, - tenant_id: &TenantId, + tenant_shard_id: &TenantShardId, ) -> Result<(), DeleteTenantError> { if let Some(remote_storage) = remote_storage { - let path = remote_tenant_delete_mark_path(conf, tenant_id)?; + let path = remote_tenant_delete_mark_path(conf, tenant_shard_id)?; backoff::retry( || async { remote_storage.delete(&path).await }, |_e| false, @@ -193,7 +190,7 @@ async fn remove_tenant_remote_delete_mark( // Cleanup fs traces: tenant config, timelines dir local delete mark, tenant dir async fn cleanup_remaining_fs_traces( conf: &PageServerConf, - tenant_id: &TenantId, + tenant_shard_id: &TenantShardId, ) -> Result<(), DeleteTenantError> { let rm = |p: Utf8PathBuf, is_dir: bool| async move { if is_dir { @@ -205,8 +202,8 @@ async fn cleanup_remaining_fs_traces( .with_context(|| format!("failed to delete {p}")) }; - rm(conf.tenant_config_path(tenant_id), false).await?; - rm(conf.tenant_location_config_path(tenant_id), false).await?; + rm(conf.tenant_config_path(tenant_shard_id), false).await?; + rm(conf.tenant_location_config_path(tenant_shard_id), false).await?; fail::fail_point!("tenant-delete-before-remove-timelines-dir", |_| { Err(anyhow::anyhow!( @@ -214,7 +211,7 @@ async fn cleanup_remaining_fs_traces( ))? }); - rm(conf.timelines_path(tenant_id), true).await?; + rm(conf.timelines_path(tenant_shard_id), true).await?; fail::fail_point!("tenant-delete-before-remove-deleted-mark", |_| { Err(anyhow::anyhow!( @@ -228,14 +225,14 @@ async fn cleanup_remaining_fs_traces( // to be reordered later and thus missed if a crash occurs. // Note that we dont need to sync after mark file is removed // because we can tolerate the case when mark file reappears on startup. - let tenant_path = &conf.tenant_path(tenant_id); + let tenant_path = &conf.tenant_path(tenant_shard_id); if tenant_path.exists() { - crashsafe::fsync_async(&conf.tenant_path(tenant_id)) + crashsafe::fsync_async(&conf.tenant_path(tenant_shard_id)) .await .context("fsync_pre_mark_remove")?; } - rm(conf.tenant_deleted_mark_file_path(tenant_id), false).await?; + rm(conf.tenant_deleted_mark_file_path(tenant_shard_id), false).await?; fail::fail_point!("tenant-delete-before-remove-tenant-dir", |_| { Err(anyhow::anyhow!( @@ -243,7 +240,7 @@ async fn cleanup_remaining_fs_traces( ))? }); - rm(conf.tenant_path(tenant_id), true).await?; + rm(conf.tenant_path(tenant_shard_id), true).await?; Ok(()) } @@ -324,7 +321,7 @@ impl DeleteTenantFlow { // Though sounds scary, different mark name? // Detach currently uses remove_dir_all so in case of a crash we can end up in a weird state. if let Some(remote_storage) = &remote_storage { - create_remote_delete_mark(conf, remote_storage, &tenant.tenant_id) + create_remote_delete_mark(conf, remote_storage, &tenant.tenant_shard_id) .await .context("remote_mark")? } @@ -335,7 +332,7 @@ impl DeleteTenantFlow { ))? }); - create_local_delete_mark(conf, &tenant.tenant_id) + create_local_delete_mark(conf, &tenant.tenant_shard_id) .await .context("local delete mark")?; @@ -377,9 +374,11 @@ impl DeleteTenantFlow { return Ok(acquire(tenant)); } - let tenant_id = tenant.tenant_id; // Check local mark first, if its there there is no need to go to s3 to check whether remote one exists. - if conf.tenant_deleted_mark_file_path(&tenant_id).exists() { + if conf + .tenant_deleted_mark_file_path(&tenant.tenant_shard_id) + .exists() + { Ok(acquire(tenant)) } else { Ok(None) @@ -462,12 +461,12 @@ impl DeleteTenantFlow { tenants: &'static std::sync::RwLock, tenant: Arc, ) { - let tenant_id = tenant.tenant_id; + let tenant_shard_id = tenant.tenant_shard_id; task_mgr::spawn( task_mgr::BACKGROUND_RUNTIME.handle(), TaskKind::TimelineDeletionWorker, - Some(tenant_id), + Some(tenant_shard_id.tenant_id), None, "tenant_delete", false, @@ -481,7 +480,7 @@ impl DeleteTenantFlow { Ok(()) } .instrument({ - let span = tracing::info_span!(parent: None, "delete_tenant", tenant_id=%tenant_id); + let span = tracing::info_span!(parent: None, "delete_tenant", tenant_id=%tenant_shard_id.tenant_id, shard_id=%tenant_shard_id.shard_slug()); span.follows_from(Span::current()); span }), @@ -519,7 +518,7 @@ impl DeleteTenantFlow { } } - let timelines_path = conf.timelines_path(&tenant.tenant_id); + let timelines_path = conf.timelines_path(&tenant.tenant_shard_id); // May not exist if we fail in cleanup_remaining_fs_traces after removing it if timelines_path.exists() { // sanity check to guard against layout changes @@ -528,7 +527,8 @@ impl DeleteTenantFlow { .context("timelines dir not empty")?; } - remove_tenant_remote_delete_mark(conf, remote_storage.as_ref(), &tenant.tenant_id).await?; + remove_tenant_remote_delete_mark(conf, remote_storage.as_ref(), &tenant.tenant_shard_id) + .await?; fail::fail_point!("tenant-delete-before-cleanup-remaining-fs-traces", |_| { Err(anyhow::anyhow!( @@ -536,7 +536,7 @@ impl DeleteTenantFlow { ))? }); - cleanup_remaining_fs_traces(conf, &tenant.tenant_id) + cleanup_remaining_fs_traces(conf, &tenant.tenant_shard_id) .await .context("cleanup_remaining_fs_traces")?; @@ -553,7 +553,7 @@ impl DeleteTenantFlow { // we encounter an InProgress marker, yield the barrier it contains and wait on it. let barrier = { let mut locked = tenants.write().unwrap(); - let removed = locked.remove(&tenant.tenant_id); + let removed = locked.remove(&tenant.tenant_shard_id.tenant_id); // FIXME: we should not be modifying this from outside of mgr.rs. // This will go away when we simplify deletion (https://github.com/neondatabase/neon/issues/5080) diff --git a/pageserver/src/tenant/ephemeral_file.rs b/pageserver/src/tenant/ephemeral_file.rs index 9a06d9df61..591eacd104 100644 --- a/pageserver/src/tenant/ephemeral_file.rs +++ b/pageserver/src/tenant/ephemeral_file.rs @@ -7,18 +7,19 @@ use crate::page_cache::{self, PAGE_SZ}; use crate::tenant::block_io::{BlockCursor, BlockLease, BlockReader}; use crate::virtual_file::VirtualFile; use camino::Utf8PathBuf; +use pageserver_api::shard::TenantShardId; use std::cmp::min; use std::fs::OpenOptions; use std::io::{self, ErrorKind}; use std::ops::DerefMut; use std::sync::atomic::AtomicU64; use tracing::*; -use utils::id::{TenantId, TimelineId}; +use utils::id::TimelineId; pub struct EphemeralFile { page_cache_file_id: page_cache::FileId, - _tenant_id: TenantId, + _tenant_shard_id: TenantShardId, _timeline_id: TimelineId, file: VirtualFile, len: u64, @@ -31,7 +32,7 @@ pub struct EphemeralFile { impl EphemeralFile { pub async fn create( conf: &PageServerConf, - tenant_id: TenantId, + tenant_shard_id: TenantShardId, timeline_id: TimelineId, ) -> Result { static NEXT_FILENAME: AtomicU64 = AtomicU64::new(1); @@ -39,7 +40,7 @@ impl EphemeralFile { NEXT_FILENAME.fetch_add(1, std::sync::atomic::Ordering::Relaxed); let filename = conf - .timeline_path(&tenant_id, &timeline_id) + .timeline_path(&tenant_shard_id, &timeline_id) .join(Utf8PathBuf::from(format!( "ephemeral-{filename_disambiguator}" ))); @@ -52,7 +53,7 @@ impl EphemeralFile { Ok(EphemeralFile { page_cache_file_id: page_cache::next_file_id(), - _tenant_id: tenant_id, + _tenant_shard_id: tenant_shard_id, _timeline_id: timeline_id, file, len: 0, @@ -282,7 +283,7 @@ mod tests { ) -> Result< ( &'static PageServerConf, - TenantId, + TenantShardId, TimelineId, RequestContext, ), @@ -295,13 +296,13 @@ mod tests { // OK in a test. let conf: &'static PageServerConf = Box::leak(Box::new(conf)); - let tenant_id = TenantId::from_str("11000000000000000000000000000000").unwrap(); + let tenant_shard_id = TenantShardId::from_str("11000000000000000000000000000000").unwrap(); let timeline_id = TimelineId::from_str("22000000000000000000000000000000").unwrap(); - fs::create_dir_all(conf.timeline_path(&tenant_id, &timeline_id))?; + fs::create_dir_all(conf.timeline_path(&tenant_shard_id, &timeline_id))?; let ctx = RequestContext::new(TaskKind::UnitTest, DownloadBehavior::Error); - Ok((conf, tenant_id, timeline_id, ctx)) + Ok((conf, tenant_shard_id, timeline_id, ctx)) } #[tokio::test] diff --git a/pageserver/src/tenant/metadata.rs b/pageserver/src/tenant/metadata.rs index 38fd426746..6fb86c65e2 100644 --- a/pageserver/src/tenant/metadata.rs +++ b/pageserver/src/tenant/metadata.rs @@ -11,15 +11,12 @@ use std::io::{self}; use anyhow::{ensure, Context}; +use pageserver_api::shard::TenantShardId; use serde::{de::Error, Deserialize, Serialize, Serializer}; use thiserror::Error; use utils::bin_ser::SerializeError; use utils::crashsafe::path_with_suffix_extension; -use utils::{ - bin_ser::BeSer, - id::{TenantId, TimelineId}, - lsn::Lsn, -}; +use utils::{bin_ser::BeSer, id::TimelineId, lsn::Lsn}; use crate::config::PageServerConf; use crate::virtual_file::VirtualFile; @@ -272,14 +269,14 @@ impl Serialize for TimelineMetadata { } /// Save timeline metadata to file -#[tracing::instrument(skip_all, fields(%tenant_id, %timeline_id))] +#[tracing::instrument(skip_all, fields(%tenant_id=tenant_shard_id.tenant_id, %shard_id=tenant_shard_id.shard_slug(), %timeline_id))] pub async fn save_metadata( conf: &'static PageServerConf, - tenant_id: &TenantId, + tenant_shard_id: &TenantShardId, timeline_id: &TimelineId, data: &TimelineMetadata, ) -> anyhow::Result<()> { - let path = conf.metadata_path(tenant_id, timeline_id); + let path = conf.metadata_path(tenant_shard_id, timeline_id); let temp_path = path_with_suffix_extension(&path, TEMP_FILE_SUFFIX); let metadata_bytes = data.to_bytes().context("serialize metadata")?; VirtualFile::crashsafe_overwrite(&path, &temp_path, &metadata_bytes) @@ -299,10 +296,10 @@ pub enum LoadMetadataError { pub fn load_metadata( conf: &'static PageServerConf, - tenant_id: &TenantId, + tenant_shard_id: &TenantShardId, timeline_id: &TimelineId, ) -> Result { - let metadata_path = conf.metadata_path(tenant_id, timeline_id); + let metadata_path = conf.metadata_path(tenant_shard_id, timeline_id); let metadata_bytes = std::fs::read(metadata_path)?; Ok(TimelineMetadata::from_bytes(&metadata_bytes)?) diff --git a/pageserver/src/tenant/mgr.rs b/pageserver/src/tenant/mgr.rs index 52d697a878..e94d29327e 100644 --- a/pageserver/src/tenant/mgr.rs +++ b/pageserver/src/tenant/mgr.rs @@ -272,8 +272,8 @@ pub struct TenantManager { } fn emergency_generations( - tenant_confs: &HashMap>, -) -> HashMap { + tenant_confs: &HashMap>, +) -> HashMap { tenant_confs .iter() .filter_map(|(tid, lc)| { @@ -293,10 +293,10 @@ fn emergency_generations( async fn init_load_generations( conf: &'static PageServerConf, - tenant_confs: &HashMap>, + tenant_confs: &HashMap>, resources: &TenantSharedResources, cancel: &CancellationToken, -) -> anyhow::Result>> { +) -> anyhow::Result>> { let generations = if conf.control_plane_emergency_mode { error!( "Emergency mode! Tenants will be attached unsafely using their last known generation" @@ -339,7 +339,7 @@ async fn init_load_generations( fn load_tenant_config( conf: &'static PageServerConf, dentry: Utf8DirEntry, -) -> anyhow::Result)>> { +) -> anyhow::Result)>> { let tenant_dir_path = dentry.path().to_path_buf(); if crate::is_temporary(&tenant_dir_path) { info!("Found temporary tenant directory, removing: {tenant_dir_path}"); @@ -375,10 +375,10 @@ fn load_tenant_config( return Ok(None); } - let tenant_id = match tenant_dir_path + let tenant_shard_id = match tenant_dir_path .file_name() .unwrap_or_default() - .parse::() + .parse::() { Ok(id) => id, Err(_) => { @@ -388,8 +388,8 @@ fn load_tenant_config( }; Ok(Some(( - tenant_id, - Tenant::load_tenant_config(conf, &tenant_id), + tenant_shard_id, + Tenant::load_tenant_config(conf, &tenant_shard_id), ))) } @@ -400,7 +400,7 @@ fn load_tenant_config( /// seconds even on reasonably fast drives. async fn init_load_tenant_configs( conf: &'static PageServerConf, -) -> anyhow::Result>> { +) -> anyhow::Result>> { let tenants_dir = conf.tenants_path(); let dentries = tokio::task::spawn_blocking(move || -> anyhow::Result> { @@ -450,19 +450,19 @@ pub async fn init_tenant_mgr( init_load_generations(conf, &tenant_configs, &resources, &cancel).await?; // Construct `Tenant` objects and start them running - for (tenant_id, location_conf) in tenant_configs { - let tenant_dir_path = conf.tenant_path(&tenant_id); + for (tenant_shard_id, location_conf) in tenant_configs { + let tenant_dir_path = conf.tenant_path(&tenant_shard_id); let mut location_conf = match location_conf { Ok(l) => l, Err(e) => { - warn!(%tenant_id, "Marking tenant broken, failed to {e:#}"); + warn!(tenant_id=%tenant_shard_id.tenant_id, shard_id=%tenant_shard_id.shard_slug(), "Marking tenant broken, failed to {e:#}"); tenants.insert( - TenantShardId::unsharded(tenant_id), + tenant_shard_id, TenantSlot::Attached(Tenant::create_broken_tenant( conf, - tenant_id, + tenant_shard_id, format!("{}", e), )), ); @@ -473,7 +473,7 @@ pub async fn init_tenant_mgr( let generation = if let Some(generations) = &tenant_generations { // We have a generation map: treat it as the authority for whether // this tenant is really attached. - if let Some(gen) = generations.get(&tenant_id) { + if let Some(gen) = generations.get(&tenant_shard_id) { *gen } else { match &location_conf.mode { @@ -481,8 +481,8 @@ pub async fn init_tenant_mgr( // We do not require the control plane's permission for secondary mode // tenants, because they do no remote writes and hence require no // generation number - info!(%tenant_id, "Loaded tenant in secondary mode"); - tenants.insert(TenantShardId::unsharded(tenant_id), TenantSlot::Secondary); + info!(tenant_id=%tenant_shard_id.tenant_id, shard_id=%tenant_shard_id.shard_slug(), "Loaded tenant in secondary mode"); + tenants.insert(tenant_shard_id, TenantSlot::Secondary); } LocationMode::Attached(_) => { // TODO: augment re-attach API to enable the control plane to @@ -490,9 +490,9 @@ pub async fn init_tenant_mgr( // away local state, we can gracefully fall back to secondary here, if the control // plane tells us so. // (https://github.com/neondatabase/neon/issues/5377) - info!(%tenant_id, "Detaching tenant, control plane omitted it in re-attach response"); + info!(tenant_id=%tenant_shard_id.tenant_id, shard_id=%tenant_shard_id.shard_slug(), "Detaching tenant, control plane omitted it in re-attach response"); if let Err(e) = safe_remove_tenant_dir_all(&tenant_dir_path).await { - error!(%tenant_id, + error!(tenant_id=%tenant_shard_id.tenant_id, shard_id=%tenant_shard_id.shard_slug(), "Failed to remove detached tenant directory '{tenant_dir_path}': {e:?}", ); } @@ -504,18 +504,18 @@ pub async fn init_tenant_mgr( } else { // Legacy mode: no generation information, any tenant present // on local disk may activate - info!(%tenant_id, "Starting tenant in legacy mode, no generation",); + info!(tenant_id=%tenant_shard_id.tenant_id, shard_id=%tenant_shard_id.shard_slug(), "Starting tenant in legacy mode, no generation",); Generation::none() }; // Presence of a generation number implies attachment: attach the tenant // if it wasn't already, and apply the generation number. location_conf.attach_in_generation(generation); - Tenant::persist_tenant_config(conf, &tenant_id, &location_conf).await?; + Tenant::persist_tenant_config(conf, &tenant_shard_id, &location_conf).await?; match tenant_spawn( conf, - tenant_id, + tenant_shard_id, &tenant_dir_path, resources.clone(), AttachedTenantConf::try_from(location_conf)?, @@ -531,7 +531,7 @@ pub async fn init_tenant_mgr( ); } Err(e) => { - error!(%tenant_id, "Failed to start tenant: {e:#}"); + error!(tenant_id=%tenant_shard_id.tenant_id, shard_id=%tenant_shard_id.shard_slug(), "Failed to start tenant: {e:#}"); } } } @@ -555,7 +555,7 @@ pub async fn init_tenant_mgr( #[allow(clippy::too_many_arguments)] pub(crate) fn tenant_spawn( conf: &'static PageServerConf, - tenant_id: TenantId, + tenant_shard_id: TenantShardId, tenant_path: &Utf8Path, resources: TenantSharedResources, location_conf: AttachedTenantConf, @@ -579,16 +579,16 @@ pub(crate) fn tenant_spawn( "Cannot load tenant from empty directory {tenant_path:?}" ); - let tenant_ignore_mark = conf.tenant_ignore_mark_file_path(&tenant_id); + let tenant_ignore_mark = conf.tenant_ignore_mark_file_path(&tenant_shard_id); anyhow::ensure!( - !conf.tenant_ignore_mark_file_path(&tenant_id).exists(), + !conf.tenant_ignore_mark_file_path(&tenant_shard_id).exists(), "Cannot load tenant, ignore mark found at {tenant_ignore_mark:?}" ); - info!("Attaching tenant {tenant_id}"); + info!("Attaching tenant {tenant_shard_id}"); let tenant = match Tenant::spawn( conf, - tenant_id, + tenant_shard_id, resources, location_conf, init_order, @@ -598,8 +598,8 @@ pub(crate) fn tenant_spawn( ) { Ok(tenant) => tenant, Err(e) => { - error!("Failed to spawn tenant {tenant_id}, reason: {e:#}"); - Tenant::create_broken_tenant(conf, tenant_id, format!("{e:#}")) + error!("Failed to spawn tenant {tenant_shard_id}, reason: {e:#}"); + Tenant::create_broken_tenant(conf, tenant_shard_id, format!("{e:#}")) } }; @@ -757,13 +757,11 @@ pub(crate) async fn create_tenant( let slot_guard = tenant_map_acquire_slot(&tenant_shard_id, TenantSlotAcquireMode::MustNotExist)?; - // TODO(sharding): make local paths shard-aware - let tenant_path = - super::create_tenant_files(conf, &location_conf, &tenant_shard_id.tenant_id).await?; + let tenant_path = super::create_tenant_files(conf, &location_conf, &tenant_shard_id).await?; let created_tenant = tenant_spawn( conf, - tenant_shard_id.tenant_id, + tenant_shard_id, &tenant_path, resources, AttachedTenantConf::try_from(location_conf)?, @@ -803,8 +801,9 @@ pub(crate) async fn set_new_tenant_config( // API to use is the location_config/ endpoint, which lets the caller provide // the full LocationConf. let location_conf = LocationConf::attached_single(new_tenant_conf, tenant.generation); + let tenant_shard_id = TenantShardId::unsharded(tenant_id); - Tenant::persist_tenant_config(conf, &tenant_id, &location_conf) + Tenant::persist_tenant_config(conf, &tenant_shard_id, &location_conf) .await .map_err(SetNewTenantConfigError::Persist)?; tenant.set_new_tenant_config(new_tenant_conf); @@ -935,8 +934,7 @@ impl TenantManager { slot_guard.drop_old_value().expect("We just shut it down"); } - // TODO(sharding): make local paths sharding-aware - let tenant_path = self.conf.tenant_path(&tenant_shard_id.tenant_id); + let tenant_path = self.conf.tenant_path(&tenant_shard_id); let new_slot = match &new_location_config.mode { LocationMode::Secondary(_) => { @@ -946,20 +944,14 @@ impl TenantManager { .await .with_context(|| format!("Creating {tenant_path}"))?; - // TODO(sharding): make local paths sharding-aware - Tenant::persist_tenant_config( - self.conf, - &tenant_shard_id.tenant_id, - &new_location_config, - ) - .await - .map_err(SetNewTenantConfigError::Persist)?; + Tenant::persist_tenant_config(self.conf, &tenant_shard_id, &new_location_config) + .await + .map_err(SetNewTenantConfigError::Persist)?; TenantSlot::Secondary } LocationMode::Attached(_attach_config) => { - // TODO(sharding): make local paths sharding-aware - let timelines_path = self.conf.timelines_path(&tenant_shard_id.tenant_id); + let timelines_path = self.conf.timelines_path(&tenant_shard_id); // Directory doesn't need to be fsync'd because we do not depend on // it to exist after crashes: it may be recreated when tenant is @@ -968,19 +960,13 @@ impl TenantManager { .await .with_context(|| format!("Creating {timelines_path}"))?; - // TODO(sharding): make local paths sharding-aware - Tenant::persist_tenant_config( - self.conf, - &tenant_shard_id.tenant_id, - &new_location_config, - ) - .await - .map_err(SetNewTenantConfigError::Persist)?; + Tenant::persist_tenant_config(self.conf, &tenant_shard_id, &new_location_config) + .await + .map_err(SetNewTenantConfigError::Persist)?; - // TODO(sharding): make spawn sharding-aware let tenant = tenant_spawn( self.conf, - tenant_shard_id.tenant_id, + tenant_shard_id, &tenant_path, self.resources.clone(), AttachedTenantConf::try_from(new_location_config)?, @@ -1282,8 +1268,7 @@ async fn detach_tenant0( deletion_queue_client: &DeletionQueueClient, ) -> Result { let tenant_dir_rename_operation = |tenant_id_to_clean: TenantShardId| async move { - // TODO(sharding): make local path helpers shard-aware - let local_tenant_directory = conf.tenant_path(&tenant_id_to_clean.tenant_id); + let local_tenant_directory = conf.tenant_path(&tenant_id_to_clean); safe_rename_tenant_dir(&local_tenant_directory) .await .with_context(|| format!("local tenant directory {local_tenant_directory:?} rename")) @@ -1308,8 +1293,7 @@ async fn detach_tenant0( Err(TenantStateError::SlotError(TenantSlotError::NotFound(_))) ) { - // TODO(sharding): make local paths sharding-aware - let tenant_ignore_mark = conf.tenant_ignore_mark_file_path(&tenant_shard_id.tenant_id); + let tenant_ignore_mark = conf.tenant_ignore_mark_file_path(&tenant_shard_id); if tenant_ignore_mark.exists() { info!("Detaching an ignored tenant"); let tmp_path = tenant_dir_rename_operation(tenant_shard_id) @@ -1338,9 +1322,9 @@ pub(crate) async fn load_tenant( let slot_guard = tenant_map_acquire_slot(&tenant_shard_id, TenantSlotAcquireMode::MustNotExist)?; - let tenant_path = conf.tenant_path(&tenant_id); + let tenant_path = conf.tenant_path(&tenant_shard_id); - let tenant_ignore_mark = conf.tenant_ignore_mark_file_path(&tenant_id); + let tenant_ignore_mark = conf.tenant_ignore_mark_file_path(&tenant_shard_id); if tenant_ignore_mark.exists() { std::fs::remove_file(&tenant_ignore_mark).with_context(|| { format!( @@ -1356,14 +1340,14 @@ pub(crate) async fn load_tenant( }; let mut location_conf = - Tenant::load_tenant_config(conf, &tenant_id).map_err(TenantMapInsertError::Other)?; + Tenant::load_tenant_config(conf, &tenant_shard_id).map_err(TenantMapInsertError::Other)?; location_conf.attach_in_generation(generation); - Tenant::persist_tenant_config(conf, &tenant_id, &location_conf).await?; + Tenant::persist_tenant_config(conf, &tenant_shard_id, &location_conf).await?; let new_tenant = tenant_spawn( conf, - tenant_id, + tenant_shard_id, &tenant_path, resources, AttachedTenantConf::try_from(location_conf)?, @@ -1394,7 +1378,7 @@ async fn ignore_tenant0( let tenant_shard_id = TenantShardId::unsharded(tenant_id); remove_tenant_from_memory(tenants, tenant_shard_id, async { - let ignore_mark_file = conf.tenant_ignore_mark_file_path(&tenant_id); + let ignore_mark_file = conf.tenant_ignore_mark_file_path(&tenant_shard_id); fs::File::create(&ignore_mark_file) .await .context("Failed to create ignore mark file") @@ -1452,13 +1436,13 @@ pub(crate) async fn attach_tenant( let slot_guard = tenant_map_acquire_slot(&tenant_shard_id, TenantSlotAcquireMode::MustNotExist)?; let location_conf = LocationConf::attached_single(tenant_conf, generation); - let tenant_dir = create_tenant_files(conf, &location_conf, &tenant_id).await?; + let tenant_dir = create_tenant_files(conf, &location_conf, &tenant_shard_id).await?; // TODO: tenant directory remains on disk if we bail out from here on. // See https://github.com/neondatabase/neon/issues/4233 let attached_tenant = tenant_spawn( conf, - tenant_id, + tenant_shard_id, &tenant_dir, resources, AttachedTenantConf::try_from(location_conf)?, @@ -1974,6 +1958,9 @@ pub(crate) async fn immediate_gc( .with_context(|| format!("tenant {tenant_id}")) .map_err(|e| ApiError::NotFound(e.into()))?; + // TODO(sharding): make callers of this function shard-aware + let tenant_shard_id = TenantShardId::unsharded(tenant_id); + let gc_horizon = gc_req.gc_horizon.unwrap_or_else(|| tenant.get_gc_horizon()); // Use tenant's pitr setting let pitr = tenant.get_pitr_interval(); @@ -1995,7 +1982,7 @@ pub(crate) async fn immediate_gc( #[allow(unused_mut)] let mut result = tenant .gc_iteration(Some(timeline_id), gc_horizon, pitr, &cancel, &ctx) - .instrument(info_span!("manual_gc", %tenant_id, %timeline_id)) + .instrument(info_span!("manual_gc", tenant_id=%tenant_shard_id.tenant_id, shard_id=%tenant_shard_id.shard_slug(), %timeline_id)) .await; // FIXME: `gc_iteration` can return an error for multiple reasons; we should handle it // better once the types support it. diff --git a/pageserver/src/tenant/remote_timeline_client.rs b/pageserver/src/tenant/remote_timeline_client.rs index 01c60ca8f8..183ee19a40 100644 --- a/pageserver/src/tenant/remote_timeline_client.rs +++ b/pageserver/src/tenant/remote_timeline_client.rs @@ -188,7 +188,7 @@ use anyhow::Context; use camino::Utf8Path; use chrono::{NaiveDateTime, Utc}; -use pageserver_api::shard::ShardIndex; +use pageserver_api::shard::{ShardIndex, TenantShardId}; use scopeguard::ScopeGuard; use tokio_util::sync::CancellationToken; pub(crate) use upload::upload_initdb_dir; @@ -301,7 +301,7 @@ pub struct RemoteTimelineClient { runtime: tokio::runtime::Handle, - tenant_id: TenantId, + tenant_shard_id: TenantShardId, timeline_id: TimelineId, generation: Generation, @@ -325,7 +325,7 @@ impl RemoteTimelineClient { remote_storage: GenericRemoteStorage, deletion_queue_client: DeletionQueueClient, conf: &'static PageServerConf, - tenant_id: TenantId, + tenant_shard_id: TenantShardId, timeline_id: TimelineId, generation: Generation, ) -> RemoteTimelineClient { @@ -337,13 +337,16 @@ impl RemoteTimelineClient { } else { BACKGROUND_RUNTIME.handle().clone() }, - tenant_id, + tenant_shard_id, timeline_id, generation, storage_impl: remote_storage, deletion_queue_client, upload_queue: Mutex::new(UploadQueue::Uninitialized), - metrics: Arc::new(RemoteTimelineClientMetrics::new(&tenant_id, &timeline_id)), + metrics: Arc::new(RemoteTimelineClientMetrics::new( + &tenant_shard_id, + &timeline_id, + )), } } @@ -403,11 +406,6 @@ impl RemoteTimelineClient { Ok(()) } - pub(crate) fn get_shard_index(&self) -> ShardIndex { - // TODO: carry this on the struct - ShardIndex::unsharded() - } - pub fn remote_consistent_lsn_projected(&self) -> Option { match &mut *self.upload_queue.lock().unwrap() { UploadQueue::Uninitialized => None, @@ -469,14 +467,13 @@ impl RemoteTimelineClient { let index_part = download::download_index_part( &self.storage_impl, - &self.tenant_id, + &self.tenant_shard_id, &self.timeline_id, - self.get_shard_index(), self.generation, cancel, ) .measure_remote_op( - self.tenant_id, + self.tenant_shard_id.tenant_id, self.timeline_id, RemoteOpFileKind::Index, RemoteOpKind::Download, @@ -512,13 +509,13 @@ impl RemoteTimelineClient { download::download_layer_file( self.conf, &self.storage_impl, - self.tenant_id, + self.tenant_shard_id, self.timeline_id, layer_file_name, layer_metadata, ) .measure_remote_op( - self.tenant_id, + self.tenant_shard_id.tenant_id, self.timeline_id, RemoteOpFileKind::Layer, RemoteOpKind::Download, @@ -966,9 +963,8 @@ impl RemoteTimelineClient { || { upload::upload_index_part( &self.storage_impl, - &self.tenant_id, + &self.tenant_shard_id, &self.timeline_id, - self.get_shard_index(), self.generation, &index_part_with_deleted_at, ) @@ -1025,7 +1021,7 @@ impl RemoteTimelineClient { .drain() .map(|(file_name, meta)| { remote_layer_path( - &self.tenant_id, + &self.tenant_shard_id.tenant_id, &self.timeline_id, meta.shard, &file_name, @@ -1040,7 +1036,7 @@ impl RemoteTimelineClient { // Do not delete index part yet, it is needed for possible retry. If we remove it first // and retry will arrive to different pageserver there wont be any traces of it on remote storage - let timeline_storage_path = remote_timeline_path(&self.tenant_id, &self.timeline_id); + let timeline_storage_path = remote_timeline_path(&self.tenant_shard_id, &self.timeline_id); // Execute all pending deletions, so that when we proceed to do a list_prefixes below, we aren't // taking the burden of listing all the layers that we already know we should delete. @@ -1076,12 +1072,7 @@ impl RemoteTimelineClient { .unwrap_or( // No generation-suffixed indices, assume we are dealing with // a legacy index. - remote_index_path( - &self.tenant_id, - &self.timeline_id, - self.get_shard_index(), - Generation::none(), - ), + remote_index_path(&self.tenant_shard_id, &self.timeline_id, Generation::none()), ); let remaining_layers: Vec = remaining @@ -1213,12 +1204,12 @@ impl RemoteTimelineClient { // Spawn task to perform the task let self_rc = Arc::clone(self); - let tenant_id = self.tenant_id; + let tenant_shard_id = self.tenant_shard_id; let timeline_id = self.timeline_id; task_mgr::spawn( &self.runtime, TaskKind::RemoteUploadTask, - Some(self.tenant_id), + Some(self.tenant_shard_id.tenant_id), Some(self.timeline_id), "remote upload", false, @@ -1226,7 +1217,7 @@ impl RemoteTimelineClient { self_rc.perform_upload_task(task).await; Ok(()) } - .instrument(info_span!(parent: None, "remote_upload", %tenant_id, %timeline_id, %upload_task_id)), + .instrument(info_span!(parent: None, "remote_upload", tenant_id=%tenant_shard_id.tenant_id, shard_id=%tenant_shard_id.shard_slug(), %timeline_id, %upload_task_id)), ); // Loop back to process next task @@ -1278,7 +1269,7 @@ impl RemoteTimelineClient { self.generation, ) .measure_remote_op( - self.tenant_id, + self.tenant_shard_id.tenant_id, self.timeline_id, RemoteOpFileKind::Layer, RemoteOpKind::Upload, @@ -1298,14 +1289,13 @@ impl RemoteTimelineClient { let res = upload::upload_index_part( &self.storage_impl, - &self.tenant_id, + &self.tenant_shard_id, &self.timeline_id, - self.get_shard_index(), self.generation, index_part, ) .measure_remote_op( - self.tenant_id, + self.tenant_shard_id.tenant_id, self.timeline_id, RemoteOpFileKind::Index, RemoteOpKind::Upload, @@ -1325,7 +1315,7 @@ impl RemoteTimelineClient { pausable_failpoint!("before-delete-layer-pausable"); self.deletion_queue_client .push_layers( - self.tenant_id, + self.tenant_shard_id, self.timeline_id, self.generation, delete.layers.clone(), @@ -1444,7 +1434,7 @@ impl RemoteTimelineClient { // data safety guarantees (see docs/rfcs/025-generation-numbers.md) self.deletion_queue_client .update_remote_consistent_lsn( - self.tenant_id, + self.tenant_shard_id, self.timeline_id, self.generation, lsn, @@ -1602,15 +1592,21 @@ impl RemoteTimelineClient { } } -pub fn remote_timelines_path(tenant_id: &TenantId) -> RemotePath { - let path = format!("tenants/{tenant_id}/{TIMELINES_SEGMENT_NAME}"); +pub fn remote_timelines_path(tenant_shard_id: &TenantShardId) -> RemotePath { + let path = format!("tenants/{tenant_shard_id}/{TIMELINES_SEGMENT_NAME}"); RemotePath::from_string(&path).expect("Failed to construct path") } -pub fn remote_timeline_path(tenant_id: &TenantId, timeline_id: &TimelineId) -> RemotePath { - remote_timelines_path(tenant_id).join(Utf8Path::new(&timeline_id.to_string())) +pub fn remote_timeline_path( + tenant_shard_id: &TenantShardId, + timeline_id: &TimelineId, +) -> RemotePath { + remote_timelines_path(tenant_shard_id).join(Utf8Path::new(&timeline_id.to_string())) } +/// Note that the shard component of a remote layer path is _not_ always the same +/// as in the TenantShardId of the caller: tenants may reference layers from a different +/// ShardIndex. Use the ShardIndex from the layer's metadata. pub fn remote_layer_path( tenant_id: &TenantId, timeline_id: &TimelineId, @@ -1637,14 +1633,12 @@ pub fn remote_initdb_archive_path(tenant_id: &TenantId, timeline_id: &TimelineId } pub fn remote_index_path( - tenant_id: &TenantId, + tenant_shard_id: &TenantShardId, timeline_id: &TimelineId, - shard: ShardIndex, generation: Generation, ) -> RemotePath { RemotePath::from_string(&format!( - "tenants/{tenant_id}{0}/{TIMELINES_SEGMENT_NAME}/{timeline_id}/{1}{2}", - shard.get_suffix(), + "tenants/{tenant_shard_id}/{TIMELINES_SEGMENT_NAME}/{timeline_id}/{0}{1}", IndexPart::FILE_NAME, generation.get_suffix() )) @@ -1786,14 +1780,14 @@ mod tests { Arc::new(RemoteTimelineClient { conf: self.harness.conf, runtime: tokio::runtime::Handle::current(), - tenant_id: self.harness.tenant_id, + tenant_shard_id: self.harness.tenant_shard_id, timeline_id: TIMELINE_ID, generation, storage_impl: self.harness.remote_storage.clone(), deletion_queue_client: self.harness.deletion_queue.new_client(), upload_queue: Mutex::new(UploadQueue::Uninitialized), metrics: Arc::new(RemoteTimelineClientMetrics::new( - &self.harness.tenant_id, + &self.harness.tenant_shard_id, &TIMELINE_ID, )), }) @@ -2100,11 +2094,7 @@ mod tests { assert_eq!(actual_c, expected_c); } - async fn inject_index_part( - test_state: &TestSetup, - generation: Generation, - shard: ShardIndex, - ) -> IndexPart { + async fn inject_index_part(test_state: &TestSetup, generation: Generation) -> IndexPart { // An empty IndexPart, just sufficient to ensure deserialization will succeed let example_metadata = TimelineMetadata::example(); let example_index_part = IndexPart::new( @@ -2126,9 +2116,8 @@ mod tests { let index_path = test_state.harness.remote_fs_dir.join( remote_index_path( - &test_state.harness.tenant_id, + &test_state.harness.tenant_shard_id, &TIMELINE_ID, - shard, generation, ) .get_path(), @@ -2168,12 +2157,7 @@ mod tests { // Simple case: we are in generation N, load the index from generation N - 1 let generation_n = 5; - let injected = inject_index_part( - &test_state, - Generation::new(generation_n - 1), - ShardIndex::unsharded(), - ) - .await; + let injected = inject_index_part(&test_state, Generation::new(generation_n - 1)).await; assert_got_index_part(&test_state, Generation::new(generation_n), &injected).await; @@ -2191,34 +2175,22 @@ mod tests { // A generation-less IndexPart exists in the bucket, we should find it let generation_n = 5; - let injected_none = - inject_index_part(&test_state, Generation::none(), ShardIndex::unsharded()).await; + let injected_none = inject_index_part(&test_state, Generation::none()).await; assert_got_index_part(&test_state, Generation::new(generation_n), &injected_none).await; // If a more recent-than-none generation exists, we should prefer to load that - let injected_1 = - inject_index_part(&test_state, Generation::new(1), ShardIndex::unsharded()).await; + let injected_1 = inject_index_part(&test_state, Generation::new(1)).await; assert_got_index_part(&test_state, Generation::new(generation_n), &injected_1).await; // If a more-recent-than-me generation exists, we should ignore it. - let _injected_10 = - inject_index_part(&test_state, Generation::new(10), ShardIndex::unsharded()).await; + let _injected_10 = inject_index_part(&test_state, Generation::new(10)).await; assert_got_index_part(&test_state, Generation::new(generation_n), &injected_1).await; // If a directly previous generation exists, _and_ an index exists in my own // generation, I should prefer my own generation. - let _injected_prev = inject_index_part( - &test_state, - Generation::new(generation_n - 1), - ShardIndex::unsharded(), - ) - .await; - let injected_current = inject_index_part( - &test_state, - Generation::new(generation_n), - ShardIndex::unsharded(), - ) - .await; + let _injected_prev = + inject_index_part(&test_state, Generation::new(generation_n - 1)).await; + let injected_current = inject_index_part(&test_state, Generation::new(generation_n)).await; assert_got_index_part( &test_state, Generation::new(generation_n), diff --git a/pageserver/src/tenant/remote_timeline_client/download.rs b/pageserver/src/tenant/remote_timeline_client/download.rs index 3b2cb5b599..1e9dcfe76a 100644 --- a/pageserver/src/tenant/remote_timeline_client/download.rs +++ b/pageserver/src/tenant/remote_timeline_client/download.rs @@ -9,7 +9,7 @@ use std::time::Duration; use anyhow::{anyhow, Context}; use camino::Utf8Path; -use pageserver_api::shard::ShardIndex; +use pageserver_api::shard::TenantShardId; use tokio::fs; use tokio::io::AsyncWriteExt; use tokio_util::sync::CancellationToken; @@ -22,7 +22,7 @@ use crate::tenant::timeline::span::debug_assert_current_span_has_tenant_and_time use crate::tenant::Generation; use remote_storage::{DownloadError, GenericRemoteStorage, ListingMode}; use utils::crashsafe::path_with_suffix_extension; -use utils::id::{TenantId, TimelineId}; +use utils::id::TimelineId; use super::index::{IndexPart, LayerFileMetadata}; use super::{ @@ -40,7 +40,7 @@ static MAX_DOWNLOAD_DURATION: Duration = Duration::from_secs(120); pub async fn download_layer_file<'a>( conf: &'static PageServerConf, storage: &'a GenericRemoteStorage, - tenant_id: TenantId, + tenant_shard_id: TenantShardId, timeline_id: TimelineId, layer_file_name: &'a LayerFileName, layer_metadata: &'a LayerFileMetadata, @@ -48,11 +48,11 @@ pub async fn download_layer_file<'a>( debug_assert_current_span_has_tenant_and_timeline_id(); let local_path = conf - .timeline_path(&tenant_id, &timeline_id) + .timeline_path(&tenant_shard_id, &timeline_id) .join(layer_file_name.file_name()); let remote_path = remote_layer_path( - &tenant_id, + &tenant_shard_id.tenant_id, &timeline_id, layer_metadata.shard, layer_file_name, @@ -171,10 +171,10 @@ pub fn is_temp_download_file(path: &Utf8Path) -> bool { /// List timelines of given tenant in remote storage pub async fn list_remote_timelines( storage: &GenericRemoteStorage, - tenant_id: TenantId, + tenant_shard_id: TenantShardId, cancel: CancellationToken, ) -> anyhow::Result<(HashSet, HashSet)> { - let remote_path = remote_timelines_path(&tenant_id); + let remote_path = remote_timelines_path(&tenant_shard_id); fail::fail_point!("storage-sync-list-remote-timelines", |_| { anyhow::bail!("storage-sync-list-remote-timelines"); @@ -182,7 +182,7 @@ pub async fn list_remote_timelines( let listing = download_retry_forever( || storage.list(Some(&remote_path), ListingMode::WithDelimiter), - &format!("list timelines for {tenant_id}"), + &format!("list timelines for {tenant_shard_id}"), cancel, ) .await?; @@ -192,7 +192,7 @@ pub async fn list_remote_timelines( for timeline_remote_storage_key in listing.prefixes { let object_name = timeline_remote_storage_key.object_name().ok_or_else(|| { - anyhow::anyhow!("failed to get timeline id for remote tenant {tenant_id}") + anyhow::anyhow!("failed to get timeline id for remote tenant {tenant_shard_id}") })?; match object_name.parse::() { @@ -213,13 +213,12 @@ pub async fn list_remote_timelines( async fn do_download_index_part( storage: &GenericRemoteStorage, - tenant_id: &TenantId, + tenant_shard_id: &TenantShardId, timeline_id: &TimelineId, - shard: ShardIndex, index_generation: Generation, cancel: CancellationToken, ) -> Result { - let remote_path = remote_index_path(tenant_id, timeline_id, shard, index_generation); + let remote_path = remote_index_path(tenant_shard_id, timeline_id, index_generation); let index_part_bytes = download_retry_forever( || async { @@ -255,9 +254,8 @@ async fn do_download_index_part( #[tracing::instrument(skip_all, fields(generation=?my_generation))] pub(super) async fn download_index_part( storage: &GenericRemoteStorage, - tenant_id: &TenantId, + tenant_shard_id: &TenantShardId, timeline_id: &TimelineId, - shard: ShardIndex, my_generation: Generation, cancel: CancellationToken, ) -> Result { @@ -267,9 +265,8 @@ pub(super) async fn download_index_part( // Operating without generations: just fetch the generation-less path return do_download_index_part( storage, - tenant_id, + tenant_shard_id, timeline_id, - shard, my_generation, cancel, ) @@ -282,9 +279,8 @@ pub(super) async fn download_index_part( // This is an optimization to avoid doing the listing for the general case below. let res = do_download_index_part( storage, - tenant_id, + tenant_shard_id, timeline_id, - shard, my_generation, cancel.clone(), ) @@ -310,9 +306,8 @@ pub(super) async fn download_index_part( // This is an optimization to avoid doing the listing for the general case below. let res = do_download_index_part( storage, - tenant_id, + tenant_shard_id, timeline_id, - shard, my_generation.previous(), cancel.clone(), ) @@ -335,7 +330,7 @@ pub(super) async fn download_index_part( // General case/fallback: if there is no index at my_generation or prev_generation, then list all index_part.json // objects, and select the highest one with a generation <= my_generation. Constructing the prefix is equivalent // to constructing a full index path with no generation, because the generation is a suffix. - let index_prefix = remote_index_path(tenant_id, timeline_id, shard, Generation::none()); + let index_prefix = remote_index_path(tenant_shard_id, timeline_id, Generation::none()); let indices = backoff::retry( || async { storage.list_files(Some(&index_prefix)).await }, |_| false, @@ -361,7 +356,7 @@ pub(super) async fn download_index_part( match max_previous_generation { Some(g) => { tracing::debug!("Found index_part in generation {g:?}"); - do_download_index_part(storage, tenant_id, timeline_id, shard, g, cancel).await + do_download_index_part(storage, tenant_shard_id, timeline_id, g, cancel).await } None => { // Migration from legacy pre-generation state: we have a generation but no prior @@ -369,9 +364,8 @@ pub(super) async fn download_index_part( tracing::info!("No index_part.json* found"); do_download_index_part( storage, - tenant_id, + tenant_shard_id, timeline_id, - shard, Generation::none(), cancel, ) diff --git a/pageserver/src/tenant/remote_timeline_client/upload.rs b/pageserver/src/tenant/remote_timeline_client/upload.rs index 789a10cf54..4ca4438003 100644 --- a/pageserver/src/tenant/remote_timeline_client/upload.rs +++ b/pageserver/src/tenant/remote_timeline_client/upload.rs @@ -4,7 +4,7 @@ use anyhow::{bail, Context}; use bytes::Bytes; use camino::Utf8Path; use fail::fail_point; -use pageserver_api::shard::ShardIndex; +use pageserver_api::shard::TenantShardId; use std::io::ErrorKind; use tokio::fs; @@ -25,9 +25,8 @@ use tracing::info; /// Serializes and uploads the given index part data to the remote storage. pub(super) async fn upload_index_part<'a>( storage: &'a GenericRemoteStorage, - tenant_id: &TenantId, + tenant_shard_id: &TenantShardId, timeline_id: &TimelineId, - shard: ShardIndex, generation: Generation, index_part: &'a IndexPart, ) -> anyhow::Result<()> { @@ -44,11 +43,11 @@ pub(super) async fn upload_index_part<'a>( let index_part_size = index_part_bytes.len(); let index_part_bytes = tokio::io::BufReader::new(std::io::Cursor::new(index_part_bytes)); - let remote_path = remote_index_path(tenant_id, timeline_id, shard, generation); + let remote_path = remote_index_path(tenant_shard_id, timeline_id, generation); storage .upload_storage_object(Box::new(index_part_bytes), index_part_size, &remote_path) .await - .with_context(|| format!("upload index part for '{tenant_id} / {timeline_id}'")) + .with_context(|| format!("upload index part for '{tenant_shard_id} / {timeline_id}'")) } /// Attempts to upload given layer files. diff --git a/pageserver/src/tenant/storage_layer.rs b/pageserver/src/tenant/storage_layer.rs index 3b2a61dcba..c933342822 100644 --- a/pageserver/src/tenant/storage_layer.rs +++ b/pageserver/src/tenant/storage_layer.rs @@ -24,10 +24,7 @@ use tracing::warn; use utils::history_buffer::HistoryBufferWithDropCounter; use utils::rate_limit::RateLimit; -use utils::{ - id::{TenantId, TimelineId}, - lsn::Lsn, -}; +use utils::{id::TimelineId, lsn::Lsn}; pub use delta_layer::{DeltaLayer, DeltaLayerWriter, ValueRef}; pub use filename::{DeltaFileName, ImageFileName, LayerFileName}; @@ -304,12 +301,14 @@ pub trait AsLayerDesc { } pub mod tests { + use pageserver_api::shard::TenantShardId; + use super::*; impl From for PersistentLayerDesc { fn from(value: DeltaFileName) -> Self { PersistentLayerDesc::new_delta( - TenantId::from_array([0; 16]), + TenantShardId::from([0; 18]), TimelineId::from_array([0; 16]), value.key_range, value.lsn_range, @@ -321,7 +320,7 @@ pub mod tests { impl From for PersistentLayerDesc { fn from(value: ImageFileName) -> Self { PersistentLayerDesc::new_img( - TenantId::from_array([0; 16]), + TenantShardId::from([0; 18]), TimelineId::from_array([0; 16]), value.key_range, value.lsn, diff --git a/pageserver/src/tenant/storage_layer/delta_layer.rs b/pageserver/src/tenant/storage_layer/delta_layer.rs index 79f37dcb2d..e9886d90c4 100644 --- a/pageserver/src/tenant/storage_layer/delta_layer.rs +++ b/pageserver/src/tenant/storage_layer/delta_layer.rs @@ -42,6 +42,7 @@ use crate::{DELTA_FILE_MAGIC, STORAGE_FORMAT_VERSION}; use anyhow::{bail, ensure, Context, Result}; use camino::{Utf8Path, Utf8PathBuf}; use pageserver_api::models::LayerAccessKind; +use pageserver_api::shard::TenantShardId; use rand::{distributions::Alphanumeric, Rng}; use serde::{Deserialize, Serialize}; use std::fs::File; @@ -86,7 +87,7 @@ pub struct Summary { impl From<&DeltaLayer> for Summary { fn from(layer: &DeltaLayer) -> Self { Self::expected( - layer.desc.tenant_id, + layer.desc.tenant_shard_id.tenant_id, layer.desc.timeline_id, layer.desc.key_range.clone(), layer.desc.lsn_range.clone(), @@ -248,7 +249,7 @@ impl DeltaLayer { fn temp_path_for( conf: &PageServerConf, - tenant_id: &TenantId, + tenant_shard_id: &TenantShardId, timeline_id: &TimelineId, key_start: Key, lsn_range: &Range, @@ -259,14 +260,15 @@ impl DeltaLayer { .map(char::from) .collect(); - conf.timeline_path(tenant_id, timeline_id).join(format!( - "{}-XXX__{:016X}-{:016X}.{}.{}", - key_start, - u64::from(lsn_range.start), - u64::from(lsn_range.end), - rand_string, - TEMP_FILE_SUFFIX, - )) + conf.timeline_path(tenant_shard_id, timeline_id) + .join(format!( + "{}-XXX__{:016X}-{:016X}.{}.{}", + key_start, + u64::from(lsn_range.start), + u64::from(lsn_range.end), + rand_string, + TEMP_FILE_SUFFIX, + )) } /// @@ -318,10 +320,14 @@ impl DeltaLayer { .metadata() .context("get file metadata to determine size")?; + // TODO(sharding): we must get the TenantShardId from the path instead of reading the Summary. + // we should also validate the path against the Summary, as both should contain the same tenant, timeline, key, lsn. + let tenant_shard_id = TenantShardId::unsharded(summary.tenant_id); + Ok(DeltaLayer { path: path.to_path_buf(), desc: PersistentLayerDesc::new_delta( - summary.tenant_id, + tenant_shard_id, summary.timeline_id, summary.key_range, summary.lsn_range, @@ -353,7 +359,7 @@ struct DeltaLayerWriterInner { conf: &'static PageServerConf, pub path: Utf8PathBuf, timeline_id: TimelineId, - tenant_id: TenantId, + tenant_shard_id: TenantShardId, key_start: Key, lsn_range: Range, @@ -370,7 +376,7 @@ impl DeltaLayerWriterInner { async fn new( conf: &'static PageServerConf, timeline_id: TimelineId, - tenant_id: TenantId, + tenant_shard_id: TenantShardId, key_start: Key, lsn_range: Range, ) -> anyhow::Result { @@ -380,7 +386,8 @@ impl DeltaLayerWriterInner { // // Note: This overwrites any existing file. There shouldn't be any. // FIXME: throw an error instead? - let path = DeltaLayer::temp_path_for(conf, &tenant_id, &timeline_id, key_start, &lsn_range); + let path = + DeltaLayer::temp_path_for(conf, &tenant_shard_id, &timeline_id, key_start, &lsn_range); let mut file = VirtualFile::create(&path).await?; // make room for the header block @@ -395,7 +402,7 @@ impl DeltaLayerWriterInner { conf, path, timeline_id, - tenant_id, + tenant_shard_id, key_start, lsn_range, tree: tree_builder, @@ -457,7 +464,7 @@ impl DeltaLayerWriterInner { let summary = Summary { magic: DELTA_FILE_MAGIC, format_version: STORAGE_FORMAT_VERSION, - tenant_id: self.tenant_id, + tenant_id: self.tenant_shard_id.tenant_id, timeline_id: self.timeline_id, key_range: self.key_start..key_end, lsn_range: self.lsn_range.clone(), @@ -498,7 +505,7 @@ impl DeltaLayerWriterInner { // set inner.file here. The first read will have to re-open it. let desc = PersistentLayerDesc::new_delta( - self.tenant_id, + self.tenant_shard_id, self.timeline_id, self.key_start..key_end, self.lsn_range.clone(), @@ -549,14 +556,20 @@ impl DeltaLayerWriter { pub async fn new( conf: &'static PageServerConf, timeline_id: TimelineId, - tenant_id: TenantId, + tenant_shard_id: TenantShardId, key_start: Key, lsn_range: Range, ) -> anyhow::Result { Ok(Self { inner: Some( - DeltaLayerWriterInner::new(conf, timeline_id, tenant_id, key_start, lsn_range) - .await?, + DeltaLayerWriterInner::new( + conf, + timeline_id, + tenant_shard_id, + key_start, + lsn_range, + ) + .await?, ), }) } diff --git a/pageserver/src/tenant/storage_layer/image_layer.rs b/pageserver/src/tenant/storage_layer/image_layer.rs index c38a9f6883..208aa07872 100644 --- a/pageserver/src/tenant/storage_layer/image_layer.rs +++ b/pageserver/src/tenant/storage_layer/image_layer.rs @@ -41,6 +41,7 @@ use bytes::Bytes; use camino::{Utf8Path, Utf8PathBuf}; use hex; use pageserver_api::models::LayerAccessKind; +use pageserver_api::shard::TenantShardId; use rand::{distributions::Alphanumeric, Rng}; use serde::{Deserialize, Serialize}; use std::fs::File; @@ -87,7 +88,7 @@ pub(super) struct Summary { impl From<&ImageLayer> for Summary { fn from(layer: &ImageLayer) -> Self { Self::expected( - layer.desc.tenant_id, + layer.desc.tenant_shard_id.tenant_id, layer.desc.timeline_id, layer.desc.key_range.clone(), layer.lsn, @@ -217,7 +218,7 @@ impl ImageLayer { fn temp_path_for( conf: &PageServerConf, timeline_id: TimelineId, - tenant_id: TenantId, + tenant_shard_id: TenantShardId, fname: &ImageFileName, ) -> Utf8PathBuf { let rand_string: String = rand::thread_rng() @@ -226,7 +227,7 @@ impl ImageLayer { .map(char::from) .collect(); - conf.timeline_path(&tenant_id, &timeline_id) + conf.timeline_path(&tenant_shard_id, &timeline_id) .join(format!("{fname}.{rand_string}.{TEMP_FILE_SUFFIX}")) } @@ -276,10 +277,15 @@ impl ImageLayer { let metadata = file .metadata() .context("get file metadata to determine size")?; + + // TODO(sharding): we should get TenantShardId from path. + // OR, not at all: any layer we load from disk should also get reconciled with remote IndexPart. + let tenant_shard_id = TenantShardId::unsharded(summary.tenant_id); + Ok(ImageLayer { path: path.to_path_buf(), desc: PersistentLayerDesc::new_img( - summary.tenant_id, + tenant_shard_id, summary.timeline_id, summary.key_range, summary.lsn, @@ -400,7 +406,7 @@ struct ImageLayerWriterInner { conf: &'static PageServerConf, path: Utf8PathBuf, timeline_id: TimelineId, - tenant_id: TenantId, + tenant_shard_id: TenantShardId, key_range: Range, lsn: Lsn, @@ -415,7 +421,7 @@ impl ImageLayerWriterInner { async fn new( conf: &'static PageServerConf, timeline_id: TimelineId, - tenant_id: TenantId, + tenant_shard_id: TenantShardId, key_range: &Range, lsn: Lsn, ) -> anyhow::Result { @@ -424,7 +430,7 @@ impl ImageLayerWriterInner { let path = ImageLayer::temp_path_for( conf, timeline_id, - tenant_id, + tenant_shard_id, &ImageFileName { key_range: key_range.clone(), lsn, @@ -448,7 +454,7 @@ impl ImageLayerWriterInner { conf, path, timeline_id, - tenant_id, + tenant_shard_id, key_range: key_range.clone(), lsn, tree: tree_builder, @@ -495,7 +501,7 @@ impl ImageLayerWriterInner { let summary = Summary { magic: IMAGE_FILE_MAGIC, format_version: STORAGE_FORMAT_VERSION, - tenant_id: self.tenant_id, + tenant_id: self.tenant_shard_id.tenant_id, timeline_id: self.timeline_id, key_range: self.key_range.clone(), lsn: self.lsn, @@ -521,7 +527,7 @@ impl ImageLayerWriterInner { .context("get metadata to determine file size")?; let desc = PersistentLayerDesc::new_img( - self.tenant_id, + self.tenant_shard_id, self.timeline_id, self.key_range.clone(), self.lsn, @@ -577,13 +583,14 @@ impl ImageLayerWriter { pub async fn new( conf: &'static PageServerConf, timeline_id: TimelineId, - tenant_id: TenantId, + tenant_shard_id: TenantShardId, key_range: &Range, lsn: Lsn, ) -> anyhow::Result { Ok(Self { inner: Some( - ImageLayerWriterInner::new(conf, timeline_id, tenant_id, key_range, lsn).await?, + ImageLayerWriterInner::new(conf, timeline_id, tenant_shard_id, key_range, lsn) + .await?, ), }) } diff --git a/pageserver/src/tenant/storage_layer/inmemory_layer.rs b/pageserver/src/tenant/storage_layer/inmemory_layer.rs index 2cb1e55b26..003cf0e92b 100644 --- a/pageserver/src/tenant/storage_layer/inmemory_layer.rs +++ b/pageserver/src/tenant/storage_layer/inmemory_layer.rs @@ -14,15 +14,11 @@ use crate::tenant::Timeline; use crate::walrecord; use anyhow::{ensure, Result}; use pageserver_api::models::InMemoryLayerInfo; +use pageserver_api::shard::TenantShardId; use std::collections::HashMap; use std::sync::{Arc, OnceLock}; use tracing::*; -use utils::{ - bin_ser::BeSer, - id::{TenantId, TimelineId}, - lsn::Lsn, - vec_map::VecMap, -}; +use utils::{bin_ser::BeSer, id::TimelineId, lsn::Lsn, vec_map::VecMap}; // avoid binding to Write (conflicts with std::io::Write) // while being able to use std::fmt::Write's methods use std::fmt::Write as _; @@ -33,7 +29,7 @@ use super::{DeltaLayerWriter, ResidentLayer}; pub struct InMemoryLayer { conf: &'static PageServerConf, - tenant_id: TenantId, + tenant_shard_id: TenantShardId, timeline_id: TimelineId, /// This layer contains all the changes from 'start_lsn'. The @@ -226,17 +222,17 @@ impl InMemoryLayer { pub async fn create( conf: &'static PageServerConf, timeline_id: TimelineId, - tenant_id: TenantId, + tenant_shard_id: TenantShardId, start_lsn: Lsn, ) -> Result { trace!("initializing new empty InMemoryLayer for writing on timeline {timeline_id} at {start_lsn}"); - let file = EphemeralFile::create(conf, tenant_id, timeline_id).await?; + let file = EphemeralFile::create(conf, tenant_shard_id, timeline_id).await?; Ok(InMemoryLayer { conf, timeline_id, - tenant_id, + tenant_shard_id, start_lsn, end_lsn: OnceLock::new(), inner: RwLock::new(InMemoryLayerInner { @@ -335,7 +331,7 @@ impl InMemoryLayer { let mut delta_layer_writer = DeltaLayerWriter::new( self.conf, self.timeline_id, - self.tenant_id, + self.tenant_shard_id, Key::MIN, self.start_lsn..end_lsn, ) diff --git a/pageserver/src/tenant/storage_layer/layer.rs b/pageserver/src/tenant/storage_layer/layer.rs index c27c3e69ed..3ed4e05bea 100644 --- a/pageserver/src/tenant/storage_layer/layer.rs +++ b/pageserver/src/tenant/storage_layer/layer.rs @@ -82,7 +82,7 @@ impl Layer { metadata: LayerFileMetadata, ) -> Self { let desc = PersistentLayerDesc::from_filename( - timeline.tenant_id, + timeline.tenant_shard_id, timeline.timeline_id, file_name, metadata.file_size(), @@ -113,7 +113,7 @@ impl Layer { metadata: LayerFileMetadata, ) -> ResidentLayer { let desc = PersistentLayerDesc::from_filename( - timeline.tenant_id, + timeline.tenant_shard_id, timeline.timeline_id, file_name, metadata.file_size(), @@ -486,7 +486,7 @@ impl Drop for LayerInner { return; } - let span = tracing::info_span!(parent: None, "layer_gc", tenant_id = %self.layer_desc().tenant_id, timeline_id = %self.layer_desc().timeline_id); + let span = tracing::info_span!(parent: None, "layer_gc", tenant_id = %self.layer_desc().tenant_shard_id.tenant_id, shard_id=%self.layer_desc().tenant_shard_id.shard_slug(), timeline_id = %self.layer_desc().timeline_id); let path = std::mem::take(&mut self.path); let file_name = self.layer_desc().filename(); @@ -561,7 +561,7 @@ impl LayerInner { shard: ShardIndex, ) -> Self { let path = conf - .timeline_path(&timeline.tenant_id, &timeline.timeline_id) + .timeline_path(&timeline.tenant_shard_id, &timeline.timeline_id) .join(desc.filename().to_string()); let (inner, version) = if let Some(inner) = downloaded { @@ -832,7 +832,7 @@ impl LayerInner { crate::task_mgr::spawn( &tokio::runtime::Handle::current(), crate::task_mgr::TaskKind::RemoteDownloadTask, - Some(self.desc.tenant_id), + Some(self.desc.tenant_shard_id.tenant_id), Some(self.desc.timeline_id), &task_name, false, @@ -997,7 +997,7 @@ impl LayerInner { if gc { // do nothing now, only in LayerInner::drop } else if can_evict && evict { - let span = tracing::info_span!(parent: None, "layer_evict", tenant_id = %self.desc.tenant_id, timeline_id = %self.desc.timeline_id, layer=%self, %version); + let span = tracing::info_span!(parent: None, "layer_evict", tenant_id = %self.desc.tenant_shard_id.tenant_id, shard_id = %self.desc.tenant_shard_id.shard_slug(), timeline_id = %self.desc.timeline_id, layer=%self, %version); // downgrade for queueing, in case there's a tear down already ongoing we should not // hold it alive. @@ -1229,7 +1229,7 @@ impl DownloadedLayer { let res = if owner.desc.is_delta { let summary = Some(delta_layer::Summary::expected( - owner.desc.tenant_id, + owner.desc.tenant_shard_id.tenant_id, owner.desc.timeline_id, owner.desc.key_range.clone(), owner.desc.lsn_range.clone(), @@ -1240,7 +1240,7 @@ impl DownloadedLayer { } else { let lsn = owner.desc.image_layer_lsn(); let summary = Some(image_layer::Summary::expected( - owner.desc.tenant_id, + owner.desc.tenant_shard_id.tenant_id, owner.desc.timeline_id, owner.desc.key_range.clone(), lsn, diff --git a/pageserver/src/tenant/storage_layer/layer_desc.rs b/pageserver/src/tenant/storage_layer/layer_desc.rs index 2e0b0b3e64..bf24407fc5 100644 --- a/pageserver/src/tenant/storage_layer/layer_desc.rs +++ b/pageserver/src/tenant/storage_layer/layer_desc.rs @@ -1,9 +1,7 @@ use core::fmt::Display; +use pageserver_api::shard::TenantShardId; use std::ops::Range; -use utils::{ - id::{TenantId, TimelineId}, - lsn::Lsn, -}; +use utils::{id::TimelineId, lsn::Lsn}; use crate::repository::Key; @@ -11,12 +9,15 @@ use super::{DeltaFileName, ImageFileName, LayerFileName}; use serde::{Deserialize, Serialize}; +#[cfg(test)] +use utils::id::TenantId; + /// A unique identifier of a persistent layer. This is different from `LayerDescriptor`, which is only used in the /// benchmarks. This struct contains all necessary information to find the image / delta layer. It also provides /// a unified way to generate layer information like file name. #[derive(Debug, PartialEq, Eq, Clone, Serialize, Deserialize)] pub struct PersistentLayerDesc { - pub tenant_id: TenantId, + pub tenant_shard_id: TenantShardId, pub timeline_id: TimelineId, /// Range of keys that this layer covers pub key_range: Range, @@ -56,7 +57,7 @@ impl PersistentLayerDesc { #[cfg(test)] pub fn new_test(key_range: Range) -> Self { Self { - tenant_id: TenantId::generate(), + tenant_shard_id: TenantShardId::unsharded(TenantId::generate()), timeline_id: TimelineId::generate(), key_range, lsn_range: Lsn(0)..Lsn(1), @@ -66,14 +67,14 @@ impl PersistentLayerDesc { } pub fn new_img( - tenant_id: TenantId, + tenant_shard_id: TenantShardId, timeline_id: TimelineId, key_range: Range, lsn: Lsn, file_size: u64, ) -> Self { Self { - tenant_id, + tenant_shard_id, timeline_id, key_range, lsn_range: Self::image_layer_lsn_range(lsn), @@ -83,14 +84,14 @@ impl PersistentLayerDesc { } pub fn new_delta( - tenant_id: TenantId, + tenant_shard_id: TenantShardId, timeline_id: TimelineId, key_range: Range, lsn_range: Range, file_size: u64, ) -> Self { Self { - tenant_id, + tenant_shard_id, timeline_id, key_range, lsn_range, @@ -100,18 +101,22 @@ impl PersistentLayerDesc { } pub fn from_filename( - tenant_id: TenantId, + tenant_shard_id: TenantShardId, timeline_id: TimelineId, filename: LayerFileName, file_size: u64, ) -> Self { match filename { LayerFileName::Image(i) => { - Self::new_img(tenant_id, timeline_id, i.key_range, i.lsn, file_size) - } - LayerFileName::Delta(d) => { - Self::new_delta(tenant_id, timeline_id, d.key_range, d.lsn_range, file_size) + Self::new_img(tenant_shard_id, timeline_id, i.key_range, i.lsn, file_size) } + LayerFileName::Delta(d) => Self::new_delta( + tenant_shard_id, + timeline_id, + d.key_range, + d.lsn_range, + file_size, + ), } } @@ -172,10 +177,6 @@ impl PersistentLayerDesc { self.timeline_id } - pub fn get_tenant_id(&self) -> TenantId { - self.tenant_id - } - /// Does this layer only contain some data for the key-range (incremental), /// or does it contain a version of every page? This is important to know /// for garbage collecting old layers: an incremental layer depends on @@ -192,7 +193,7 @@ impl PersistentLayerDesc { if self.is_delta { println!( "----- delta layer for ten {} tli {} keys {}-{} lsn {}-{} is_incremental {} size {} ----", - self.tenant_id, + self.tenant_shard_id, self.timeline_id, self.key_range.start, self.key_range.end, @@ -204,7 +205,7 @@ impl PersistentLayerDesc { } else { println!( "----- image layer for ten {} tli {} key {}-{} at {} is_incremental {} size {} ----", - self.tenant_id, + self.tenant_shard_id, self.timeline_id, self.key_range.start, self.key_range.end, diff --git a/pageserver/src/tenant/tasks.rs b/pageserver/src/tenant/tasks.rs index 860bb255ca..138578ec8a 100644 --- a/pageserver/src/tenant/tasks.rs +++ b/pageserver/src/tenant/tasks.rs @@ -86,7 +86,7 @@ pub fn start_background_loops( tenant: &Arc, background_jobs_can_start: Option<&completion::Barrier>, ) { - let tenant_id = tenant.tenant_id; + let tenant_id = tenant.tenant_shard_id.tenant_id; task_mgr::spawn( BACKGROUND_RUNTIME.handle(), TaskKind::Compaction, diff --git a/pageserver/src/tenant/timeline.rs b/pageserver/src/tenant/timeline.rs index a7cf427de5..24f59673c1 100644 --- a/pageserver/src/tenant/timeline.rs +++ b/pageserver/src/tenant/timeline.rs @@ -13,8 +13,12 @@ use camino::{Utf8Path, Utf8PathBuf}; use enumset::EnumSet; use fail::fail_point; use itertools::Itertools; -use pageserver_api::models::{ - DownloadRemoteLayersTaskInfo, DownloadRemoteLayersTaskSpawnRequest, LayerMapInfo, TimelineState, +use pageserver_api::{ + models::{ + DownloadRemoteLayersTaskInfo, DownloadRemoteLayersTaskSpawnRequest, LayerMapInfo, + TimelineState, + }, + shard::TenantShardId, }; use serde_with::serde_as; use storage_broker::BrokerClientChannel; @@ -149,7 +153,7 @@ pub struct Timeline { myself: Weak, - pub tenant_id: TenantId, + pub(crate) tenant_shard_id: TenantShardId, pub timeline_id: TimelineId, /// The generation of the tenant that instantiated us: this is used for safety when writing remote objects. @@ -701,7 +705,7 @@ impl Timeline { } /// Flush to disk all data that was written with the put_* functions - #[instrument(skip(self), fields(tenant_id=%self.tenant_id, timeline_id=%self.timeline_id))] + #[instrument(skip(self), fields(tenant_id=%self.tenant_shard_id.tenant_id, shard_id=%self.tenant_shard_id.shard_slug(), timeline_id=%self.timeline_id))] pub async fn freeze_and_flush(&self) -> anyhow::Result<()> { self.freeze_inmem_layer(false).await; self.flush_frozen_layers_and_wait().await @@ -937,7 +941,7 @@ impl Timeline { tracing::debug!("Waiting for WalReceiverManager..."); task_mgr::shutdown_tasks( Some(TaskKind::WalReceiverManager), - Some(self.tenant_id), + Some(self.tenant_shard_id.tenant_id), Some(self.timeline_id), ) .await; @@ -988,7 +992,7 @@ impl Timeline { // Shut down the layer flush task before the remote client, as one depends on the other task_mgr::shutdown_tasks( Some(TaskKind::LayerFlushTask), - Some(self.tenant_id), + Some(self.tenant_shard_id.tenant_id), Some(self.timeline_id), ) .await; @@ -1006,7 +1010,12 @@ impl Timeline { tracing::debug!("Waiting for tasks..."); - task_mgr::shutdown_tasks(None, Some(self.tenant_id), Some(self.timeline_id)).await; + task_mgr::shutdown_tasks( + None, + Some(self.tenant_shard_id.tenant_id), + Some(self.timeline_id), + ) + .await; // Finally wait until any gate-holders are complete self.gate.close().await; @@ -1125,7 +1134,7 @@ impl Timeline { } } - #[instrument(skip_all, fields(tenant_id = %self.tenant_id, timeline_id = %self.timeline_id))] + #[instrument(skip_all, fields(tenant_id = %self.tenant_shard_id.tenant_id, shard_id = %self.tenant_shard_id.shard_slug(), timeline_id = %self.timeline_id))] pub async fn download_layer(&self, layer_file_name: &str) -> anyhow::Result> { let Some(layer) = self.find_layer(layer_file_name).await else { return Ok(None); @@ -1330,7 +1339,11 @@ impl Timeline { &self.tenant_conf.read().unwrap().tenant_conf, &self.conf.default_tenant_conf, ); - let tenant_id_str = self.tenant_id.to_string(); + + // TODO(sharding): make evictions state shard aware + // (https://github.com/neondatabase/neon/issues/5953) + let tenant_id_str = self.tenant_shard_id.tenant_id.to_string(); + let timeline_id_str = self.timeline_id.to_string(); self.metrics .evictions_with_low_residence_duration @@ -1350,7 +1363,7 @@ impl Timeline { metadata: &TimelineMetadata, ancestor: Option>, timeline_id: TimelineId, - tenant_id: TenantId, + tenant_shard_id: TenantShardId, generation: Generation, walredo_mgr: Arc, resources: TimelineResources, @@ -1381,7 +1394,7 @@ impl Timeline { tenant_conf, myself: myself.clone(), timeline_id, - tenant_id, + tenant_shard_id, generation, pg_version, layers: Arc::new(tokio::sync::RwLock::new(LayerManager::create())), @@ -1408,7 +1421,7 @@ impl Timeline { ancestor_lsn: metadata.ancestor_lsn(), metrics: TimelineMetrics::new( - &tenant_id, + &tenant_shard_id.tenant_id, &timeline_id, crate::metrics::EvictionsWithLowResidenceDurationBuilder::new( "mtime", @@ -1459,7 +1472,7 @@ impl Timeline { initial_logical_size_can_start, initial_logical_size_attempt: Mutex::new(initial_logical_size_attempt), cancel, - gate: Gate::new(format!("Timeline<{tenant_id}/{timeline_id}>")), + gate: Gate::new(format!("Timeline<{tenant_shard_id}/{timeline_id}>")), compaction_lock: tokio::sync::Mutex::default(), gc_lock: tokio::sync::Mutex::default(), @@ -1481,14 +1494,14 @@ impl Timeline { FlushLoopState::Running { .. } => { info!( "skipping attempt to start flush_loop twice {}/{}", - self.tenant_id, self.timeline_id + self.tenant_shard_id, self.timeline_id ); return; } FlushLoopState::Exited => { warn!( "ignoring attempt to restart exited flush_loop {}/{}", - self.tenant_id, self.timeline_id + self.tenant_shard_id, self.timeline_id ); return; } @@ -1507,7 +1520,7 @@ impl Timeline { task_mgr::spawn( task_mgr::BACKGROUND_RUNTIME.handle(), task_mgr::TaskKind::LayerFlushTask, - Some(self.tenant_id), + Some(self.tenant_shard_id.tenant_id), Some(self.timeline_id), "layer flush task", false, @@ -1519,7 +1532,7 @@ impl Timeline { *flush_loop_state = FlushLoopState::Exited; Ok(()) } - .instrument(info_span!(parent: None, "layer flush task", tenant_id = %self.tenant_id, timeline_id = %self.timeline_id)) + .instrument(info_span!(parent: None, "layer flush task", tenant_id = %self.tenant_shard_id.tenant_id, shard_id = %self.tenant_shard_id.shard_slug(), timeline_id = %self.timeline_id)) ); } @@ -1534,7 +1547,7 @@ impl Timeline { ) { info!( "launching WAL receiver for timeline {} of tenant {}", - self.timeline_id, self.tenant_id + self.timeline_id, self.tenant_shard_id ); let tenant_conf_guard = self.tenant_conf.read().unwrap(); @@ -1595,7 +1608,9 @@ impl Timeline { // Scan timeline directory and create ImageFileName and DeltaFilename // structs representing all files on disk - let timeline_path = self.conf.timeline_path(&self.tenant_id, &self.timeline_id); + let timeline_path = self + .conf + .timeline_path(&self.tenant_shard_id, &self.timeline_id); let conf = self.conf; let span = tracing::Span::current(); @@ -1802,7 +1817,7 @@ impl Timeline { task_mgr::spawn( task_mgr::BACKGROUND_RUNTIME.handle(), task_mgr::TaskKind::InitialLogicalSizeCalculation, - Some(self.tenant_id), + Some(self.tenant_shard_id.tenant_id), Some(self.timeline_id), "initial size calculation", false, @@ -1912,7 +1927,7 @@ impl Timeline { task_mgr::spawn( task_mgr::BACKGROUND_RUNTIME.handle(), task_mgr::TaskKind::OndemandLogicalSizeCalculation, - Some(self.tenant_id), + Some(self.tenant_shard_id.tenant_id), Some(self.timeline_id), "ondemand logical size calculation", false, @@ -1988,7 +2003,7 @@ impl Timeline { fail::fail_point!("timeline-calculate-logical-size-check-dir-exists", |_| { if !self .conf - .metadata_path(&self.tenant_id, &self.timeline_id) + .metadata_path(&self.tenant_shard_id, &self.timeline_id) .exists() { error!("timeline-calculate-logical-size-pre metadata file does not exist") @@ -2341,7 +2356,13 @@ impl Timeline { // FIXME: It's pointless to check the cache for things that are not 8kB pages. // We should look at the key to determine if it's a cacheable object let (lsn, read_guard) = cache - .lookup_materialized_page(self.tenant_id, self.timeline_id, key, lsn, ctx) + .lookup_materialized_page( + self.tenant_shard_id.tenant_id, + self.timeline_id, + key, + lsn, + ctx, + ) .await?; let img = Bytes::from(read_guard.to_vec()); Some((lsn, img)) @@ -2369,7 +2390,7 @@ impl Timeline { self.get_last_record_lsn(), self.conf, self.timeline_id, - self.tenant_id, + self.tenant_shard_id, ) .await?; Ok(layer) @@ -2535,7 +2556,7 @@ impl Timeline { } /// Flush one frozen in-memory layer to disk, as a new delta layer. - #[instrument(skip_all, fields(tenant_id=%self.tenant_id, timeline_id=%self.timeline_id, layer=%frozen_layer))] + #[instrument(skip_all, fields(tenant_id=%self.tenant_shard_id.tenant_id, shard_id = %self.tenant_shard_id.shard_slug(), timeline_id=%self.timeline_id, layer=%frozen_layer))] async fn flush_frozen_layer( self: &Arc, frozen_layer: Arc, @@ -2656,9 +2677,14 @@ impl Timeline { // If we updated our disk_consistent_lsn, persist the updated metadata to local disk. if let Some(metadata) = metadata { - save_metadata(self.conf, &self.tenant_id, &self.timeline_id, &metadata) - .await - .context("save_metadata")?; + save_metadata( + self.conf, + &self.tenant_shard_id, + &self.timeline_id, + &metadata, + ) + .await + .context("save_metadata")?; } Ok(()) } @@ -2722,9 +2748,14 @@ impl Timeline { ) -> anyhow::Result<()> { let metadata = self.schedule_uploads(disk_consistent_lsn, layers_to_upload)?; - save_metadata(self.conf, &self.tenant_id, &self.timeline_id, &metadata) - .await - .context("save_metadata")?; + save_metadata( + self.conf, + &self.tenant_shard_id, + &self.timeline_id, + &metadata, + ) + .await + .context("save_metadata")?; Ok(()) } @@ -2772,7 +2803,7 @@ impl Timeline { par_fsync::par_fsync(&[new_delta_path]).context("fsync of delta layer")?; par_fsync::par_fsync(&[self_clone .conf - .timeline_path(&self_clone.tenant_id, &self_clone.timeline_id)]) + .timeline_path(&self_clone.tenant_shard_id, &self_clone.timeline_id)]) .context("fsync of timeline dir")?; anyhow::Ok(new_delta) @@ -2928,7 +2959,7 @@ impl Timeline { let mut image_layer_writer = ImageLayerWriter::new( self.conf, self.timeline_id, - self.tenant_id, + self.tenant_shard_id, &img_range, lsn, ) @@ -3001,9 +3032,11 @@ impl Timeline { .await .context("fsync of newly created layer files")?; - par_fsync::par_fsync_async(&[self.conf.timeline_path(&self.tenant_id, &self.timeline_id)]) - .await - .context("fsync of timeline dir")?; + par_fsync::par_fsync_async(&[self + .conf + .timeline_path(&self.tenant_shard_id, &self.timeline_id)]) + .await + .context("fsync of timeline dir")?; let mut guard = self.layers.write().await; @@ -3489,7 +3522,7 @@ impl Timeline { DeltaLayerWriter::new( self.conf, self.timeline_id, - self.tenant_id, + self.tenant_shard_id, key, if dup_end_lsn.is_valid() { // this is a layer containing slice of values of the same key @@ -3550,7 +3583,9 @@ impl Timeline { .await .context("fsync all new layers")?; - let timeline_dir = self.conf.timeline_path(&self.tenant_id, &self.timeline_id); + let timeline_dir = self + .conf + .timeline_path(&self.tenant_shard_id, &self.timeline_id); par_fsync::par_fsync_async(&[timeline_dir]) .await @@ -3601,7 +3636,7 @@ impl Timeline { let ctx = ctx.attached_child(); let mut stats = CompactLevel0Phase1StatsBuilder { version: Some(2), - tenant_id: Some(self.tenant_id), + tenant_id: Some(self.tenant_shard_id.tenant_id), timeline_id: Some(self.timeline_id), ..Default::default() }; @@ -4062,7 +4097,7 @@ impl Timeline { let cache = page_cache::get(); if let Err(e) = cache .memorize_materialized_page( - self.tenant_id, + self.tenant_shard_id.tenant_id, self.timeline_id, key, last_rec_lsn, @@ -4106,7 +4141,7 @@ impl Timeline { let task_id = task_mgr::spawn( task_mgr::BACKGROUND_RUNTIME.handle(), task_mgr::TaskKind::DownloadAllRemoteLayers, - Some(self.tenant_id), + Some(self.tenant_shard_id.tenant_id), Some(self.timeline_id), "download all remote layers task", false, @@ -4128,7 +4163,7 @@ impl Timeline { }; Ok(()) } - .instrument(info_span!(parent: None, "download_all_remote_layers", tenant_id = %self.tenant_id, timeline_id = %self.timeline_id)) + .instrument(info_span!(parent: None, "download_all_remote_layers", tenant_id = %self.tenant_shard_id.tenant_id, shard_id = %self.tenant_shard_id.shard_slug(), timeline_id = %self.timeline_id)) ); let initial_info = DownloadRemoteLayersTaskInfo { @@ -4329,8 +4364,10 @@ impl Timeline { } pub(crate) fn get_shard_index(&self) -> ShardIndex { - // TODO: carry this on the struct - ShardIndex::unsharded() + ShardIndex { + shard_number: self.tenant_shard_id.shard_number, + shard_count: self.tenant_shard_id.shard_count, + } } } diff --git a/pageserver/src/tenant/timeline/delete.rs b/pageserver/src/tenant/timeline/delete.rs index fefeafb7d3..497796c80a 100644 --- a/pageserver/src/tenant/timeline/delete.rs +++ b/pageserver/src/tenant/timeline/delete.rs @@ -4,13 +4,10 @@ use std::{ }; use anyhow::Context; -use pageserver_api::models::TimelineState; +use pageserver_api::{models::TimelineState, shard::TenantShardId}; use tokio::sync::OwnedMutexGuard; use tracing::{debug, error, info, instrument, warn, Instrument, Span}; -use utils::{ - crashsafe, fs_ext, - id::{TenantId, TimelineId}, -}; +use utils::{crashsafe, fs_ext, id::TimelineId}; use crate::{ config::PageServerConf, @@ -47,7 +44,7 @@ async fn stop_tasks(timeline: &Timeline) -> Result<(), DeleteTimelineError> { // Shut down the layer flush task before the remote client, as one depends on the other task_mgr::shutdown_tasks( Some(TaskKind::LayerFlushTask), - Some(timeline.tenant_id), + Some(timeline.tenant_shard_id.tenant_id), Some(timeline.timeline_id), ) .await; @@ -73,7 +70,12 @@ async fn stop_tasks(timeline: &Timeline) -> Result<(), DeleteTimelineError> { // NB: This and other delete_timeline calls do not run as a task_mgr task, // so, they are not affected by this shutdown_tasks() call. info!("waiting for timeline tasks to shutdown"); - task_mgr::shutdown_tasks(None, Some(timeline.tenant_id), Some(timeline.timeline_id)).await; + task_mgr::shutdown_tasks( + None, + Some(timeline.tenant_shard_id.tenant_id), + Some(timeline.timeline_id), + ) + .await; fail::fail_point!("timeline-delete-before-index-deleted-at", |_| { Err(anyhow::anyhow!( @@ -125,7 +127,7 @@ async fn set_deleted_in_remote_index(timeline: &Timeline) -> Result<(), DeleteTi // pub(super): documentation link pub(super) async fn delete_local_layer_files( conf: &PageServerConf, - tenant_id: TenantId, + tenant_shard_id: TenantShardId, timeline: &Timeline, ) -> anyhow::Result<()> { let guards = async { tokio::join!(timeline.gc_lock.lock(), timeline.compaction_lock.lock()) }; @@ -139,7 +141,7 @@ pub(super) async fn delete_local_layer_files( // NB: storage_sync upload tasks that reference these layers have been cancelled // by the caller. - let local_timeline_directory = conf.timeline_path(&tenant_id, &timeline.timeline_id); + let local_timeline_directory = conf.timeline_path(&tenant_shard_id, &timeline.timeline_id); fail::fail_point!("timeline-delete-before-rm", |_| { Err(anyhow::anyhow!("failpoint: timeline-delete-before-rm"))? @@ -175,7 +177,7 @@ pub(super) async fn delete_local_layer_files( return Ok(()); } - let metadata_path = conf.metadata_path(&tenant_id, &timeline.timeline_id); + let metadata_path = conf.metadata_path(&tenant_shard_id, &timeline.timeline_id); for entry in walkdir::WalkDir::new(&local_timeline_directory).contents_first(true) { #[cfg(feature = "testing")] @@ -250,11 +252,11 @@ async fn delete_remote_layers_and_index(timeline: &Timeline) -> anyhow::Result<( // (nothing can fail after its deletion) async fn cleanup_remaining_timeline_fs_traces( conf: &PageServerConf, - tenant_id: TenantId, + tenant_shard_id: TenantShardId, timeline_id: TimelineId, ) -> anyhow::Result<()> { // Remove local metadata - tokio::fs::remove_file(conf.metadata_path(&tenant_id, &timeline_id)) + tokio::fs::remove_file(conf.metadata_path(&tenant_shard_id, &timeline_id)) .await .or_else(fs_ext::ignore_not_found) .context("remove metadata")?; @@ -266,7 +268,7 @@ async fn cleanup_remaining_timeline_fs_traces( }); // Remove timeline dir - tokio::fs::remove_dir(conf.timeline_path(&tenant_id, &timeline_id)) + tokio::fs::remove_dir(conf.timeline_path(&tenant_shard_id, &timeline_id)) .await .or_else(fs_ext::ignore_not_found) .context("timeline dir")?; @@ -281,7 +283,7 @@ async fn cleanup_remaining_timeline_fs_traces( // to be reordered later and thus missed if a crash occurs. // Note that we dont need to sync after mark file is removed // because we can tolerate the case when mark file reappears on startup. - let timeline_path = conf.timelines_path(&tenant_id); + let timeline_path = conf.timelines_path(&tenant_shard_id); crashsafe::fsync_async(timeline_path) .await .context("fsync_pre_mark_remove")?; @@ -289,7 +291,7 @@ async fn cleanup_remaining_timeline_fs_traces( // Remove delete mark // TODO: once we are confident that no more exist in the field, remove this // line. It cleans up a legacy marker file that might in rare cases be present. - tokio::fs::remove_file(conf.timeline_delete_mark_file_path(tenant_id, timeline_id)) + tokio::fs::remove_file(conf.timeline_delete_mark_file_path(tenant_shard_id, timeline_id)) .await .or_else(fs_ext::ignore_not_found) .context("remove delete mark") @@ -355,7 +357,7 @@ impl DeleteTimelineFlow { // NB: If this fails half-way through, and is retried, the retry will go through // all the same steps again. Make sure the code here is idempotent, and don't // error out if some of the shutdown tasks have already been completed! - #[instrument(skip(tenant), fields(tenant_id=%tenant.tenant_id))] + #[instrument(skip(tenant), fields(tenant_id=%tenant.tenant_shard_id.tenant_id, shard_id=%tenant.tenant_shard_id.shard_slug()))] pub async fn run( tenant: &Arc, timeline_id: TimelineId, @@ -451,7 +453,8 @@ impl DeleteTimelineFlow { timeline_id: TimelineId, ) -> anyhow::Result<()> { let r = - cleanup_remaining_timeline_fs_traces(tenant.conf, tenant.tenant_id, timeline_id).await; + cleanup_remaining_timeline_fs_traces(tenant.conf, tenant.tenant_shard_id, timeline_id) + .await; info!("Done"); r } @@ -522,13 +525,13 @@ impl DeleteTimelineFlow { tenant: Arc, timeline: Arc, ) { - let tenant_id = timeline.tenant_id; + let tenant_shard_id = timeline.tenant_shard_id; let timeline_id = timeline.timeline_id; task_mgr::spawn( task_mgr::BACKGROUND_RUNTIME.handle(), TaskKind::TimelineDeletionWorker, - Some(tenant_id), + Some(tenant_shard_id.tenant_id), Some(timeline_id), "timeline_delete", false, @@ -541,7 +544,7 @@ impl DeleteTimelineFlow { } .instrument({ let span = - tracing::info_span!(parent: None, "delete_timeline", tenant_id=%tenant_id, timeline_id=%timeline_id); + tracing::info_span!(parent: None, "delete_timeline", tenant_id=%tenant_shard_id.tenant_id, shard_id=%tenant_shard_id.shard_slug(),timeline_id=%timeline_id); span.follows_from(Span::current()); span }), @@ -554,13 +557,14 @@ impl DeleteTimelineFlow { tenant: &Tenant, timeline: &Timeline, ) -> Result<(), DeleteTimelineError> { - delete_local_layer_files(conf, tenant.tenant_id, timeline).await?; + delete_local_layer_files(conf, tenant.tenant_shard_id, timeline).await?; delete_remote_layers_and_index(timeline).await?; pausable_failpoint!("in_progress_delete"); - cleanup_remaining_timeline_fs_traces(conf, tenant.tenant_id, timeline.timeline_id).await?; + cleanup_remaining_timeline_fs_traces(conf, tenant.tenant_shard_id, timeline.timeline_id) + .await?; remove_timeline_from_tenant(tenant, timeline.timeline_id, &guard).await?; diff --git a/pageserver/src/tenant/timeline/eviction_task.rs b/pageserver/src/tenant/timeline/eviction_task.rs index 2d0f1c609b..3fe4bc0f83 100644 --- a/pageserver/src/tenant/timeline/eviction_task.rs +++ b/pageserver/src/tenant/timeline/eviction_task.rs @@ -60,9 +60,12 @@ impl Timeline { task_mgr::spawn( BACKGROUND_RUNTIME.handle(), TaskKind::Eviction, - Some(self.tenant_id), + Some(self.tenant_shard_id.tenant_id), Some(self.timeline_id), - &format!("layer eviction for {}/{}", self.tenant_id, self.timeline_id), + &format!( + "layer eviction for {}/{}", + self.tenant_shard_id, self.timeline_id + ), false, async move { let cancel = task_mgr::shutdown_token(); @@ -77,7 +80,7 @@ impl Timeline { ); } - #[instrument(skip_all, fields(tenant_id = %self.tenant_id, timeline_id = %self.timeline_id))] + #[instrument(skip_all, fields(tenant_id = %self.tenant_shard_id.tenant_id, shard_id = %self.tenant_shard_id.shard_slug(), timeline_id = %self.timeline_id))] async fn eviction_task(self: Arc, cancel: CancellationToken) { use crate::tenant::tasks::random_init_delay; { @@ -340,7 +343,7 @@ impl Timeline { // Make one of the tenant's timelines draw the short straw and run the calculation. // The others wait until the calculation is done so that they take into account the // imitated accesses that the winner made. - let tenant = match crate::tenant::mgr::get_tenant(self.tenant_id, true) { + let tenant = match crate::tenant::mgr::get_tenant(self.tenant_shard_id.tenant_id, true) { Ok(t) => t, Err(_) => { return ControlFlow::Break(()); diff --git a/pageserver/src/tenant/timeline/layer_manager.rs b/pageserver/src/tenant/timeline/layer_manager.rs index 7e1aa279d3..dcd82949dd 100644 --- a/pageserver/src/tenant/timeline/layer_manager.rs +++ b/pageserver/src/tenant/timeline/layer_manager.rs @@ -1,8 +1,9 @@ use anyhow::{bail, ensure, Context, Result}; +use pageserver_api::shard::TenantShardId; use std::{collections::HashMap, sync::Arc}; use tracing::trace; use utils::{ - id::{TenantId, TimelineId}, + id::TimelineId, lsn::{AtomicLsn, Lsn}, }; @@ -73,7 +74,7 @@ impl LayerManager { last_record_lsn: Lsn, conf: &'static PageServerConf, timeline_id: TimelineId, - tenant_id: TenantId, + tenant_shard_id: TenantShardId, ) -> Result> { ensure!(lsn.is_aligned()); @@ -109,7 +110,8 @@ impl LayerManager { lsn ); - let new_layer = InMemoryLayer::create(conf, timeline_id, tenant_id, start_lsn).await?; + let new_layer = + InMemoryLayer::create(conf, timeline_id, tenant_shard_id, start_lsn).await?; let layer = Arc::new(new_layer); self.layer_map.open_layer = Some(layer.clone()); diff --git a/pageserver/src/tenant/timeline/uninit.rs b/pageserver/src/tenant/timeline/uninit.rs index f9bb6ca419..61130f541a 100644 --- a/pageserver/src/tenant/timeline/uninit.rs +++ b/pageserver/src/tenant/timeline/uninit.rs @@ -43,11 +43,11 @@ impl<'t> UninitializedTimeline<'t> { /// The caller is responsible for activating the timeline (function `.activate()`). pub(crate) fn finish_creation(mut self) -> anyhow::Result> { let timeline_id = self.timeline_id; - let tenant_id = self.owning_tenant.tenant_id; + let tenant_shard_id = self.owning_tenant.tenant_shard_id; if self.raw_timeline.is_none() { return Err(anyhow::anyhow!( - "No timeline for initialization found for {tenant_id}/{timeline_id}" + "No timeline for initialization found for {tenant_shard_id}/{timeline_id}" )); } @@ -61,13 +61,13 @@ impl<'t> UninitializedTimeline<'t> { anyhow::ensure!( new_disk_consistent_lsn.is_valid(), - "new timeline {tenant_id}/{timeline_id} has invalid disk_consistent_lsn" + "new timeline {tenant_shard_id}/{timeline_id} has invalid disk_consistent_lsn" ); let mut timelines = self.owning_tenant.timelines.lock().unwrap(); match timelines.entry(timeline_id) { Entry::Occupied(_) => anyhow::bail!( - "Found freshly initialized timeline {tenant_id}/{timeline_id} in the tenant map" + "Found freshly initialized timeline {tenant_shard_id}/{timeline_id} in the tenant map" ), Entry::Vacant(v) => { // after taking here should be no fallible operations, because the drop guard will not @@ -79,7 +79,7 @@ impl<'t> UninitializedTimeline<'t> { // this should be an assertion. uninit_mark.remove_uninit_mark().with_context(|| { format!( - "Failed to remove uninit mark file for timeline {tenant_id}/{timeline_id}" + "Failed to remove uninit mark file for timeline {tenant_shard_id}/{timeline_id}" ) })?; v.insert(Arc::clone(&new_timeline)); @@ -134,7 +134,7 @@ impl<'t> UninitializedTimeline<'t> { .with_context(|| { format!( "No raw timeline {}/{} found", - self.owning_tenant.tenant_id, self.timeline_id + self.owning_tenant.tenant_shard_id, self.timeline_id ) })? .0) @@ -144,7 +144,7 @@ impl<'t> UninitializedTimeline<'t> { impl Drop for UninitializedTimeline<'_> { fn drop(&mut self) { if let Some((_, uninit_mark)) = self.raw_timeline.take() { - let _entered = info_span!("drop_uninitialized_timeline", tenant_id = %self.owning_tenant.tenant_id, timeline_id = %self.timeline_id).entered(); + let _entered = info_span!("drop_uninitialized_timeline", tenant_id = %self.owning_tenant.tenant_shard_id.tenant_id, shard_id = %self.owning_tenant.tenant_shard_id.shard_slug(), timeline_id = %self.timeline_id).entered(); error!("Timeline got dropped without initializing, cleaning its files"); cleanup_timeline_directory(uninit_mark); } diff --git a/pageserver/src/tenant/timeline/walreceiver.rs b/pageserver/src/tenant/timeline/walreceiver.rs index 842bc3675c..04ff8602d6 100644 --- a/pageserver/src/tenant/timeline/walreceiver.rs +++ b/pageserver/src/tenant/timeline/walreceiver.rs @@ -71,7 +71,7 @@ impl WalReceiver { mut broker_client: BrokerClientChannel, ctx: &RequestContext, ) -> Self { - let tenant_id = timeline.tenant_id; + let tenant_id = timeline.tenant_shard_id.tenant_id; let timeline_id = timeline.timeline_id; let walreceiver_ctx = ctx.detached_child(TaskKind::WalReceiverManager, DownloadBehavior::Error); diff --git a/pageserver/src/tenant/timeline/walreceiver/connection_manager.rs b/pageserver/src/tenant/timeline/walreceiver/connection_manager.rs index 3077712445..7bfa246eeb 100644 --- a/pageserver/src/tenant/timeline/walreceiver/connection_manager.rs +++ b/pageserver/src/tenant/timeline/walreceiver/connection_manager.rs @@ -75,7 +75,7 @@ pub(super) async fn connection_manager_loop_step( } let id = TenantTimelineId { - tenant_id: connection_manager_state.timeline.tenant_id, + tenant_id: connection_manager_state.timeline.tenant_shard_id.tenant_id, timeline_id: connection_manager_state.timeline.timeline_id, }; @@ -388,7 +388,7 @@ struct BrokerSkTimeline { impl ConnectionManagerState { pub(super) fn new(timeline: Arc, conf: WalReceiverConf) -> Self { let id = TenantTimelineId { - tenant_id: timeline.tenant_id, + tenant_id: timeline.tenant_shard_id.tenant_id, timeline_id: timeline.timeline_id, }; Self { diff --git a/pageserver/src/tenant/timeline/walreceiver/walreceiver_connection.rs b/pageserver/src/tenant/timeline/walreceiver/walreceiver_connection.rs index 3e56753ad4..2b4aea7596 100644 --- a/pageserver/src/tenant/timeline/walreceiver/walreceiver_connection.rs +++ b/pageserver/src/tenant/timeline/walreceiver/walreceiver_connection.rs @@ -163,7 +163,7 @@ pub(super) async fn handle_walreceiver_connection( task_mgr::spawn( WALRECEIVER_RUNTIME.handle(), TaskKind::WalReceiverConnectionPoller, - Some(timeline.tenant_id), + Some(timeline.tenant_shard_id.tenant_id), Some(timeline.timeline_id), "walreceiver connection", false, diff --git a/pageserver/src/walredo.rs b/pageserver/src/walredo.rs index 4e684dec2d..ed468f220e 100644 --- a/pageserver/src/walredo.rs +++ b/pageserver/src/walredo.rs @@ -41,6 +41,9 @@ use utils::{bin_ser::BeSer, id::TenantId, lsn::Lsn, nonblock::set_nonblock}; #[cfg(feature = "testing")] use std::sync::atomic::{AtomicUsize, Ordering}; +#[cfg(feature = "testing")] +use pageserver_api::shard::TenantShardId; + use crate::config::PageServerConf; use crate::metrics::{ WalRedoKillCause, WAL_REDO_BYTES_HISTOGRAM, WAL_REDO_PROCESS_COUNTERS, @@ -991,7 +994,11 @@ impl WalRedoProcess { // these files will be collected to an allure report let filename = format!("walredo-{millis}-{}-{seq}.walredo", writebuf.len()); - let path = self.conf.tenant_path(&self.tenant_id).join(&filename); + // TODO(sharding): update this call when WalRedoProcess gets a TenantShardId. + let path = self + .conf + .tenant_path(&TenantShardId::unsharded(self.tenant_id)) + .join(&filename); let res = std::fs::OpenOptions::new() .write(true) From 46f20faa0dcb2652b0dce35060b36fda708fd2ab Mon Sep 17 00:00:00 2001 From: Rahul Modpur Date: Wed, 29 Nov 2023 23:08:03 +0530 Subject: [PATCH 10/38] neon_local: fix endpoint api to prevent two primary endpoints (#5520) `neon_local endpoint` subcommand currently allows creating two primary endpoints for the same branch which leads to shutdown of both endpoints `neon_local endpoint start` new behavior: 1. Fail if endpoint doesn't exist 2. Fail if two primary conflict detected Fixes #4959 Closes #5426 Signed-off-by: Rahul Modpur Co-authored-by: Joonas Koivunen --- README.md | 8 +- control_plane/src/bin/neon_local.rs | 106 ++++++--------------- control_plane/src/endpoint.rs | 25 +++++ test_runner/fixtures/neon_fixtures.py | 25 +---- test_runner/regress/test_compatibility.py | 7 +- test_runner/regress/test_neon_local_cli.py | 44 +++++++-- 6 files changed, 104 insertions(+), 111 deletions(-) diff --git a/README.md b/README.md index 75fad605c5..3e3123f5ee 100644 --- a/README.md +++ b/README.md @@ -149,6 +149,9 @@ tenant 9ef87a5bf0d92544f6fafeeb3239695c successfully created on the pageserver Created an initial timeline 'de200bd42b49cc1814412c7e592dd6e9' at Lsn 0/16B5A50 for tenant: 9ef87a5bf0d92544f6fafeeb3239695c Setting tenant 9ef87a5bf0d92544f6fafeeb3239695c as a default one +# create postgres compute node +> cargo neon endpoint create main + # start postgres compute node > cargo neon endpoint start main Starting new endpoint main (PostgreSQL v14) on timeline de200bd42b49cc1814412c7e592dd6e9 ... @@ -185,8 +188,11 @@ Created timeline 'b3b863fa45fa9e57e615f9f2d944e601' at Lsn 0/16F9A00 for tenant: (L) main [de200bd42b49cc1814412c7e592dd6e9] (L) ┗━ @0/16F9A00: migration_check [b3b863fa45fa9e57e615f9f2d944e601] +# create postgres on that branch +> cargo neon endpoint create migration_check --branch-name migration_check + # start postgres on that branch -> cargo neon endpoint start migration_check --branch-name migration_check +> cargo neon endpoint start migration_check Starting new endpoint migration_check (PostgreSQL v14) on timeline b3b863fa45fa9e57e615f9f2d944e601 ... Starting postgres at 'postgresql://cloud_admin@127.0.0.1:55434/postgres' diff --git a/control_plane/src/bin/neon_local.rs b/control_plane/src/bin/neon_local.rs index 3053122f6a..f7442c02c7 100644 --- a/control_plane/src/bin/neon_local.rs +++ b/control_plane/src/bin/neon_local.rs @@ -608,11 +608,9 @@ fn handle_endpoint(ep_match: &ArgMatches, env: &local_env::LocalEnv) -> Result<( }; let mut cplane = ComputeControlPlane::load(env.clone())?; - // All subcommands take an optional --tenant-id option - let tenant_id = get_tenant_id(sub_args, env)?; - match sub_name { "list" => { + let tenant_id = get_tenant_id(sub_args, env)?; let timeline_infos = get_timeline_infos(env, &tenant_id).unwrap_or_else(|e| { eprintln!("Failed to load timeline info: {}", e); HashMap::new() @@ -672,6 +670,7 @@ fn handle_endpoint(ep_match: &ArgMatches, env: &local_env::LocalEnv) -> Result<( println!("{table}"); } "create" => { + let tenant_id = get_tenant_id(sub_args, env)?; let branch_name = sub_args .get_one::("branch-name") .map(|s| s.as_str()) @@ -716,6 +715,18 @@ fn handle_endpoint(ep_match: &ArgMatches, env: &local_env::LocalEnv) -> Result<( (Some(_), true) => anyhow::bail!("cannot specify both lsn and hot-standby"), }; + match (mode, hot_standby) { + (ComputeMode::Static(_), true) => { + bail!("Cannot start a node in hot standby mode when it is already configured as a static replica") + } + (ComputeMode::Primary, true) => { + bail!("Cannot start a node as a hot standby replica, it is already configured as primary node") + } + _ => {} + } + + cplane.check_conflicting_endpoints(mode, tenant_id, timeline_id)?; + cplane.new_endpoint( &endpoint_id, tenant_id, @@ -728,8 +739,6 @@ fn handle_endpoint(ep_match: &ArgMatches, env: &local_env::LocalEnv) -> Result<( )?; } "start" => { - let pg_port: Option = sub_args.get_one::("pg-port").copied(); - let http_port: Option = sub_args.get_one::("http-port").copied(); let endpoint_id = sub_args .get_one::("endpoint_id") .ok_or_else(|| anyhow!("No endpoint ID was provided to start"))?; @@ -758,80 +767,28 @@ fn handle_endpoint(ep_match: &ArgMatches, env: &local_env::LocalEnv) -> Result<( env.safekeepers.iter().map(|sk| sk.id).collect() }; - let endpoint = cplane.endpoints.get(endpoint_id.as_str()); + let endpoint = cplane + .endpoints + .get(endpoint_id.as_str()) + .ok_or_else(|| anyhow::anyhow!("endpoint {endpoint_id} not found"))?; + + cplane.check_conflicting_endpoints( + endpoint.mode, + endpoint.tenant_id, + endpoint.timeline_id, + )?; let ps_conf = env.get_pageserver_conf(pageserver_id)?; let auth_token = if matches!(ps_conf.pg_auth_type, AuthType::NeonJWT) { - let claims = Claims::new(Some(tenant_id), Scope::Tenant); + let claims = Claims::new(Some(endpoint.tenant_id), Scope::Tenant); Some(env.generate_auth_token(&claims)?) } else { None }; - let hot_standby = sub_args - .get_one::("hot-standby") - .copied() - .unwrap_or(false); - - if let Some(endpoint) = endpoint { - match (&endpoint.mode, hot_standby) { - (ComputeMode::Static(_), true) => { - bail!("Cannot start a node in hot standby mode when it is already configured as a static replica") - } - (ComputeMode::Primary, true) => { - bail!("Cannot start a node as a hot standby replica, it is already configured as primary node") - } - _ => {} - } - println!("Starting existing endpoint {endpoint_id}..."); - endpoint.start(&auth_token, safekeepers, remote_ext_config)?; - } else { - let branch_name = sub_args - .get_one::("branch-name") - .map(|s| s.as_str()) - .unwrap_or(DEFAULT_BRANCH_NAME); - let timeline_id = env - .get_branch_timeline_id(branch_name, tenant_id) - .ok_or_else(|| { - anyhow!("Found no timeline id for branch name '{branch_name}'") - })?; - let lsn = sub_args - .get_one::("lsn") - .map(|lsn_str| Lsn::from_str(lsn_str)) - .transpose() - .context("Failed to parse Lsn from the request")?; - let pg_version = sub_args - .get_one::("pg-version") - .copied() - .context("Failed to `pg-version` from the argument string")?; - - let mode = match (lsn, hot_standby) { - (Some(lsn), false) => ComputeMode::Static(lsn), - (None, true) => ComputeMode::Replica, - (None, false) => ComputeMode::Primary, - (Some(_), true) => anyhow::bail!("cannot specify both lsn and hot-standby"), - }; - - // when used with custom port this results in non obvious behaviour - // port is remembered from first start command, i e - // start --port X - // stop - // start <-- will also use port X even without explicit port argument - println!("Starting new endpoint {endpoint_id} (PostgreSQL v{pg_version}) on timeline {timeline_id} ..."); - - let ep = cplane.new_endpoint( - endpoint_id, - tenant_id, - timeline_id, - pg_port, - http_port, - pg_version, - mode, - pageserver_id, - )?; - ep.start(&auth_token, safekeepers, remote_ext_config)?; - } + println!("Starting existing endpoint {endpoint_id}..."); + endpoint.start(&auth_token, safekeepers, remote_ext_config)?; } "reconfigure" => { let endpoint_id = sub_args @@ -1437,15 +1394,7 @@ fn cli() -> Command { .subcommand(Command::new("start") .about("Start postgres.\n If the endpoint doesn't exist yet, it is created.") .arg(endpoint_id_arg.clone()) - .arg(tenant_id_arg.clone()) - .arg(branch_name_arg.clone()) - .arg(timeline_id_arg.clone()) - .arg(lsn_arg) - .arg(pg_port_arg) - .arg(http_port_arg) .arg(endpoint_pageserver_id_arg.clone()) - .arg(pg_version_arg) - .arg(hot_standby_arg) .arg(safekeepers_arg) .arg(remote_ext_config_args) ) @@ -1458,7 +1407,6 @@ fn cli() -> Command { .subcommand( Command::new("stop") .arg(endpoint_id_arg) - .arg(tenant_id_arg.clone()) .arg( Arg::new("destroy") .help("Also delete data directory (now optional, should be default in future)") diff --git a/control_plane/src/endpoint.rs b/control_plane/src/endpoint.rs index ae45746925..12b1250764 100644 --- a/control_plane/src/endpoint.rs +++ b/control_plane/src/endpoint.rs @@ -125,6 +125,7 @@ impl ComputeControlPlane { let http_port = http_port.unwrap_or_else(|| self.get_port() + 1); let pageserver = PageServerNode::from_env(&self.env, self.env.get_pageserver_conf(pageserver_id)?); + let ep = Arc::new(Endpoint { endpoint_id: endpoint_id.to_owned(), pg_address: SocketAddr::new("127.0.0.1".parse().unwrap(), pg_port), @@ -169,6 +170,30 @@ impl ComputeControlPlane { Ok(ep) } + + pub fn check_conflicting_endpoints( + &self, + mode: ComputeMode, + tenant_id: TenantId, + timeline_id: TimelineId, + ) -> Result<()> { + if matches!(mode, ComputeMode::Primary) { + // this check is not complete, as you could have a concurrent attempt at + // creating another primary, both reading the state before checking it here, + // but it's better than nothing. + let mut duplicates = self.endpoints.iter().filter(|(_k, v)| { + v.tenant_id == tenant_id + && v.timeline_id == timeline_id + && v.mode == mode + && v.status() != "stopped" + }); + + if let Some((key, _)) = duplicates.next() { + bail!("attempting to create a duplicate primary endpoint on tenant {tenant_id}, timeline {timeline_id}: endpoint {key:?} exists already. please don't do this, it is not supported."); + } + } + Ok(()) + } } /////////////////////////////////////////////////////////////////////////////// diff --git a/test_runner/fixtures/neon_fixtures.py b/test_runner/fixtures/neon_fixtures.py index f060a63344..214bcc3a53 100644 --- a/test_runner/fixtures/neon_fixtures.py +++ b/test_runner/fixtures/neon_fixtures.py @@ -1414,34 +1414,19 @@ class NeonCli(AbstractNeonCli): def endpoint_start( self, endpoint_id: str, - pg_port: int, - http_port: int, safekeepers: Optional[List[int]] = None, - tenant_id: Optional[TenantId] = None, - lsn: Optional[Lsn] = None, - branch_name: Optional[str] = None, remote_ext_config: Optional[str] = None, pageserver_id: Optional[int] = None, ) -> "subprocess.CompletedProcess[str]": args = [ "endpoint", "start", - "--tenant-id", - str(tenant_id or self.env.initial_tenant), - "--pg-version", - self.env.pg_version, ] if remote_ext_config is not None: args.extend(["--remote-ext-config", remote_ext_config]) - if lsn is not None: - args.append(f"--lsn={lsn}") - args.extend(["--pg-port", str(pg_port)]) - args.extend(["--http-port", str(http_port)]) if safekeepers is not None: args.extend(["--safekeepers", (",".join(map(str, safekeepers)))]) - if branch_name is not None: - args.extend(["--branch-name", branch_name]) if endpoint_id is not None: args.append(endpoint_id) if pageserver_id is not None: @@ -1468,15 +1453,12 @@ class NeonCli(AbstractNeonCli): def endpoint_stop( self, endpoint_id: str, - tenant_id: Optional[TenantId] = None, destroy=False, check_return_code=True, ) -> "subprocess.CompletedProcess[str]": args = [ "endpoint", "stop", - "--tenant-id", - str(tenant_id or self.env.initial_tenant), ] if destroy: args.append("--destroy") @@ -2507,9 +2489,6 @@ class Endpoint(PgProtocol): self.env.neon_cli.endpoint_start( self.endpoint_id, - pg_port=self.pg_port, - http_port=self.http_port, - tenant_id=self.tenant_id, safekeepers=self.active_safekeepers, remote_ext_config=remote_ext_config, pageserver_id=pageserver_id, @@ -2589,7 +2568,7 @@ class Endpoint(PgProtocol): if self.running: assert self.endpoint_id is not None self.env.neon_cli.endpoint_stop( - self.endpoint_id, self.tenant_id, check_return_code=self.check_stop_result + self.endpoint_id, check_return_code=self.check_stop_result ) self.running = False @@ -2603,7 +2582,7 @@ class Endpoint(PgProtocol): assert self.endpoint_id is not None self.env.neon_cli.endpoint_stop( - self.endpoint_id, self.tenant_id, True, check_return_code=self.check_stop_result + self.endpoint_id, True, check_return_code=self.check_stop_result ) self.endpoint_id = None self.running = False diff --git a/test_runner/regress/test_compatibility.py b/test_runner/regress/test_compatibility.py index 9a2980280c..f3c6af4427 100644 --- a/test_runner/regress/test_compatibility.py +++ b/test_runner/regress/test_compatibility.py @@ -434,8 +434,11 @@ def check_neon_works( pg_port = port_distributor.get_port() http_port = port_distributor.get_port() - cli_current.endpoint_start("main", pg_port=pg_port, http_port=http_port) - request.addfinalizer(lambda: cli_current.endpoint_stop("main")) + cli_current.endpoint_create( + branch_name="main", pg_port=pg_port, http_port=http_port, endpoint_id="ep-main" + ) + cli_current.endpoint_start("ep-main") + request.addfinalizer(lambda: cli_current.endpoint_stop("ep-main")) connstr = f"host=127.0.0.1 port={pg_port} user=cloud_admin dbname=postgres" pg_bin.run_capture( diff --git a/test_runner/regress/test_neon_local_cli.py b/test_runner/regress/test_neon_local_cli.py index becdd9ff80..46b72fbca5 100644 --- a/test_runner/regress/test_neon_local_cli.py +++ b/test_runner/regress/test_neon_local_cli.py @@ -1,3 +1,4 @@ +import pytest from fixtures.neon_fixtures import NeonEnvBuilder from fixtures.port_distributor import PortDistributor @@ -11,19 +12,50 @@ def test_neon_cli_basics(neon_env_builder: NeonEnvBuilder, port_distributor: Por env.neon_cli.start() env.neon_cli.create_tenant(tenant_id=env.initial_tenant, set_default=True) + main_branch_name = "main" pg_port = port_distributor.get_port() http_port = port_distributor.get_port() - env.neon_cli.endpoint_start( - endpoint_id="ep-basic-main", pg_port=pg_port, http_port=http_port + env.neon_cli.endpoint_create( + main_branch_name, pg_port, http_port, endpoint_id="ep-basic-main" ) + env.neon_cli.endpoint_start("ep-basic-main") branch_name = "migration-check" - - env.neon_cli.create_branch(new_branch_name=branch_name) + env.neon_cli.create_branch(branch_name) pg_port = port_distributor.get_port() http_port = port_distributor.get_port() - env.neon_cli.endpoint_start( - f"ep-{branch_name}", pg_port, http_port, branch_name=branch_name + env.neon_cli.endpoint_create( + branch_name, pg_port, http_port, endpoint_id=f"ep-{branch_name}" ) + env.neon_cli.endpoint_start(f"ep-{branch_name}") finally: env.neon_cli.stop() + + +def test_neon_two_primary_endpoints_fail( + neon_env_builder: NeonEnvBuilder, port_distributor: PortDistributor +): + """ + Two primary endpoints with same tenant and timeline will not run together + """ + env = neon_env_builder.init_start() + branch_name = "main" + + pg_port = port_distributor.get_port() + http_port = port_distributor.get_port() + env.neon_cli.endpoint_create(branch_name, pg_port, http_port, "ep1") + + pg_port = port_distributor.get_port() + http_port = port_distributor.get_port() + # ep1 is not running so create will succeed + env.neon_cli.endpoint_create(branch_name, pg_port, http_port, "ep2") + + env.neon_cli.endpoint_start("ep1") + + expected_message = f'attempting to create a duplicate primary endpoint on tenant {env.initial_tenant}, timeline {env.initial_timeline}: endpoint "ep1" exists already. please don\'t do this, it is not supported.' + with pytest.raises(RuntimeError): + assert expected_message in env.neon_cli.endpoint_start("ep2").stderr + + env.neon_cli.endpoint_stop("ep1") + # ep1 is stopped so create ep2 will succeed + env.neon_cli.endpoint_start("ep2") From ca597206b8ebd52b29cec8ddc1e2591211fb22d6 Mon Sep 17 00:00:00 2001 From: Christian Schwarz Date: Wed, 29 Nov 2023 19:44:37 +0100 Subject: [PATCH 11/38] walredo: latency histogram for spawn duration (#5925) fixes https://github.com/neondatabase/neon/issues/5891 --- pageserver/src/metrics.rs | 10 ++++++++++ pageserver/src/walredo.rs | 6 +++++- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/pageserver/src/metrics.rs b/pageserver/src/metrics.rs index b178024b9a..c15b1fce63 100644 --- a/pageserver/src/metrics.rs +++ b/pageserver/src/metrics.rs @@ -1253,6 +1253,15 @@ pub(crate) static WAL_REDO_RECORD_COUNTER: Lazy = Lazy::new(|| { .unwrap() }); +pub(crate) static WAL_REDO_PROCESS_LAUNCH_DURATION_HISTOGRAM: Lazy = Lazy::new(|| { + register_histogram!( + "pageserver_wal_redo_process_launch_duration", + "Histogram of the duration of successful WalRedoProcess::launch calls", + redo_histogram_time_buckets!(), + ) + .expect("failed to define a metric") +}); + pub(crate) struct WalRedoProcessCounters { pub(crate) started: IntCounter, pub(crate) killed_by_cause: enum_map::EnumMap, @@ -1962,6 +1971,7 @@ pub fn preinitialize_metrics() { &WAL_REDO_TIME, &WAL_REDO_RECORDS_HISTOGRAM, &WAL_REDO_BYTES_HISTOGRAM, + &WAL_REDO_PROCESS_LAUNCH_DURATION_HISTOGRAM, ] .into_iter() .for_each(|h| { diff --git a/pageserver/src/walredo.rs b/pageserver/src/walredo.rs index ed468f220e..edce158e75 100644 --- a/pageserver/src/walredo.rs +++ b/pageserver/src/walredo.rs @@ -47,7 +47,8 @@ use pageserver_api::shard::TenantShardId; use crate::config::PageServerConf; use crate::metrics::{ WalRedoKillCause, WAL_REDO_BYTES_HISTOGRAM, WAL_REDO_PROCESS_COUNTERS, - WAL_REDO_RECORDS_HISTOGRAM, WAL_REDO_RECORD_COUNTER, WAL_REDO_TIME, + WAL_REDO_PROCESS_LAUNCH_DURATION_HISTOGRAM, WAL_REDO_RECORDS_HISTOGRAM, + WAL_REDO_RECORD_COUNTER, WAL_REDO_TIME, }; use crate::pgdatadir_mapping::{key_to_rel_block, key_to_slru_block}; use crate::repository::Key; @@ -241,10 +242,13 @@ impl PostgresRedoManager { let mut proc_guard = self.redo_process.write().unwrap(); match &*proc_guard { None => { + let timer = + WAL_REDO_PROCESS_LAUNCH_DURATION_HISTOGRAM.start_timer(); let proc = Arc::new( WalRedoProcess::launch(self.conf, self.tenant_id, pg_version) .context("launch walredo process")?, ); + timer.observe_duration(); *proc_guard = Some(Arc::clone(&proc)); proc } From f05d1b598aef8c5143893b84c59ee60f854cf323 Mon Sep 17 00:00:00 2001 From: Conrad Ludgate Date: Thu, 30 Nov 2023 10:18:59 +0000 Subject: [PATCH 12/38] proxy: add more db error info (#5951) ## Problem https://github.com/neondatabase/serverless/issues/51 ## Summary of changes include more error fields in the json response --- proxy/src/serverless/sql_over_http.rs | 53 ++++++++++++++++++++++----- 1 file changed, 44 insertions(+), 9 deletions(-) diff --git a/proxy/src/serverless/sql_over_http.rs b/proxy/src/serverless/sql_over_http.rs index 4a9829e360..b449a6e471 100644 --- a/proxy/src/serverless/sql_over_http.rs +++ b/proxy/src/serverless/sql_over_http.rs @@ -13,6 +13,7 @@ use hyper::{Body, HeaderMap, Request}; use serde_json::json; use serde_json::Map; use serde_json::Value; +use tokio_postgres::error::DbError; use tokio_postgres::types::Kind; use tokio_postgres::types::Type; use tokio_postgres::GenericClient; @@ -212,14 +213,33 @@ pub async fn handle( Ok(r) => r, Err(e) => { let message = format!("{:?}", e); - let code = e.downcast_ref::().and_then(|e| { - e.code() - .map(|s| serde_json::to_value(s.code()).unwrap_or_default()) - }); - let code = match code { - Some(c) => c, - None => Value::Null, - }; + let db_error = e + .downcast_ref::() + .and_then(|e| e.as_db_error()); + fn get<'a, T: serde::Serialize>( + db: Option<&'a DbError>, + x: impl FnOnce(&'a DbError) -> T, + ) -> Value { + db.map(x) + .and_then(|t| serde_json::to_value(t).ok()) + .unwrap_or_default() + } + + // TODO(conrad): db_error.position() + let code = get(db_error, |db| db.code().code()); + let severity = get(db_error, |db| db.severity()); + let detail = get(db_error, |db| db.detail()); + let hint = get(db_error, |db| db.hint()); + let where_ = get(db_error, |db| db.where_()); + let table = get(db_error, |db| db.table()); + let column = get(db_error, |db| db.column()); + let schema = get(db_error, |db| db.schema()); + let datatype = get(db_error, |db| db.datatype()); + let constraint = get(db_error, |db| db.constraint()); + let file = get(db_error, |db| db.file()); + let line = get(db_error, |db| db.line()); + let routine = get(db_error, |db| db.routine()); + error!( ?code, "sql-over-http per-client task finished with an error: {e:#}" @@ -227,7 +247,22 @@ pub async fn handle( // TODO: this shouldn't always be bad request. json_response( StatusCode::BAD_REQUEST, - json!({ "message": message, "code": code }), + json!({ + "message": message, + "code": code, + "detail": detail, + "hint": hint, + "severity": severity, + "where": where_, + "table": table, + "column": column, + "schema": schema, + "datatype": datatype, + "constraint": constraint, + "file": file, + "line": line, + "routine": routine, + }), )? } }, From fc77c42c574f7ce023af7022cdafcb6530734605 Mon Sep 17 00:00:00 2001 From: Conrad Ludgate Date: Thu, 30 Nov 2023 10:19:30 +0000 Subject: [PATCH 13/38] proxy: add flag to enable http pool for all users (#5959) ## Problem #5123 ## Summary of changes Add `--sql-over-http-pool-opt-in true` default cli arg. Allows us to set `--sql-over-http-pool-opt-in false` region-by-region --- proxy/src/bin/proxy.rs | 6 +++++- proxy/src/config.rs | 3 ++- proxy/src/serverless/sql_over_http.rs | 10 ++++++---- test_runner/fixtures/neon_fixtures.py | 1 + 4 files changed, 14 insertions(+), 6 deletions(-) diff --git a/proxy/src/bin/proxy.rs b/proxy/src/bin/proxy.rs index 570cf0943a..d90ac86a82 100644 --- a/proxy/src/bin/proxy.rs +++ b/proxy/src/bin/proxy.rs @@ -90,6 +90,9 @@ struct ProxyCliArgs { /// timeout for http connections #[clap(long, default_value = "15s", value_parser = humantime::parse_duration)] sql_over_http_timeout: tokio::time::Duration, + /// Whether the SQL over http pool is opt-in + #[clap(long, default_value_t = true, value_parser = clap::builder::BoolishValueParser::new(), action = clap::ArgAction::Set)] + sql_over_http_pool_opt_in: bool, /// timeout for scram authentication protocol #[clap(long, default_value = "15s", value_parser = humantime::parse_duration)] scram_protocol_timeout: tokio::time::Duration, @@ -275,7 +278,8 @@ fn build_config(args: &ProxyCliArgs) -> anyhow::Result<&'static ProxyConfig> { } }; let http_config = HttpConfig { - sql_over_http_timeout: args.sql_over_http_timeout, + timeout: args.sql_over_http_timeout, + pool_opt_in: args.sql_over_http_pool_opt_in, }; let authentication_config = AuthenticationConfig { scram_protocol_timeout: args.scram_protocol_timeout, diff --git a/proxy/src/config.rs b/proxy/src/config.rs index 0c094ff4aa..89b432df92 100644 --- a/proxy/src/config.rs +++ b/proxy/src/config.rs @@ -34,7 +34,8 @@ pub struct TlsConfig { } pub struct HttpConfig { - pub sql_over_http_timeout: tokio::time::Duration, + pub timeout: tokio::time::Duration, + pub pool_opt_in: bool, } pub struct AuthenticationConfig { diff --git a/proxy/src/serverless/sql_over_http.rs b/proxy/src/serverless/sql_over_http.rs index b449a6e471..2df2be1d3d 100644 --- a/proxy/src/serverless/sql_over_http.rs +++ b/proxy/src/serverless/sql_over_http.rs @@ -204,8 +204,8 @@ pub async fn handle( config: &'static HttpConfig, ) -> Result, ApiError> { let result = tokio::time::timeout( - config.sql_over_http_timeout, - handle_inner(request, sni_hostname, conn_pool, session_id), + config.timeout, + handle_inner(config, request, sni_hostname, conn_pool, session_id), ) .await; let mut response = match result { @@ -269,7 +269,7 @@ pub async fn handle( Err(_) => { let message = format!( "HTTP-Connection timed out, execution time exeeded {} seconds", - config.sql_over_http_timeout.as_secs() + config.timeout.as_secs() ); error!(message); json_response( @@ -287,6 +287,7 @@ pub async fn handle( #[instrument(name = "sql-over-http", fields(pid = tracing::field::Empty), skip_all)] async fn handle_inner( + config: &'static HttpConfig, request: Request, sni_hostname: Option, conn_pool: Arc, @@ -311,7 +312,8 @@ async fn handle_inner( let array_mode = headers.get(&ARRAY_MODE) == Some(&HEADER_VALUE_TRUE); // Allow connection pooling only if explicitly requested - let allow_pool = headers.get(&ALLOW_POOL) == Some(&HEADER_VALUE_TRUE); + // or if we have decided that http pool is no longer opt-in + let allow_pool = !config.pool_opt_in || headers.get(&ALLOW_POOL) == Some(&HEADER_VALUE_TRUE); // isolation level, read only and deferrable diff --git a/test_runner/fixtures/neon_fixtures.py b/test_runner/fixtures/neon_fixtures.py index 214bcc3a53..23a36ad6c9 100644 --- a/test_runner/fixtures/neon_fixtures.py +++ b/test_runner/fixtures/neon_fixtures.py @@ -2114,6 +2114,7 @@ class NeonProxy(PgProtocol): # Console auth backend params *["--auth-backend", "console"], *["--auth-endpoint", self.endpoint], + *["--sql-over-http-pool-opt-in", "false"], ] if self.fixed_rate_limit is not None: args += [ From 50d959fddc4e4133fbf48ed187fdfdd456f5ad30 Mon Sep 17 00:00:00 2001 From: Rahul Modpur Date: Thu, 30 Nov 2023 16:17:13 +0530 Subject: [PATCH 14/38] refactor: use serde for TenantConf deserialization Fixes: #5300 (#5310) Remove handcrafted TenantConf deserialization code. Use `serde_path_to_error` to include the field which failed parsing. Leaves the duplicated TenantConf in pageserver and models, does not touch PageserverConf handcrafted deserialization. Error change: - before change: "configure option `checkpoint_distance` cannot be negative" - after change: "`checkpoint_distance`: invalid value: integer `-1`, expected u64" Fixes: #5300 Cc: #3682 --------- Signed-off-by: Rahul Modpur Co-authored-by: Shany Pozin Co-authored-by: Joonas Koivunen --- Cargo.lock | 1 + Cargo.toml | 1 + libs/pageserver_api/src/models.rs | 20 +--- pageserver/Cargo.toml | 1 + pageserver/src/config.rs | 138 +++++++-------------------- pageserver/src/tenant.rs | 4 +- pageserver/src/tenant/config.rs | 152 +++++++++++++----------------- 7 files changed, 103 insertions(+), 214 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index dc463942ed..23c64cf0e9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2977,6 +2977,7 @@ dependencies = [ "scopeguard", "serde", "serde_json", + "serde_path_to_error", "serde_with", "signal-hook", "smallvec", diff --git a/Cargo.toml b/Cargo.toml index 6df48ffc55..ab00c11dda 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -126,6 +126,7 @@ sd-notify = "0.4.1" sentry = { version = "0.31", default-features = false, features = ["backtrace", "contexts", "panic", "rustls", "reqwest" ] } serde = { version = "1.0", features = ["derive"] } serde_json = "1" +serde_path_to_error = "0.1" serde_with = "2.0" serde_assert = "0.5.0" sha2 = "0.10.2" diff --git a/libs/pageserver_api/src/models.rs b/libs/pageserver_api/src/models.rs index c792a5eff7..60c508037e 100644 --- a/libs/pageserver_api/src/models.rs +++ b/libs/pageserver_api/src/models.rs @@ -314,25 +314,7 @@ impl std::ops::Deref for TenantConfigRequest { impl TenantConfigRequest { pub fn new(tenant_id: TenantId) -> TenantConfigRequest { - let config = TenantConfig { - checkpoint_distance: None, - checkpoint_timeout: None, - compaction_target_size: None, - compaction_period: None, - compaction_threshold: None, - gc_horizon: None, - gc_period: None, - image_creation_threshold: None, - pitr_interval: None, - walreceiver_connect_timeout: None, - lagging_wal_timeout: None, - max_lsn_wal_lag: None, - trace_read_requests: None, - eviction_policy: None, - min_resident_size_override: None, - evictions_low_residence_duration_metric_threshold: None, - gc_feedback: None, - }; + let config = TenantConfig::default(); TenantConfigRequest { tenant_id, config } } } diff --git a/pageserver/Cargo.toml b/pageserver/Cargo.toml index 3eb01003df..35c260740c 100644 --- a/pageserver/Cargo.toml +++ b/pageserver/Cargo.toml @@ -51,6 +51,7 @@ regex.workspace = true scopeguard.workspace = true serde.workspace = true serde_json = { workspace = true, features = ["raw_value"] } +serde_path_to_error.workspace = true serde_with.workspace = true signal-hook.workspace = true smallvec = { workspace = true, features = ["write"] } diff --git a/pageserver/src/config.rs b/pageserver/src/config.rs index 5b170af4ef..737495d414 100644 --- a/pageserver/src/config.rs +++ b/pageserver/src/config.rs @@ -779,7 +779,7 @@ impl PageServerConf { builder.remote_storage_config(RemoteStorageConfig::from_toml(item)?) } "tenant_config" => { - t_conf = Self::parse_toml_tenant_conf(item)?; + t_conf = TenantConfOpt::try_from(item.to_owned()).context(format!("failed to parse: '{key}'"))?; } "id" => builder.id(NodeId(parse_toml_u64(key, item)?)), "broker_endpoint" => builder.broker_endpoint(parse_toml_string(key, item)?.parse().context("failed to parse broker endpoint")?), @@ -853,111 +853,6 @@ impl PageServerConf { Ok(conf) } - // subroutine of parse_and_validate to parse `[tenant_conf]` section - - pub fn parse_toml_tenant_conf(item: &toml_edit::Item) -> Result { - let mut t_conf: TenantConfOpt = Default::default(); - if let Some(checkpoint_distance) = item.get("checkpoint_distance") { - t_conf.checkpoint_distance = - Some(parse_toml_u64("checkpoint_distance", checkpoint_distance)?); - } - - if let Some(checkpoint_timeout) = item.get("checkpoint_timeout") { - t_conf.checkpoint_timeout = Some(parse_toml_duration( - "checkpoint_timeout", - checkpoint_timeout, - )?); - } - - if let Some(compaction_target_size) = item.get("compaction_target_size") { - t_conf.compaction_target_size = Some(parse_toml_u64( - "compaction_target_size", - compaction_target_size, - )?); - } - - if let Some(compaction_period) = item.get("compaction_period") { - t_conf.compaction_period = - Some(parse_toml_duration("compaction_period", compaction_period)?); - } - - if let Some(compaction_threshold) = item.get("compaction_threshold") { - t_conf.compaction_threshold = - Some(parse_toml_u64("compaction_threshold", compaction_threshold)?.try_into()?); - } - - if let Some(image_creation_threshold) = item.get("image_creation_threshold") { - t_conf.image_creation_threshold = Some( - parse_toml_u64("image_creation_threshold", image_creation_threshold)?.try_into()?, - ); - } - - if let Some(gc_horizon) = item.get("gc_horizon") { - t_conf.gc_horizon = Some(parse_toml_u64("gc_horizon", gc_horizon)?); - } - - if let Some(gc_period) = item.get("gc_period") { - t_conf.gc_period = Some(parse_toml_duration("gc_period", gc_period)?); - } - - if let Some(pitr_interval) = item.get("pitr_interval") { - t_conf.pitr_interval = Some(parse_toml_duration("pitr_interval", pitr_interval)?); - } - if let Some(walreceiver_connect_timeout) = item.get("walreceiver_connect_timeout") { - t_conf.walreceiver_connect_timeout = Some(parse_toml_duration( - "walreceiver_connect_timeout", - walreceiver_connect_timeout, - )?); - } - if let Some(lagging_wal_timeout) = item.get("lagging_wal_timeout") { - t_conf.lagging_wal_timeout = Some(parse_toml_duration( - "lagging_wal_timeout", - lagging_wal_timeout, - )?); - } - if let Some(max_lsn_wal_lag) = item.get("max_lsn_wal_lag") { - t_conf.max_lsn_wal_lag = - Some(deserialize_from_item("max_lsn_wal_lag", max_lsn_wal_lag)?); - } - if let Some(trace_read_requests) = item.get("trace_read_requests") { - t_conf.trace_read_requests = - Some(trace_read_requests.as_bool().with_context(|| { - "configure option trace_read_requests is not a bool".to_string() - })?); - } - - if let Some(eviction_policy) = item.get("eviction_policy") { - t_conf.eviction_policy = Some( - deserialize_from_item("eviction_policy", eviction_policy) - .context("parse eviction_policy")?, - ); - } - - if let Some(item) = item.get("min_resident_size_override") { - t_conf.min_resident_size_override = Some( - deserialize_from_item("min_resident_size_override", item) - .context("parse min_resident_size_override")?, - ); - } - - if let Some(item) = item.get("evictions_low_residence_duration_metric_threshold") { - t_conf.evictions_low_residence_duration_metric_threshold = Some(parse_toml_duration( - "evictions_low_residence_duration_metric_threshold", - item, - )?); - } - - if let Some(gc_feedback) = item.get("gc_feedback") { - t_conf.gc_feedback = Some( - gc_feedback - .as_bool() - .with_context(|| "configure option gc_feedback is not a bool".to_string())?, - ); - } - - Ok(t_conf) - } - #[cfg(test)] pub fn test_repo_dir(test_name: &str) -> Utf8PathBuf { Utf8PathBuf::from(format!("../tmp_check/test_{test_name}")) @@ -1429,6 +1324,37 @@ trace_read_requests = {trace_read_requests}"#, Ok(()) } + #[test] + fn parse_incorrect_tenant_config() -> anyhow::Result<()> { + let config_string = r#" + [tenant_config] + checkpoint_distance = -1 # supposed to be an u64 + "# + .to_string(); + + let toml: Document = config_string.parse()?; + let item = toml.get("tenant_config").unwrap(); + let error = TenantConfOpt::try_from(item.to_owned()).unwrap_err(); + + let expected_error_str = "checkpoint_distance: invalid value: integer `-1`, expected u64"; + assert_eq!(error.to_string(), expected_error_str); + + Ok(()) + } + + #[test] + fn parse_override_tenant_config() -> anyhow::Result<()> { + let config_string = r#"tenant_config={ min_resident_size_override = 400 }"#.to_string(); + + let toml: Document = config_string.parse()?; + let item = toml.get("tenant_config").unwrap(); + let conf = TenantConfOpt::try_from(item.to_owned()).unwrap(); + + assert_eq!(conf.min_resident_size_override, Some(400)); + + Ok(()) + } + #[test] fn eviction_pageserver_config_parse() -> anyhow::Result<()> { let tempdir = tempdir()?; diff --git a/pageserver/src/tenant.rs b/pageserver/src/tenant.rs index fe3a5bfb79..2a63f193e3 100644 --- a/pageserver/src/tenant.rs +++ b/pageserver/src/tenant.rs @@ -2442,9 +2442,7 @@ impl Tenant { for (key, item) in deserialized.iter() { match key { "tenant_config" => { - tenant_conf = PageServerConf::parse_toml_tenant_conf(item).with_context(|| { - format!("Failed to parse config from file '{legacy_config_path}' as pageserver config") - })?; + tenant_conf = TenantConfOpt::try_from(item.to_owned()).context(format!("Failed to parse config from file '{legacy_config_path}' as pageserver config"))?; } _ => bail!( "config file {legacy_config_path} has unrecognized pageserver option '{key}'" diff --git a/pageserver/src/tenant/config.rs b/pageserver/src/tenant/config.rs index 4ad6a71f67..7a454b53d2 100644 --- a/pageserver/src/tenant/config.rs +++ b/pageserver/src/tenant/config.rs @@ -8,10 +8,12 @@ //! We cannot use global or default config instead, because wrong settings //! may lead to a data loss. //! -use anyhow::Context; +use anyhow::bail; use pageserver_api::models; use pageserver_api::shard::{ShardCount, ShardIdentity, ShardNumber, ShardStripeSize}; +use serde::de::IntoDeserializer; use serde::{Deserialize, Serialize}; +use serde_json::Value; use std::num::NonZeroU64; use std::time::Duration; use utils::generation::Generation; @@ -521,105 +523,49 @@ impl Default for TenantConf { } } -// Helper function to standardize the error messages we produce on bad durations -// -// Intended to be used with anyhow's `with_context`, e.g.: -// -// let value = result.with_context(bad_duration("name", &value))?; -// -fn bad_duration<'a>(field_name: &'static str, value: &'a str) -> impl 'a + Fn() -> String { - move || format!("Cannot parse `{field_name}` duration {value:?}") -} - impl TryFrom<&'_ models::TenantConfig> for TenantConfOpt { type Error = anyhow::Error; fn try_from(request_data: &'_ models::TenantConfig) -> Result { - let mut tenant_conf = TenantConfOpt::default(); + // Convert the request_data to a JSON Value + let json_value: Value = serde_json::to_value(request_data)?; - if let Some(gc_period) = &request_data.gc_period { - tenant_conf.gc_period = Some( - humantime::parse_duration(gc_period) - .with_context(bad_duration("gc_period", gc_period))?, - ); - } - tenant_conf.gc_horizon = request_data.gc_horizon; - tenant_conf.image_creation_threshold = request_data.image_creation_threshold; + // Create a Deserializer from the JSON Value + let deserializer = json_value.into_deserializer(); - if let Some(pitr_interval) = &request_data.pitr_interval { - tenant_conf.pitr_interval = Some( - humantime::parse_duration(pitr_interval) - .with_context(bad_duration("pitr_interval", pitr_interval))?, - ); - } - - if let Some(walreceiver_connect_timeout) = &request_data.walreceiver_connect_timeout { - tenant_conf.walreceiver_connect_timeout = Some( - humantime::parse_duration(walreceiver_connect_timeout).with_context( - bad_duration("walreceiver_connect_timeout", walreceiver_connect_timeout), - )?, - ); - } - if let Some(lagging_wal_timeout) = &request_data.lagging_wal_timeout { - tenant_conf.lagging_wal_timeout = Some( - humantime::parse_duration(lagging_wal_timeout) - .with_context(bad_duration("lagging_wal_timeout", lagging_wal_timeout))?, - ); - } - if let Some(max_lsn_wal_lag) = request_data.max_lsn_wal_lag { - tenant_conf.max_lsn_wal_lag = Some(max_lsn_wal_lag); - } - if let Some(trace_read_requests) = request_data.trace_read_requests { - tenant_conf.trace_read_requests = Some(trace_read_requests); - } - - tenant_conf.checkpoint_distance = request_data.checkpoint_distance; - if let Some(checkpoint_timeout) = &request_data.checkpoint_timeout { - tenant_conf.checkpoint_timeout = Some( - humantime::parse_duration(checkpoint_timeout) - .with_context(bad_duration("checkpoint_timeout", checkpoint_timeout))?, - ); - } - - tenant_conf.compaction_target_size = request_data.compaction_target_size; - tenant_conf.compaction_threshold = request_data.compaction_threshold; - - if let Some(compaction_period) = &request_data.compaction_period { - tenant_conf.compaction_period = Some( - humantime::parse_duration(compaction_period) - .with_context(bad_duration("compaction_period", compaction_period))?, - ); - } - - if let Some(eviction_policy) = &request_data.eviction_policy { - tenant_conf.eviction_policy = Some( - serde::Deserialize::deserialize(eviction_policy) - .context("parse field `eviction_policy`")?, - ); - } - - tenant_conf.min_resident_size_override = request_data.min_resident_size_override; - - if let Some(evictions_low_residence_duration_metric_threshold) = - &request_data.evictions_low_residence_duration_metric_threshold - { - tenant_conf.evictions_low_residence_duration_metric_threshold = Some( - humantime::parse_duration(evictions_low_residence_duration_metric_threshold) - .with_context(bad_duration( - "evictions_low_residence_duration_metric_threshold", - evictions_low_residence_duration_metric_threshold, - ))?, - ); - } - tenant_conf.gc_feedback = request_data.gc_feedback; + // Use serde_path_to_error to deserialize the JSON Value into TenantConfOpt + let tenant_conf: TenantConfOpt = serde_path_to_error::deserialize(deserializer)?; Ok(tenant_conf) } } +impl TryFrom for TenantConfOpt { + type Error = anyhow::Error; + + fn try_from(item: toml_edit::Item) -> Result { + match item { + toml_edit::Item::Value(value) => { + let d = value.into_deserializer(); + return serde_path_to_error::deserialize(d) + .map_err(|e| anyhow::anyhow!("{}: {}", e.path(), e.inner().message())); + } + toml_edit::Item::Table(table) => { + let deserializer = toml_edit::de::Deserializer::new(table.into()); + return serde_path_to_error::deserialize(deserializer) + .map_err(|e| anyhow::anyhow!("{}: {}", e.path(), e.inner().message())); + } + _ => { + bail!("expected non-inline table but found {item}") + } + } + } +} + #[cfg(test)] mod tests { use super::*; + use models::TenantConfig; #[test] fn de_serializing_pageserver_config_omits_empty_values() { @@ -636,4 +582,38 @@ mod tests { assert_eq!(json_form, "{\"gc_horizon\":42}"); assert_eq!(small_conf, serde_json::from_str(&json_form).unwrap()); } + + #[test] + fn test_try_from_models_tenant_config_err() { + let tenant_config = models::TenantConfig { + lagging_wal_timeout: Some("5a".to_string()), + ..TenantConfig::default() + }; + + let tenant_conf_opt = TenantConfOpt::try_from(&tenant_config); + + assert!( + tenant_conf_opt.is_err(), + "Suceeded to convert TenantConfig to TenantConfOpt" + ); + + let expected_error_str = + "lagging_wal_timeout: invalid value: string \"5a\", expected a duration"; + assert_eq!(tenant_conf_opt.unwrap_err().to_string(), expected_error_str); + } + + #[test] + fn test_try_from_models_tenant_config_success() { + let tenant_config = models::TenantConfig { + lagging_wal_timeout: Some("5s".to_string()), + ..TenantConfig::default() + }; + + let tenant_conf_opt = TenantConfOpt::try_from(&tenant_config).unwrap(); + + assert_eq!( + tenant_conf_opt.lagging_wal_timeout, + Some(Duration::from_secs(5)) + ); + } } From 292281c9dfb24152b728b1a846cc45105dac7fe0 Mon Sep 17 00:00:00 2001 From: Christian Schwarz Date: Thu, 30 Nov 2023 12:34:30 +0100 Subject: [PATCH 15/38] pagectl: add subcommand to rewrite layer file summary (#5933) Part of getpage@lsn benchmark epic: https://github.com/neondatabase/neon/issues/5771 --- pageserver/ctl/src/layers.rs | 72 +++++++++++++++++- pageserver/src/tenant/storage_layer.rs | 2 +- .../src/tenant/storage_layer/delta_layer.rs | 67 +++++++++++++++-- .../src/tenant/storage_layer/image_layer.rs | 73 ++++++++++++++++--- 4 files changed, 196 insertions(+), 18 deletions(-) diff --git a/pageserver/ctl/src/layers.rs b/pageserver/ctl/src/layers.rs index 22ebe70b16..ebf4a4bec3 100644 --- a/pageserver/ctl/src/layers.rs +++ b/pageserver/ctl/src/layers.rs @@ -1,13 +1,15 @@ use std::path::{Path, PathBuf}; use anyhow::Result; -use camino::Utf8Path; +use camino::{Utf8Path, Utf8PathBuf}; use clap::Subcommand; use pageserver::context::{DownloadBehavior, RequestContext}; use pageserver::task_mgr::TaskKind; use pageserver::tenant::block_io::BlockCursor; use pageserver::tenant::disk_btree::DiskBtreeReader; use pageserver::tenant::storage_layer::delta_layer::{BlobRef, Summary}; +use pageserver::tenant::storage_layer::{delta_layer, image_layer}; +use pageserver::tenant::storage_layer::{DeltaLayer, ImageLayer}; use pageserver::tenant::{TENANTS_SEGMENT_NAME, TIMELINES_SEGMENT_NAME}; use pageserver::{page_cache, virtual_file}; use pageserver::{ @@ -20,6 +22,7 @@ use pageserver::{ }; use std::fs; use utils::bin_ser::BeSer; +use utils::id::{TenantId, TimelineId}; use crate::layer_map_analyzer::parse_filename; @@ -45,6 +48,13 @@ pub(crate) enum LayerCmd { /// The id from list-layer command id: usize, }, + RewriteSummary { + layer_file_path: Utf8PathBuf, + #[clap(long)] + new_tenant_id: Option, + #[clap(long)] + new_timeline_id: Option, + }, } async fn read_delta_file(path: impl AsRef, ctx: &RequestContext) -> Result<()> { @@ -100,6 +110,7 @@ pub(crate) async fn main(cmd: &LayerCmd) -> Result<()> { println!("- timeline {}", timeline.file_name().to_string_lossy()); } } + Ok(()) } LayerCmd::ListLayer { path, @@ -128,6 +139,7 @@ pub(crate) async fn main(cmd: &LayerCmd) -> Result<()> { idx += 1; } } + Ok(()) } LayerCmd::DumpLayer { path, @@ -168,7 +180,63 @@ pub(crate) async fn main(cmd: &LayerCmd) -> Result<()> { idx += 1; } } + Ok(()) + } + LayerCmd::RewriteSummary { + layer_file_path, + new_tenant_id, + new_timeline_id, + } => { + pageserver::virtual_file::init(10); + pageserver::page_cache::init(100); + + let ctx = RequestContext::new(TaskKind::DebugTool, DownloadBehavior::Error); + + macro_rules! rewrite_closure { + ($($summary_ty:tt)*) => {{ + |summary| $($summary_ty)* { + tenant_id: new_tenant_id.unwrap_or(summary.tenant_id), + timeline_id: new_timeline_id.unwrap_or(summary.timeline_id), + ..summary + } + }}; + } + + let res = ImageLayer::rewrite_summary( + layer_file_path, + rewrite_closure!(image_layer::Summary), + &ctx, + ) + .await; + match res { + Ok(()) => { + println!("Successfully rewrote summary of image layer {layer_file_path}"); + return Ok(()); + } + Err(image_layer::RewriteSummaryError::MagicMismatch) => (), // fallthrough + Err(image_layer::RewriteSummaryError::Other(e)) => { + return Err(e); + } + } + + let res = DeltaLayer::rewrite_summary( + layer_file_path, + rewrite_closure!(delta_layer::Summary), + &ctx, + ) + .await; + match res { + Ok(()) => { + println!("Successfully rewrote summary of delta layer {layer_file_path}"); + return Ok(()); + } + Err(delta_layer::RewriteSummaryError::MagicMismatch) => (), // fallthrough + Err(delta_layer::RewriteSummaryError::Other(e)) => { + return Err(e); + } + } + + anyhow::bail!("not an image or delta layer: {layer_file_path}"); } } - Ok(()) } diff --git a/pageserver/src/tenant/storage_layer.rs b/pageserver/src/tenant/storage_layer.rs index c933342822..944e05883f 100644 --- a/pageserver/src/tenant/storage_layer.rs +++ b/pageserver/src/tenant/storage_layer.rs @@ -2,7 +2,7 @@ pub mod delta_layer; mod filename; -mod image_layer; +pub mod image_layer; mod inmemory_layer; mod layer; mod layer_desc; diff --git a/pageserver/src/tenant/storage_layer/delta_layer.rs b/pageserver/src/tenant/storage_layer/delta_layer.rs index e9886d90c4..d339204127 100644 --- a/pageserver/src/tenant/storage_layer/delta_layer.rs +++ b/pageserver/src/tenant/storage_layer/delta_layer.rs @@ -70,13 +70,13 @@ use super::{AsLayerDesc, LayerAccessStats, PersistentLayerDesc, ResidentLayer}; #[derive(Debug, Serialize, Deserialize, PartialEq, Eq)] pub struct Summary { /// Magic value to identify this as a neon delta file. Always DELTA_FILE_MAGIC. - magic: u16, - format_version: u16, + pub magic: u16, + pub format_version: u16, - tenant_id: TenantId, - timeline_id: TimelineId, - key_range: Range, - lsn_range: Range, + pub tenant_id: TenantId, + pub timeline_id: TimelineId, + pub key_range: Range, + pub lsn_range: Range, /// Block number where the 'index' part of the file begins. pub index_start_blk: u32, @@ -624,6 +624,61 @@ impl Drop for DeltaLayerWriter { } } +#[derive(thiserror::Error, Debug)] +pub enum RewriteSummaryError { + #[error("magic mismatch")] + MagicMismatch, + #[error(transparent)] + Other(#[from] anyhow::Error), +} + +impl From for RewriteSummaryError { + fn from(e: std::io::Error) -> Self { + Self::Other(anyhow::anyhow!(e)) + } +} + +impl DeltaLayer { + pub async fn rewrite_summary( + path: &Utf8Path, + rewrite: F, + ctx: &RequestContext, + ) -> Result<(), RewriteSummaryError> + where + F: Fn(Summary) -> Summary, + { + let file = VirtualFile::open_with_options( + path, + &*std::fs::OpenOptions::new().read(true).write(true), + ) + .await + .with_context(|| format!("Failed to open file '{}'", path))?; + let file = FileBlockReader::new(file); + let summary_blk = file.read_blk(0, ctx).await?; + let actual_summary = Summary::des_prefix(summary_blk.as_ref()).context("deserialize")?; + let mut file = file.file; + if actual_summary.magic != DELTA_FILE_MAGIC { + return Err(RewriteSummaryError::MagicMismatch); + } + + let new_summary = rewrite(actual_summary); + + let mut buf = smallvec::SmallVec::<[u8; PAGE_SZ]>::new(); + Summary::ser_into(&new_summary, &mut buf).context("serialize")?; + if buf.spilled() { + // The code in DeltaLayerWriterInner just warn!()s for this. + // It should probably error out as well. + return Err(RewriteSummaryError::Other(anyhow::anyhow!( + "Used more than one page size for summary buffer: {}", + buf.len() + ))); + } + file.seek(SeekFrom::Start(0)).await?; + file.write_all(&buf).await?; + Ok(()) + } +} + impl DeltaLayerInner { /// Returns nested result following Result, Critical>: /// - inner has the success or transient failure diff --git a/pageserver/src/tenant/storage_layer/image_layer.rs b/pageserver/src/tenant/storage_layer/image_layer.rs index 208aa07872..023122c0b1 100644 --- a/pageserver/src/tenant/storage_layer/image_layer.rs +++ b/pageserver/src/tenant/storage_layer/image_layer.rs @@ -68,20 +68,20 @@ use super::{AsLayerDesc, Layer, PersistentLayerDesc, ResidentLayer}; /// the 'index' starts at the block indicated by 'index_start_blk' /// #[derive(Debug, Serialize, Deserialize, PartialEq, Eq)] -pub(super) struct Summary { +pub struct Summary { /// Magic value to identify this as a neon image file. Always IMAGE_FILE_MAGIC. - magic: u16, - format_version: u16, + pub magic: u16, + pub format_version: u16, - tenant_id: TenantId, - timeline_id: TimelineId, - key_range: Range, - lsn: Lsn, + pub tenant_id: TenantId, + pub timeline_id: TimelineId, + pub key_range: Range, + pub lsn: Lsn, /// Block number where the 'index' part of the file begins. - index_start_blk: u32, + pub index_start_blk: u32, /// Block within the 'index', where the B-tree root page is stored - index_root_blk: u32, + pub index_root_blk: u32, // the 'values' part starts after the summary header, on block 1. } @@ -302,6 +302,61 @@ impl ImageLayer { } } +#[derive(thiserror::Error, Debug)] +pub enum RewriteSummaryError { + #[error("magic mismatch")] + MagicMismatch, + #[error(transparent)] + Other(#[from] anyhow::Error), +} + +impl From for RewriteSummaryError { + fn from(e: std::io::Error) -> Self { + Self::Other(anyhow::anyhow!(e)) + } +} + +impl ImageLayer { + pub async fn rewrite_summary( + path: &Utf8Path, + rewrite: F, + ctx: &RequestContext, + ) -> Result<(), RewriteSummaryError> + where + F: Fn(Summary) -> Summary, + { + let file = VirtualFile::open_with_options( + path, + &*std::fs::OpenOptions::new().read(true).write(true), + ) + .await + .with_context(|| format!("Failed to open file '{}'", path))?; + let file = FileBlockReader::new(file); + let summary_blk = file.read_blk(0, ctx).await?; + let actual_summary = Summary::des_prefix(summary_blk.as_ref()).context("deserialize")?; + let mut file = file.file; + if actual_summary.magic != IMAGE_FILE_MAGIC { + return Err(RewriteSummaryError::MagicMismatch); + } + + let new_summary = rewrite(actual_summary); + + let mut buf = smallvec::SmallVec::<[u8; PAGE_SZ]>::new(); + Summary::ser_into(&new_summary, &mut buf).context("serialize")?; + if buf.spilled() { + // The code in ImageLayerWriterInner just warn!()s for this. + // It should probably error out as well. + return Err(RewriteSummaryError::Other(anyhow::anyhow!( + "Used more than one page size for summary buffer: {}", + buf.len() + ))); + } + file.seek(SeekFrom::Start(0)).await?; + file.write_all(&buf).await?; + Ok(()) + } +} + impl ImageLayerInner { /// Returns nested result following Result, Critical>: /// - inner has the success or transient failure From 3e094e90d7ccad2ab63c5d8662d1010b5dc3562a Mon Sep 17 00:00:00 2001 From: John Khvatov Date: Thu, 30 Nov 2023 15:17:58 +0300 Subject: [PATCH 16/38] update aws sdk to 1.0.x (#5976) This change will be useful for experimenting with S3 performance. Co-authored-by: Joonas Koivunen --- Cargo.lock | 451 +++++++++++++++++++-------- Cargo.toml | 11 +- libs/remote_storage/Cargo.toml | 3 +- libs/remote_storage/src/s3_bucket.rs | 35 ++- s3_scrubber/Cargo.toml | 4 +- s3_scrubber/src/checks.rs | 10 +- s3_scrubber/src/garbage.rs | 2 +- s3_scrubber/src/lib.rs | 2 + s3_scrubber/src/metadata_stream.rs | 14 +- workspace_hack/Cargo.toml | 16 +- 10 files changed, 365 insertions(+), 183 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 23c64cf0e9..3f0f21eb4a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -256,19 +256,21 @@ checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" [[package]] name = "aws-config" -version = "0.56.0" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de3d533e0263bf453cc80af4c8bcc4d64e2aca293bd16f81633a36f1bf4a97cb" +checksum = "80c950a809d39bc9480207cb1cfc879ace88ea7e3a4392a8e9999e45d6e5692e" dependencies = [ "aws-credential-types", "aws-http", + "aws-runtime", "aws-sdk-sso", + "aws-sdk-ssooidc", "aws-sdk-sts", "aws-smithy-async", - "aws-smithy-client", "aws-smithy-http", - "aws-smithy-http-tower", "aws-smithy-json", + "aws-smithy-runtime", + "aws-smithy-runtime-api", "aws-smithy-types", "aws-types", "bytes", @@ -276,52 +278,46 @@ dependencies = [ "hex", "http", "hyper", - "ring", + "ring 0.17.6", "time", "tokio", - "tower", "tracing", "zeroize", ] [[package]] name = "aws-credential-types" -version = "0.56.0" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e4834ba01c5ad1ed9740aa222de62190e3c565d11ab7e72cc68314a258994567" +checksum = "8c1317e1a3514b103cf7d5828bbab3b4d30f56bd22d684f8568bc51b6cfbbb1c" dependencies = [ "aws-smithy-async", + "aws-smithy-runtime-api", "aws-smithy-types", - "fastrand 2.0.0", - "tokio", - "tracing", "zeroize", ] [[package]] name = "aws-http" -version = "0.56.0" +version = "0.60.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72badf9de83cc7d66b21b004f09241836823b8302afb25a24708769e576a8d8f" +checksum = "361c4310fdce94328cc2d1ca0c8a48c13f43009c61d3367585685a50ca8c66b6" dependencies = [ - "aws-credential-types", - "aws-smithy-http", + "aws-smithy-runtime-api", "aws-smithy-types", "aws-types", "bytes", "http", "http-body", - "lazy_static", - "percent-encoding", "pin-project-lite", "tracing", ] [[package]] name = "aws-runtime" -version = "0.56.0" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf832f522111225c02547e1e1c28137e840e4b082399d93a236e4b29193a4667" +checksum = "1ed7ef604a15fd0d4d9e43701295161ea6b504b63c44990ead352afea2bc15e9" dependencies = [ "aws-credential-types", "aws-http", @@ -341,9 +337,9 @@ dependencies = [ [[package]] name = "aws-sdk-s3" -version = "0.29.0" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e30370b61599168d38190ad272bb91842cd81870a6ca035c05dd5726d22832c" +checksum = "9dcafc2fe52cc30b2d56685e2fa6a879ba50d79704594852112337a472ddbd24" dependencies = [ "aws-credential-types", "aws-http", @@ -351,7 +347,6 @@ dependencies = [ "aws-sigv4", "aws-smithy-async", "aws-smithy-checksums", - "aws-smithy-client", "aws-smithy-eventstream", "aws-smithy-http", "aws-smithy-json", @@ -366,22 +361,20 @@ dependencies = [ "once_cell", "percent-encoding", "regex", - "tokio-stream", "tracing", "url", ] [[package]] name = "aws-sdk-sso" -version = "0.29.0" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f41bf2c28d32dbb9894a8fcfcb148265d034d3f4a170552a47553a09de890895" +checksum = "0619ab97a5ca8982e7de073cdc66f93e5f6a1b05afc09e696bec1cb3607cd4df" dependencies = [ "aws-credential-types", "aws-http", "aws-runtime", "aws-smithy-async", - "aws-smithy-client", "aws-smithy-http", "aws-smithy-json", "aws-smithy-runtime", @@ -391,21 +384,41 @@ dependencies = [ "bytes", "http", "regex", - "tokio-stream", "tracing", ] [[package]] -name = "aws-sdk-sts" -version = "0.29.0" +name = "aws-sdk-ssooidc" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "79e21aa1a5b0853969a1ef96ccfaa8ff5d57c761549786a4d5f86c1902b2586a" +checksum = "f04b9f5474cc0f35d829510b2ec8c21e352309b46bf9633c5a81fb9321e9b1c7" +dependencies = [ + "aws-credential-types", + "aws-http", + "aws-runtime", + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-types", + "bytes", + "http", + "regex", + "tracing", +] + +[[package]] +name = "aws-sdk-sts" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5700da387716ccfc30b27f44b008f457e1baca5b0f05b6b95455778005e3432a" dependencies = [ "aws-credential-types", "aws-http", "aws-runtime", "aws-smithy-async", - "aws-smithy-client", "aws-smithy-http", "aws-smithy-json", "aws-smithy-query", @@ -421,42 +434,49 @@ dependencies = [ [[package]] name = "aws-sigv4" -version = "0.56.0" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2cb40a93429794065f41f0581734fc56a345f6a38d8e2e3c25c7448d930cd132" +checksum = "380adcc8134ad8bbdfeb2ace7626a869914ee266322965276cbc54066186d236" dependencies = [ + "aws-credential-types", "aws-smithy-eventstream", "aws-smithy-http", + "aws-smithy-runtime-api", + "aws-smithy-types", "bytes", + "crypto-bigint 0.5.5", "form_urlencoded", "hex", "hmac", "http", "once_cell", + "p256", "percent-encoding", "regex", + "ring 0.17.6", "sha2", + "subtle", "time", "tracing", + "zeroize", ] [[package]] name = "aws-smithy-async" -version = "0.56.0" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ee6d17d487c8b579423067718b3580c0908d0f01d7461813f94ec4323bad623" +checksum = "3e37ca17d25fe1e210b6d4bdf59b81caebfe99f986201a1228cb5061233b4b13" dependencies = [ "futures-util", "pin-project-lite", "tokio", - "tokio-stream", ] [[package]] name = "aws-smithy-checksums" -version = "0.56.0" +version = "0.60.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d1849fd5916904513fb0862543b36f8faab43c07984dbc476132b7da1aed056" +checksum = "c5a373ec01aede3dd066ec018c1bc4e8f5dd11b2c11c59c8eef1a5c68101f397" dependencies = [ "aws-smithy-http", "aws-smithy-types", @@ -473,35 +493,11 @@ dependencies = [ "tracing", ] -[[package]] -name = "aws-smithy-client" -version = "0.56.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bdbe0a3ad15283cc5f863a68cb6adc8e256e7c109c43c01bdd09be407219a1e9" -dependencies = [ - "aws-smithy-async", - "aws-smithy-http", - "aws-smithy-http-tower", - "aws-smithy-types", - "bytes", - "fastrand 2.0.0", - "http", - "http-body", - "hyper", - "hyper-rustls", - "lazy_static", - "pin-project-lite", - "rustls", - "tokio", - "tower", - "tracing", -] - [[package]] name = "aws-smithy-eventstream" -version = "0.56.0" +version = "0.60.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a56afef1aa766f512b4970b4c3150b9bf2df8035939723830df4b30267e2d7cb" +checksum = "1c669e1e5fc0d79561bf7a122b118bd50c898758354fe2c53eb8f2d31507cbc3" dependencies = [ "aws-smithy-types", "bytes", @@ -510,57 +506,39 @@ dependencies = [ [[package]] name = "aws-smithy-http" -version = "0.56.0" +version = "0.60.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34dc313472d727f5ef44fdda93e668ebfe17380c99dee512c403e3ca51863bb9" +checksum = "5b1de8aee22f67de467b2e3d0dd0fb30859dc53f579a63bd5381766b987db644" dependencies = [ "aws-smithy-eventstream", + "aws-smithy-runtime-api", "aws-smithy-types", "bytes", "bytes-utils", "futures-core", "http", "http-body", - "hyper", "once_cell", "percent-encoding", "pin-project-lite", "pin-utils", - "tokio", - "tokio-util", - "tracing", -] - -[[package]] -name = "aws-smithy-http-tower" -version = "0.56.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1dd50fca5a4ea4ec3771689ee93bf06b32de02a80af01ed93a8f8a4ed90e8483" -dependencies = [ - "aws-smithy-http", - "aws-smithy-types", - "bytes", - "http", - "http-body", - "pin-project-lite", - "tower", "tracing", ] [[package]] name = "aws-smithy-json" -version = "0.56.0" +version = "0.60.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3591dd7c2fe01ab8025e4847a0a0f6d0c2b2269714688ffb856f9cf6c6d465cf" +checksum = "6a46dd338dc9576d6a6a5b5a19bd678dcad018ececee11cf28ecd7588bd1a55c" dependencies = [ "aws-smithy-types", ] [[package]] name = "aws-smithy-query" -version = "0.56.0" +version = "0.60.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dbabb1145e65dd57ae72d91a2619d3f5fba40b68a5f40ba009c30571dfd60aff" +checksum = "feb5b8c7a86d4b6399169670723b7e6f21a39fc833a30f5c5a2f997608178129" dependencies = [ "aws-smithy-types", "urlencoding", @@ -568,74 +546,86 @@ dependencies = [ [[package]] name = "aws-smithy-runtime" -version = "0.56.0" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3687fb838d4ad1c883b62eb59115bc9fb02c4f308aac49a7df89627067f6eb0d" +checksum = "273479291efc55e7b0bce985b139d86b6031adb8e50f65c1f712f20ba38f6388" dependencies = [ "aws-smithy-async", - "aws-smithy-client", "aws-smithy-http", "aws-smithy-runtime-api", "aws-smithy-types", "bytes", "fastrand 2.0.0", + "h2", "http", "http-body", + "hyper", + "hyper-rustls", "once_cell", "pin-project-lite", "pin-utils", + "rustls", "tokio", "tracing", ] [[package]] name = "aws-smithy-runtime-api" -version = "0.56.0" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5cfbf1e5c2108b41f5ca607cde40dd5109fecc448f5d30c8e614b61f36dce704" +checksum = "c6cebff0d977b6b6feed2fd07db52aac58ba3ccaf26cdd49f1af4add5061bef9" dependencies = [ "aws-smithy-async", - "aws-smithy-http", "aws-smithy-types", "bytes", "http", + "pin-project-lite", "tokio", "tracing", + "zeroize", ] [[package]] name = "aws-smithy-types" -version = "0.56.0" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eed0a94eefd845a2a78677f1b72f02fa75802d38f7f59be675add140279aa8bf" +checksum = "d7f48b3f27ddb40ab19892a5abda331f403e3cb877965e4e51171447807104af" dependencies = [ "base64-simd", + "bytes", + "bytes-utils", + "futures-core", + "http", + "http-body", "itoa", "num-integer", + "pin-project-lite", + "pin-utils", "ryu", "serde", "time", + "tokio", + "tokio-util", ] [[package]] name = "aws-smithy-xml" -version = "0.56.0" +version = "0.60.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c88052c812f696143ad7ba729c63535209ff0e0f49e31a6d2b1205208ea6ea79" +checksum = "0ec40d74a67fd395bc3f6b4ccbdf1543672622d905ef3f979689aea5b730cb95" dependencies = [ "xmlparser", ] [[package]] name = "aws-types" -version = "0.56.0" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6bceb8cf724ad057ad7f327d0d256d7147b3eac777b39849a26189e003dc9782" +checksum = "8403fc56b1f3761e8efe45771ddc1165e47ec3417c68e68a4519b5cb030159ca" dependencies = [ "aws-credential-types", "aws-smithy-async", - "aws-smithy-client", - "aws-smithy-http", + "aws-smithy-runtime-api", "aws-smithy-types", "http", "rustc_version", @@ -705,7 +695,7 @@ dependencies = [ "bytes", "dyn-clone", "futures", - "getrandom 0.2.9", + "getrandom 0.2.11", "http-types", "log", "paste", @@ -799,6 +789,12 @@ dependencies = [ "rustc-demangle", ] +[[package]] +name = "base16ct" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "349a06037c7bf932dd7e7d1f653678b2038b9ad46a74102f1fc7bd7872678cce" + [[package]] name = "base64" version = "0.13.1" @@ -953,11 +949,12 @@ checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" [[package]] name = "cc" -version = "1.0.79" +version = "1.0.83" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50d30906286121d95be3d479533b458f87493b30a4b5f79a607db8f5d11aa91f" +checksum = "f1174fb0b6ec23863f8b971027804a42614e347eafb0a95bf0b12cdae21fc4d0" dependencies = [ "jobserver", + "libc", ] [[package]] @@ -1174,6 +1171,12 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "const-oid" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28c122c3980598d243d63d9a704629a2d748d101f278052ff068be5a4423ab6f" + [[package]] name = "const_fn" version = "0.4.9" @@ -1401,6 +1404,28 @@ dependencies = [ "winapi", ] +[[package]] +name = "crypto-bigint" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef2b4b23cddf68b89b8f8069890e8c270d54e2d5fe1b143820234805e4cb17ef" +dependencies = [ + "generic-array", + "rand_core 0.6.4", + "subtle", + "zeroize", +] + +[[package]] +name = "crypto-bigint" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0dc92fb57ca44df6db8059111ab3af99a63d5d0f8375d9972e319a379c6bab76" +dependencies = [ + "rand_core 0.6.4", + "subtle", +] + [[package]] name = "crypto-common" version = "0.1.6" @@ -1475,6 +1500,16 @@ dependencies = [ "uuid", ] +[[package]] +name = "der" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1a467a65c5e759bce6e65eaf91cc29f466cdc57cb65777bd646872a8a1fd4de" +dependencies = [ + "const-oid", + "zeroize", +] + [[package]] name = "der-parser" version = "8.2.0" @@ -1517,12 +1552,44 @@ version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "23d2f3407d9a573d666de4b5bdf10569d73ca9478087346697dcbae6244bfbcd" +[[package]] +name = "ecdsa" +version = "0.14.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "413301934810f597c1d19ca71c8710e99a3f1ba28a0d2ebc01551a2daeea3c5c" +dependencies = [ + "der", + "elliptic-curve", + "rfc6979", + "signature", +] + [[package]] name = "either" version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7fcaabb2fef8c910e7f4c7ce9f67a1283a1715879a7c230ca9d6d1ae31f16d91" +[[package]] +name = "elliptic-curve" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7bb888ab5300a19b8e5bceef25ac745ad065f3c9f7efc6de1b91958110891d3" +dependencies = [ + "base16ct", + "crypto-bigint 0.4.9", + "der", + "digest", + "ff", + "generic-array", + "group", + "pkcs8", + "rand_core 0.6.4", + "sec1", + "subtle", + "zeroize", +] + [[package]] name = "encoding_rs" version = "0.8.32" @@ -1645,6 +1712,16 @@ version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6999dc1837253364c2ebb0704ba97994bd874e8f195d665c50b7548f6ea92764" +[[package]] +name = "ff" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d013fc25338cc558c5c2cfbad646908fb23591e2404481826742b651c9af7160" +dependencies = [ + "rand_core 0.6.4", + "subtle", +] + [[package]] name = "filetime" version = "0.2.21" @@ -1855,9 +1932,9 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.2.9" +version = "0.2.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c85e1d9ab2eadba7e5040d4e09cbd6d072b76a557ad64e797c2cb9d4da21d7e4" +checksum = "fe9006bed769170c11f845cf00c7c1e9092aeb3f268e007c3e760ac68008070f" dependencies = [ "cfg-if", "js-sys", @@ -1900,6 +1977,17 @@ version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b" +[[package]] +name = "group" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5dfbfb3a6cfbd390d5c9564ab283a0349b9b9fcd46a706c1eb10e0db70bfbac7" +dependencies = [ + "ff", + "rand_core 0.6.4", + "subtle", +] + [[package]] name = "h2" version = "0.3.19" @@ -2351,7 +2439,7 @@ checksum = "6971da4d9c3aa03c3d8f3ff0f4155b534aad021292003895a469716b2a230378" dependencies = [ "base64 0.21.1", "pem 1.1.1", - "ring", + "ring 0.16.20", "serde", "serde_json", "simple_asn1", @@ -2391,9 +2479,9 @@ checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" [[package]] name = "libc" -version = "0.2.144" +version = "0.2.150" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b00cc1c228a6782d0f076e7b232802e0c5689d41bb5df366f2a6b6621cfdfe1" +checksum = "89d92a4743f9a61002fae18374ed11e7973f530cb3a3255fb354818118b2203c" [[package]] name = "libloading" @@ -2700,7 +2788,7 @@ checksum = "c38841cdd844847e3e7c8d29cef9dcfed8877f8f56f9071f77843ecf3baf937f" dependencies = [ "base64 0.13.1", "chrono", - "getrandom 0.2.9", + "getrandom 0.2.11", "http", "rand 0.8.5", "serde", @@ -2903,6 +2991,17 @@ version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4030760ffd992bef45b0ae3f10ce1aba99e33464c90d14dd7c039884963ddc7a" +[[package]] +name = "p256" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51f44edd08f51e2ade572f141051021c5af22677e42b7dd28a88155151c33594" +dependencies = [ + "ecdsa", + "elliptic-curve", + "sha2", +] + [[package]] name = "pagectl" version = "0.1.0" @@ -3197,6 +3296,16 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" +[[package]] +name = "pkcs8" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9eca2c590a5f85da82668fa685c09ce2888b9430e83299debf1f34b65fd4a4ba" +dependencies = [ + "der", + "spki", +] + [[package]] name = "pkg-config" version = "0.3.27" @@ -3633,7 +3742,7 @@ version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" dependencies = [ - "getrandom 0.2.9", + "getrandom 0.2.11", ] [[package]] @@ -3674,7 +3783,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4954fbc00dcd4d8282c987710e50ba513d351400dbdd00e803a05172a90d8976" dependencies = [ "pem 2.0.1", - "ring", + "ring 0.16.20", "time", "yasna", ] @@ -3745,8 +3854,7 @@ dependencies = [ "aws-credential-types", "aws-sdk-s3", "aws-smithy-async", - "aws-smithy-http", - "aws-types", + "aws-smithy-types", "azure_core", "azure_identity", "azure_storage", @@ -3844,7 +3952,7 @@ dependencies = [ "async-trait", "chrono", "futures", - "getrandom 0.2.9", + "getrandom 0.2.11", "http", "hyper", "parking_lot 0.11.2", @@ -3865,7 +3973,7 @@ checksum = "1b97ad83c2fc18113346b7158d79732242002427c30f620fa817c1f32901e0a8" dependencies = [ "anyhow", "async-trait", - "getrandom 0.2.9", + "getrandom 0.2.11", "matchit", "opentelemetry", "reqwest", @@ -3886,6 +3994,17 @@ dependencies = [ "rand 0.8.5", ] +[[package]] +name = "rfc6979" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7743f17af12fa0b03b803ba12cd6a8d9483a587e89c69445e3909655c0b9fabb" +dependencies = [ + "crypto-bigint 0.4.9", + "hmac", + "zeroize", +] + [[package]] name = "ring" version = "0.16.20" @@ -3896,11 +4015,25 @@ dependencies = [ "libc", "once_cell", "spin 0.5.2", - "untrusted", + "untrusted 0.7.1", "web-sys", "winapi", ] +[[package]] +name = "ring" +version = "0.17.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "684d5e6e18f669ccebf64a92236bb7db9a34f07be010e3627368182027180866" +dependencies = [ + "cc", + "getrandom 0.2.11", + "libc", + "spin 0.9.8", + "untrusted 0.9.0", + "windows-sys 0.48.0", +] + [[package]] name = "routerify" version = "3.0.0" @@ -4012,13 +4145,13 @@ dependencies = [ [[package]] name = "rustls" -version = "0.21.6" +version = "0.21.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d1feddffcfcc0b33f5c6ce9a29e341e4cd59c3f78e7ee45f4a40c038b1d6cbb" +checksum = "629648aced5775d558af50b2b4c7b02983a04b312126d45eeead26e7caa498b9" dependencies = [ "log", - "ring", - "rustls-webpki 0.101.4", + "ring 0.17.6", + "rustls-webpki 0.101.7", "sct", ] @@ -4049,18 +4182,18 @@ version = "0.100.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e98ff011474fa39949b7e5c0428f9b4937eda7da7848bbb947786b7be0b27dab" dependencies = [ - "ring", - "untrusted", + "ring 0.16.20", + "untrusted 0.7.1", ] [[package]] name = "rustls-webpki" -version = "0.101.4" +version = "0.101.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d93931baf2d282fff8d3a532bbfd7653f734643161b87e3e01e59a04439bf0d" +checksum = "8b6275d1ee7a1cd780b64aca7726599a1dbc893b1e64144529e55c3c2f745765" dependencies = [ - "ring", - "untrusted", + "ring 0.17.6", + "untrusted 0.9.0", ] [[package]] @@ -4083,8 +4216,6 @@ dependencies = [ "async-stream", "aws-config", "aws-sdk-s3", - "aws-smithy-http", - "aws-types", "bincode", "bytes", "chrono", @@ -4206,8 +4337,8 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d53dcdb7c9f8158937a7981b48accfd39a43af418591a5d008c7b22b5e1b7ca4" dependencies = [ - "ring", - "untrusted", + "ring 0.16.20", + "untrusted 0.7.1", ] [[package]] @@ -4216,6 +4347,20 @@ version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "621e3680f3e07db4c9c2c3fb07c6223ab2fab2e54bd3c04c3ae037990f428c32" +[[package]] +name = "sec1" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3be24c1842290c45df0a7bf069e0c268a747ad05a192f2fd7dcfdbc1cba40928" +dependencies = [ + "base16ct", + "der", + "generic-array", + "pkcs8", + "subtle", + "zeroize", +] + [[package]] name = "security-framework" version = "2.9.1" @@ -4332,7 +4477,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "99dc599bd6646884fc403d593cdcb9816dd67c50cff3271c01ff123617908dcd" dependencies = [ "debugid", - "getrandom 0.2.9", + "getrandom 0.2.11", "hex", "serde", "serde_json", @@ -4520,6 +4665,16 @@ dependencies = [ "libc", ] +[[package]] +name = "signature" +version = "1.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "74233d3b3b2f6d4b006dc19dee745e73e2a6bfb6f93607cd3b02bd5b00797d7c" +dependencies = [ + "digest", + "rand_core 0.6.4", +] + [[package]] name = "simple_asn1" version = "0.6.2" @@ -4588,6 +4743,16 @@ dependencies = [ "lock_api", ] +[[package]] +name = "spki" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67cf02bbac7a337dc36e4f5a693db6c21e7863f45070f7064577eb4367a3212b" +dependencies = [ + "base64ct", + "der", +] + [[package]] name = "stable_deref_trait" version = "1.2.0" @@ -4981,7 +5146,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dd5831152cb0d3f79ef5523b357319ba154795d64c7078b2daa95a803b54057f" dependencies = [ "futures", - "ring", + "ring 0.16.20", "rustls", "tokio", "tokio-postgres", @@ -5437,6 +5602,12 @@ version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + [[package]] name = "ureq" version = "2.7.1" @@ -5538,7 +5709,7 @@ version = "1.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "345444e32442451b267fc254ae85a209c64be56d2890e601a0c37ff0c3c5ecd2" dependencies = [ - "getrandom 0.2.9", + "getrandom 0.2.11", "serde", ] @@ -5997,9 +6168,13 @@ dependencies = [ "aws-config", "aws-runtime", "aws-sigv4", + "aws-smithy-async", "aws-smithy-http", + "aws-smithy-runtime-api", + "aws-smithy-types", "axum", "base64 0.21.1", + "base64ct", "bytes", "cc", "chrono", @@ -6017,6 +6192,7 @@ dependencies = [ "futures-sink", "futures-util", "hex", + "hmac", "hyper", "itertools", "libc", @@ -6031,12 +6207,13 @@ dependencies = [ "regex", "regex-syntax 0.7.2", "reqwest", - "ring", + "ring 0.16.20", "rustls", "scopeguard", "serde", "serde_json", "smallvec", + "subtle", "syn 1.0.109", "syn 2.0.28", "time", diff --git a/Cargo.toml b/Cargo.toml index ab00c11dda..28b58179ea 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -45,12 +45,11 @@ azure_storage_blobs = "0.16" flate2 = "1.0.26" async-stream = "0.3" async-trait = "0.1" -aws-config = { version = "0.56", default-features = false, features=["rustls"] } -aws-sdk-s3 = "0.29" -aws-smithy-http = "0.56" -aws-smithy-async = { version = "0.56", default-features = false, features=["rt-tokio"] } -aws-credential-types = "0.56" -aws-types = "0.56" +aws-config = { version = "1.0", default-features = false, features=["rustls"] } +aws-sdk-s3 = "1.0" +aws-smithy-async = { version = "1.0", default-features = false, features=["rt-tokio"] } +aws-smithy-types = "1.0" +aws-credential-types = "1.0" axum = { version = "0.6.20", features = ["ws"] } base64 = "0.13.0" bincode = "1.3" diff --git a/libs/remote_storage/Cargo.toml b/libs/remote_storage/Cargo.toml index d7bcce28cb..e8bfc005d3 100644 --- a/libs/remote_storage/Cargo.toml +++ b/libs/remote_storage/Cargo.toml @@ -9,8 +9,7 @@ anyhow.workspace = true async-trait.workspace = true once_cell.workspace = true aws-smithy-async.workspace = true -aws-smithy-http.workspace = true -aws-types.workspace = true +aws-smithy-types.workspace = true aws-config.workspace = true aws-sdk-s3.workspace = true aws-credential-types.workspace = true diff --git a/libs/remote_storage/src/s3_bucket.rs b/libs/remote_storage/src/s3_bucket.rs index ab3fd3fe62..0cb73f73b7 100644 --- a/libs/remote_storage/src/s3_bucket.rs +++ b/libs/remote_storage/src/s3_bucket.rs @@ -14,18 +14,20 @@ use aws_config::{ provider_config::ProviderConfig, retry::{RetryConfigBuilder, RetryMode}, web_identity_token::WebIdentityTokenCredentialsProvider, + BehaviorVersion, }; -use aws_credential_types::cache::CredentialsCache; +use aws_credential_types::provider::SharedCredentialsProvider; use aws_sdk_s3::{ - config::{AsyncSleep, Config, Region, SharedAsyncSleep}, + config::{AsyncSleep, Builder, IdentityCache, Region, SharedAsyncSleep}, error::SdkError, operation::get_object::GetObjectError, - primitives::ByteStream, types::{Delete, ObjectIdentifier}, Client, }; use aws_smithy_async::rt::sleep::TokioSleep; -use aws_smithy_http::body::SdkBody; + +use aws_smithy_types::body::SdkBody; +use aws_smithy_types::byte_stream::ByteStream; use hyper::Body; use scopeguard::ScopeGuard; use tokio::io::{self, AsyncRead}; @@ -78,7 +80,6 @@ impl S3Bucket { // needed to access remote extensions bucket .or_else("token", { let provider_conf = ProviderConfig::without_region().with_region(region.clone()); - WebIdentityTokenCredentialsProvider::builder() .configure(&provider_conf) .build() @@ -98,18 +99,20 @@ impl S3Bucket { .set_max_attempts(Some(1)) .set_mode(Some(RetryMode::Adaptive)); - let mut config_builder = Config::builder() + let mut config_builder = Builder::default() + .behavior_version(BehaviorVersion::v2023_11_09()) .region(region) - .credentials_cache(CredentialsCache::lazy()) - .credentials_provider(credentials_provider) - .sleep_impl(SharedAsyncSleep::from(sleep_impl)) - .retry_config(retry_config.build()); + .identity_cache(IdentityCache::lazy().build()) + .credentials_provider(SharedCredentialsProvider::new(credentials_provider)) + .retry_config(retry_config.build()) + .sleep_impl(SharedAsyncSleep::from(sleep_impl)); if let Some(custom_endpoint) = aws_config.endpoint.clone() { config_builder = config_builder .endpoint_url(custom_endpoint) .force_path_style(true); } + let client = Client::from_conf(config_builder.build()); let prefix_in_bucket = aws_config.prefix_in_bucket.as_deref().map(|prefix| { @@ -371,7 +374,7 @@ impl RemoteStorage for S3Bucket { let response = response?; - let keys = response.contents().unwrap_or_default(); + let keys = response.contents(); let empty = Vec::new(); let prefixes = response.common_prefixes.as_ref().unwrap_or(&empty); @@ -411,7 +414,7 @@ impl RemoteStorage for S3Bucket { let started_at = start_measuring_requests(kind); let body = Body::wrap_stream(ReaderStream::new(from)); - let bytes_stream = ByteStream::new(SdkBody::from(body)); + let bytes_stream = ByteStream::new(SdkBody::from_body_0_4(body)); let res = self .client @@ -474,7 +477,7 @@ impl RemoteStorage for S3Bucket { for path in paths { let obj_id = ObjectIdentifier::builder() .set_key(Some(self.relative_path_to_s3_object(path))) - .build(); + .build()?; delete_objects.push(obj_id); } @@ -485,7 +488,11 @@ impl RemoteStorage for S3Bucket { .client .delete_objects() .bucket(self.bucket_name.clone()) - .delete(Delete::builder().set_objects(Some(chunk.to_vec())).build()) + .delete( + Delete::builder() + .set_objects(Some(chunk.to_vec())) + .build()?, + ) .send() .await; diff --git a/s3_scrubber/Cargo.toml b/s3_scrubber/Cargo.toml index 0f3e5630e8..e26f2c6d6b 100644 --- a/s3_scrubber/Cargo.toml +++ b/s3_scrubber/Cargo.toml @@ -6,8 +6,6 @@ license.workspace = true [dependencies] aws-sdk-s3.workspace = true -aws-smithy-http.workspace = true -aws-types.workspace = true either.workspace = true tokio-rustls.workspace = true anyhow.workspace = true @@ -30,7 +28,7 @@ itertools.workspace = true tokio = { workspace = true, features = ["macros", "rt-multi-thread"] } chrono = { workspace = true, default-features = false, features = ["clock", "serde"] } reqwest = { workspace = true, default-features = false, features = ["rustls-tls", "json"] } -aws-config = { workspace = true, default-features = false, features = ["rustls", "credentials-sso"] } +aws-config = { workspace = true, default-features = false, features = ["rustls", "sso"] } pageserver = { path = "../pageserver" } remote_storage = { path = "../libs/remote_storage" } diff --git a/s3_scrubber/src/checks.rs b/s3_scrubber/src/checks.rs index 0686225d1b..510a128663 100644 --- a/s3_scrubber/src/checks.rs +++ b/s3_scrubber/src/checks.rs @@ -250,10 +250,7 @@ pub(crate) async fn list_timeline_blobs( pin_mut!(stream); while let Some(obj) = stream.next().await { let obj = obj?; - let key = match obj.key() { - Some(k) => k, - None => continue, - }; + let key = obj.key(); let blob_name = key.strip_prefix(&timeline_dir_target.prefix_in_bucket); match blob_name { @@ -286,7 +283,7 @@ pub(crate) async fn list_timeline_blobs( let (index_part_object, index_part_generation) = match index_parts .iter() .filter_map(|k| { - let key = k.key().unwrap(); + let key = k.key(); // Stripping the index key to the last part, because RemotePath doesn't // like absolute paths, and depending on prefix_in_bucket it's possible // for the keys we read back to start with a slash. @@ -307,8 +304,7 @@ pub(crate) async fn list_timeline_blobs( errors.push("S3 list response got no index_part.json file".to_string()); } - if let Some(index_part_object_key) = index_part_object.as_ref().and_then(|object| object.key()) - { + if let Some(index_part_object_key) = index_part_object.as_ref().map(|object| object.key()) { let index_part_bytes = download_object_with_retries( s3_client, &timeline_dir_target.bucket_name, diff --git a/s3_scrubber/src/garbage.rs b/s3_scrubber/src/garbage.rs index daeb5e9777..f27e1d7f65 100644 --- a/s3_scrubber/src/garbage.rs +++ b/s3_scrubber/src/garbage.rs @@ -323,7 +323,7 @@ async fn do_delete( let delete_request = s3_client .delete_objects() .bucket(bucket_name) - .delete(Delete::builder().set_objects(Some(request_keys)).build()); + .delete(Delete::builder().set_objects(Some(request_keys)).build()?); delete_request .send() .await diff --git a/s3_scrubber/src/lib.rs b/s3_scrubber/src/lib.rs index 777276a4d1..e5465952fb 100644 --- a/s3_scrubber/src/lib.rs +++ b/s3_scrubber/src/lib.rs @@ -16,6 +16,7 @@ use aws_config::environment::EnvironmentVariableCredentialsProvider; use aws_config::imds::credentials::ImdsCredentialsProvider; use aws_config::meta::credentials::CredentialsProviderChain; use aws_config::sso::SsoCredentialsProvider; +use aws_config::BehaviorVersion; use aws_sdk_s3::config::Region; use aws_sdk_s3::{Client, Config}; @@ -245,6 +246,7 @@ pub fn init_s3_client(account_id: Option, bucket_region: Region) -> Clie }; let mut builder = Config::builder() + .behavior_version(BehaviorVersion::v2023_11_09()) .region(bucket_region) .credentials_provider(credentials_provider); diff --git a/s3_scrubber/src/metadata_stream.rs b/s3_scrubber/src/metadata_stream.rs index 8095071c1f..4cfa77cfc1 100644 --- a/s3_scrubber/src/metadata_stream.rs +++ b/s3_scrubber/src/metadata_stream.rs @@ -20,7 +20,6 @@ pub fn stream_tenants<'a>( let new_entry_ids = fetch_response .common_prefixes() - .unwrap_or_default() .iter() .filter_map(|prefix| prefix.prefix()) .filter_map(|prefix| -> Option<&str> { @@ -72,7 +71,6 @@ pub async fn stream_tenant_timelines<'a>( let new_entry_ids = fetch_response .common_prefixes() - .unwrap_or_default() .iter() .filter_map(|prefix| prefix.prefix()) .filter_map(|prefix| -> Option<&str> { @@ -116,15 +114,15 @@ pub(crate) fn stream_listing<'a>( list_objects_with_retries(s3_client, target, continuation_token.clone()).await?; if target.delimiter.is_empty() { - for object_id in fetch_response.contents().unwrap_or_default().iter().filter_map(|object| object.key()).map(|i| - ObjectIdentifier::builder().key(i).build() - ) { + for object_key in fetch_response.contents().iter().filter_map(|object| object.key()) + { + let object_id = ObjectIdentifier::builder().key(object_key).build()?; yield object_id; } } else { - for prefix in fetch_response.common_prefixes().unwrap_or_default() - .iter().filter_map(|p| p.prefix().map(|k| ObjectIdentifier::builder().key(k).build())) { - yield prefix; + for prefix in fetch_response.common_prefixes().iter().filter_map(|p| p.prefix()) { + let object_id = ObjectIdentifier::builder().key(prefix).build()?; + yield object_id; } } diff --git a/workspace_hack/Cargo.toml b/workspace_hack/Cargo.toml index 66828fb53d..3e46731adf 100644 --- a/workspace_hack/Cargo.toml +++ b/workspace_hack/Cargo.toml @@ -14,12 +14,16 @@ publish = false ### BEGIN HAKARI SECTION [dependencies] anyhow = { version = "1", features = ["backtrace"] } -aws-config = { version = "0.56", default-features = false, features = ["credentials-sso", "rustls"] } -aws-runtime = { version = "0.56", default-features = false, features = ["event-stream"] } -aws-sigv4 = { version = "0.56", features = ["sign-eventstream"] } -aws-smithy-http = { version = "0.56", default-features = false, features = ["event-stream", "rt-tokio"] } +aws-config = { version = "1", default-features = false, features = ["rustls", "sso"] } +aws-runtime = { version = "1", default-features = false, features = ["event-stream", "sigv4a"] } +aws-sigv4 = { version = "1", features = ["http0-compat", "sign-eventstream", "sigv4a"] } +aws-smithy-async = { version = "1", default-features = false, features = ["rt-tokio"] } +aws-smithy-http = { version = "0.60", default-features = false, features = ["event-stream"] } +aws-smithy-runtime-api = { version = "1", features = ["client", "http-02x", "http-auth"] } +aws-smithy-types = { version = "1", default-features = false, features = ["byte-stream-poll-next", "http-body-0-4-x", "rt-tokio"] } axum = { version = "0.6", features = ["ws"] } base64 = { version = "0.21", features = ["alloc"] } +base64ct = { version = "1", default-features = false, features = ["std"] } bytes = { version = "1", features = ["serde"] } chrono = { version = "0.4", default-features = false, features = ["clock", "serde", "wasmbind"] } clap = { version = "4", features = ["derive", "string"] } @@ -36,6 +40,7 @@ futures-io = { version = "0.3" } futures-sink = { version = "0.3" } futures-util = { version = "0.3", features = ["channel", "io", "sink"] } hex = { version = "0.4", features = ["serde"] } +hmac = { version = "0.12", default-features = false, features = ["reset"] } hyper = { version = "0.14", features = ["full"] } itertools = { version = "0.10" } libc = { version = "0.2", features = ["extra_traits"] } @@ -56,13 +61,14 @@ scopeguard = { version = "1" } serde = { version = "1", features = ["alloc", "derive"] } serde_json = { version = "1", features = ["raw_value"] } smallvec = { version = "1", default-features = false, features = ["write"] } +subtle = { version = "2" } time = { version = "0.3", features = ["local-offset", "macros", "serde-well-known"] } tokio = { version = "1", features = ["fs", "io-std", "io-util", "macros", "net", "process", "rt-multi-thread", "signal", "test-util"] } tokio-rustls = { version = "0.24" } tokio-util = { version = "0.7", features = ["codec", "io"] } toml_datetime = { version = "0.6", default-features = false, features = ["serde"] } toml_edit = { version = "0.19", features = ["serde"] } -tower = { version = "0.4", features = ["balance", "buffer", "limit", "retry", "timeout", "util"] } +tower = { version = "0.4", default-features = false, features = ["balance", "buffer", "limit", "log", "timeout", "util"] } tracing = { version = "0.1", features = ["log"] } tracing-core = { version = "0.1" } tungstenite = { version = "0.20" } From 1e57ddaabc2b69752bf7b9329382da4b1e24ce63 Mon Sep 17 00:00:00 2001 From: Joonas Koivunen Date: Thu, 30 Nov 2023 14:26:11 +0200 Subject: [PATCH 17/38] fix: flush loop should also keep the gate open (#5987) I was expecting this to already be in place, because this should not conflict how we shutdown (0. cancel, 1. shutdown_tasks, 2. close gate). --- pageserver/src/tenant/timeline.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pageserver/src/tenant/timeline.rs b/pageserver/src/tenant/timeline.rs index 24f59673c1..9a7c9a6df3 100644 --- a/pageserver/src/tenant/timeline.rs +++ b/pageserver/src/tenant/timeline.rs @@ -1488,6 +1488,10 @@ impl Timeline { } pub(super) fn maybe_spawn_flush_loop(self: &Arc) { + let Ok(guard) = self.gate.enter() else { + info!("cannot start flush loop when the timeline gate has already been closed"); + return; + }; let mut flush_loop_state = self.flush_loop_state.lock().unwrap(); match *flush_loop_state { FlushLoopState::NotStarted => (), @@ -1525,6 +1529,7 @@ impl Timeline { "layer flush task", false, async move { + let _guard = guard; let background_ctx = RequestContext::todo_child(TaskKind::LayerFlushTask, DownloadBehavior::Error); self_clone.flush_loop(layer_flush_start_rx, &background_ctx).await; let mut flush_loop_state = self_clone.flush_loop_state.lock().unwrap(); From e12e2681e97d997924383e3785bd734c9be8c21a Mon Sep 17 00:00:00 2001 From: Anna Khanova <32508607+khanova@users.noreply.github.com> Date: Thu, 30 Nov 2023 14:14:33 +0100 Subject: [PATCH 18/38] IP allowlist on the proxy side (#5906) ## Problem Per-project IP allowlist: https://github.com/neondatabase/cloud/issues/8116 ## Summary of changes Implemented IP filtering on the proxy side. To retrieve ip allowlist for all scenarios, added `get_auth_info` call to the control plane for: * sql-over-http * password_hack * cleartext_hack Added cache with ttl for sql-over-http path This might slow down a bit, consider using redis in the future. --------- Co-authored-by: Conrad Ludgate --- Cargo.lock | 5 +- Cargo.toml | 1 + proxy/Cargo.toml | 1 + proxy/src/auth.rs | 13 +- proxy/src/auth/backend.rs | 52 ++++- proxy/src/auth/backend/classic.rs | 22 +- proxy/src/auth/credentials.rs | 197 +++++++++++++++--- proxy/src/bin/proxy.rs | 29 ++- proxy/src/cache.rs | 16 +- proxy/src/config.rs | 2 + proxy/src/console.rs | 2 +- proxy/src/console/provider.rs | 20 +- proxy/src/console/provider/mock.rs | 95 ++++++--- proxy/src/console/provider/neon.rs | 53 ++++- proxy/src/http.rs | 10 +- proxy/src/proxy.rs | 46 +++- proxy/src/proxy/tests.rs | 4 + proxy/src/serverless.rs | 17 +- proxy/src/serverless/conn_pool.rs | 34 ++- proxy/src/serverless/sql_over_http.rs | 16 +- proxy/src/serverless/websocket.rs | 3 + test_runner/fixtures/neon_fixtures.py | 4 + test_runner/regress/test_proxy_allowed_ips.py | 74 +++++++ 23 files changed, 601 insertions(+), 115 deletions(-) create mode 100644 test_runner/regress/test_proxy_allowed_ips.py diff --git a/Cargo.lock b/Cargo.lock index 3f0f21eb4a..6546590f6c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2382,9 +2382,9 @@ dependencies = [ [[package]] name = "ipnet" -version = "2.7.2" +version = "2.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12b6ee2129af8d4fb011108c73d99a1b83a85977f23b82460c0ae2e25bb4b57f" +checksum = "8f518f335dce6725a761382244631d86cf0ccb2863413590b31338feb467f9c3" [[package]] name = "is-terminal" @@ -3612,6 +3612,7 @@ dependencies = [ "humantime", "hyper", "hyper-tungstenite", + "ipnet", "itertools", "md5", "metrics", diff --git a/Cargo.toml b/Cargo.toml index 28b58179ea..cbcb25359d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -88,6 +88,7 @@ humantime-serde = "1.1.1" hyper = "0.14" hyper-tungstenite = "0.11" inotify = "0.10.2" +ipnet = "2.9.0" itertools = "0.10" jsonwebtoken = "8" libc = "0.2" diff --git a/proxy/Cargo.toml b/proxy/Cargo.toml index 39a9c3ddb0..0822718bae 100644 --- a/proxy/Cargo.toml +++ b/proxy/Cargo.toml @@ -24,6 +24,7 @@ hostname.workspace = true humantime.workspace = true hyper-tungstenite.workspace = true hyper.workspace = true +ipnet.workspace = true itertools.workspace = true md5.workspace = true metrics.workspace = true diff --git a/proxy/src/auth.rs b/proxy/src/auth.rs index 58dceb3bb6..7d79d34045 100644 --- a/proxy/src/auth.rs +++ b/proxy/src/auth.rs @@ -4,7 +4,7 @@ pub mod backend; pub use backend::BackendType; mod credentials; -pub use credentials::ClientCredentials; +pub use credentials::{check_peer_addr_is_in_list, ClientCredentials}; mod password_hack; pub use password_hack::parse_endpoint_param; @@ -56,6 +56,12 @@ pub enum AuthErrorImpl { /// Errors produced by e.g. [`crate::stream::PqStream`]. #[error(transparent)] Io(#[from] io::Error), + + #[error( + "This IP address is not allowed to connect to this endpoint. \ + Please add it to the allowed list in the Neon console." + )] + IpAddressNotAllowed, } #[derive(Debug, Error)] @@ -70,6 +76,10 @@ impl AuthError { pub fn auth_failed(user: impl Into>) -> Self { AuthErrorImpl::AuthFailed(user.into()).into() } + + pub fn ip_address_not_allowed() -> Self { + AuthErrorImpl::IpAddressNotAllowed.into() + } } impl> From for AuthError { @@ -91,6 +101,7 @@ impl UserFacingError for AuthError { MalformedPassword(_) => self.to_string(), MissingEndpointName => self.to_string(), Io(_) => "Internal error".to_string(), + IpAddressNotAllowed => self.to_string(), } } } diff --git a/proxy/src/auth/backend.rs b/proxy/src/auth/backend.rs index f0197cc31b..aa872285b1 100644 --- a/proxy/src/auth/backend.rs +++ b/proxy/src/auth/backend.rs @@ -5,7 +5,12 @@ mod link; pub use link::LinkAuthError; use tokio_postgres::config::AuthKeys; +use crate::auth::credentials::check_peer_addr_is_in_list; +use crate::console::errors::GetAuthInfoError; +use crate::console::provider::AuthInfo; +use crate::console::AuthSecret; use crate::proxy::{handle_try_wake, retry_after, LatencyTimer}; +use crate::scram; use crate::stream::Stream; use crate::{ auth::{self, ClientCredentials}, @@ -20,6 +25,7 @@ use crate::{ use futures::TryFutureExt; use std::borrow::Cow; use std::ops::ControlFlow; +use std::sync::Arc; use tokio::io::{AsyncRead, AsyncWrite}; use tracing::{error, info, warn}; @@ -64,6 +70,7 @@ pub enum BackendType<'a, T> { pub trait TestBackend: Send + Sync + 'static { fn wake_compute(&self) -> Result; + fn get_allowed_ips(&self) -> Result>, console::errors::GetAuthInfoError>; } impl std::fmt::Display for BackendType<'_, ()> { @@ -140,14 +147,38 @@ async fn auth_quirks_creds( // If there's no project so far, that entails that client doesn't // support SNI or other means of passing the endpoint (project) name. // We now expect to see a very specific payload in the place of password. - if creds.project.is_none() { + let maybe_success = if creds.project.is_none() { // Password will be checked by the compute node later. - return hacks::password_hack(creds, client, latency_timer).await; - } + Some(hacks::password_hack(creds, client, latency_timer).await?) + } else { + None + }; // Password hack should set the project name. // TODO: make `creds.project` more type-safe. assert!(creds.project.is_some()); + info!("fetching user's authentication info"); + // TODO(anna): this will slow down both "hacks" below; we probably need a cache. + let AuthInfo { + secret, + allowed_ips, + } = api.get_auth_info(extra, creds).await?; + + // check allowed list + if !check_peer_addr_is_in_list(&creds.peer_addr.ip(), &allowed_ips) { + return Err(auth::AuthError::ip_address_not_allowed()); + } + let secret = secret.unwrap_or_else(|| { + // If we don't have an authentication secret, we mock one to + // prevent malicious probing (possible due to missing protocol steps). + // This mocked secret will never lead to successful authentication. + info!("authentication info not found, mocking it"); + AuthSecret::Scram(scram::ServerSecret::mock(creds.user, rand::random())) + }); + + if let Some(success) = maybe_success { + return Ok(success); + } // Perform cleartext auth if we're allowed to do that. // Currently, we use it for websocket connections (latency). @@ -157,7 +188,7 @@ async fn auth_quirks_creds( } // Finally, proceed with the main auth flow (SCRAM-based). - classic::authenticate(api, extra, creds, client, config, latency_timer).await + classic::authenticate(creds, client, config, latency_timer, secret).await } /// True to its name, this function encapsulates our current auth trade-offs. @@ -305,6 +336,19 @@ impl BackendType<'_, ClientCredentials<'_>> { Ok(res) } + pub async fn get_allowed_ips( + &self, + extra: &ConsoleReqExtra<'_>, + ) -> Result>, GetAuthInfoError> { + use BackendType::*; + match self { + Console(api, creds) => api.get_allowed_ips(extra, creds).await, + Postgres(api, creds) => api.get_allowed_ips(extra, creds).await, + Link(_) => Ok(Arc::new(vec![])), + Test(x) => x.get_allowed_ips(), + } + } + /// When applicable, wake the compute node, gaining its connection info in the process. /// The link auth flow doesn't support this, so we return [`None`] in that case. pub async fn wake_compute( diff --git a/proxy/src/auth/backend/classic.rs b/proxy/src/auth/backend/classic.rs index ac0d490db1..bb210821cd 100644 --- a/proxy/src/auth/backend/classic.rs +++ b/proxy/src/auth/backend/classic.rs @@ -3,38 +3,28 @@ use crate::{ auth::{self, AuthFlow, ClientCredentials}, compute, config::AuthenticationConfig, - console::{self, AuthInfo, ConsoleReqExtra}, + console::AuthSecret, proxy::LatencyTimer, - sasl, scram, + sasl, stream::{PqStream, Stream}, }; use tokio::io::{AsyncRead, AsyncWrite}; use tracing::{info, warn}; pub(super) async fn authenticate( - api: &impl console::Api, - extra: &ConsoleReqExtra<'_>, creds: &ClientCredentials<'_>, client: &mut PqStream>, config: &'static AuthenticationConfig, latency_timer: &mut LatencyTimer, + secret: AuthSecret, ) -> auth::Result> { - info!("fetching user's authentication info"); - let info = api.get_auth_info(extra, creds).await?.unwrap_or_else(|| { - // If we don't have an authentication secret, we mock one to - // prevent malicious probing (possible due to missing protocol steps). - // This mocked secret will never lead to successful authentication. - info!("authentication info not found, mocking it"); - AuthInfo::Scram(scram::ServerSecret::mock(creds.user, rand::random())) - }); - let flow = AuthFlow::new(client); - let scram_keys = match info { - AuthInfo::Md5(_) => { + let scram_keys = match secret { + AuthSecret::Md5(_) => { info!("auth endpoint chooses MD5"); return Err(auth::AuthError::bad_auth_method("MD5")); } - AuthInfo::Scram(secret) => { + AuthSecret::Scram(secret) => { info!("auth endpoint chooses SCRAM"); let scram = auth::Scram(&secret); diff --git a/proxy/src/auth/credentials.rs b/proxy/src/auth/credentials.rs index 9fe9c26f0c..facb8da8cd 100644 --- a/proxy/src/auth/credentials.rs +++ b/proxy/src/auth/credentials.rs @@ -7,9 +7,12 @@ use crate::{ }; use itertools::Itertools; use pq_proto::StartupMessageParams; -use std::collections::HashSet; +use std::{ + collections::HashSet, + net::{IpAddr, SocketAddr}, +}; use thiserror::Error; -use tracing::info; +use tracing::{info, warn}; #[derive(Debug, Error, PartialEq, Eq, Clone)] pub enum ClientCredsParseError { @@ -44,6 +47,7 @@ pub struct ClientCredentials<'a> { pub project: Option, pub cache_key: String, + pub peer_addr: SocketAddr, } impl ClientCredentials<'_> { @@ -54,19 +58,11 @@ impl ClientCredentials<'_> { } impl<'a> ClientCredentials<'a> { - #[cfg(test)] - pub fn new_noop() -> Self { - ClientCredentials { - user: "", - project: None, - cache_key: "".to_string(), - } - } - pub fn parse( params: &'a StartupMessageParams, sni: Option<&str>, common_names: Option>, + peer_addr: SocketAddr, ) -> Result { use ClientCredsParseError::*; @@ -153,10 +149,59 @@ impl<'a> ClientCredentials<'a> { user, project, cache_key, + peer_addr, }) } } +pub fn check_peer_addr_is_in_list(peer_addr: &IpAddr, ip_list: &Vec) -> bool { + if ip_list.is_empty() { + return true; + } + for ip in ip_list { + // We expect that all ip addresses from control plane are correct. + // However, if some of them are broken, we still can check the others. + match parse_ip_pattern(ip) { + Ok(pattern) => { + if check_ip(peer_addr, &pattern) { + return true; + } + } + Err(err) => warn!("Cannot parse ip: {}; err: {}", ip, err), + } + } + false +} + +#[derive(Debug, Clone, Eq, PartialEq)] +enum IpPattern { + Subnet(ipnet::IpNet), + Range(IpAddr, IpAddr), + Single(IpAddr), +} + +fn parse_ip_pattern(pattern: &str) -> anyhow::Result { + if pattern.contains('/') { + let subnet: ipnet::IpNet = pattern.parse()?; + return Ok(IpPattern::Subnet(subnet)); + } + if let Some((start, end)) = pattern.split_once('-') { + let start: IpAddr = start.parse()?; + let end: IpAddr = end.parse()?; + return Ok(IpPattern::Range(start, end)); + } + let addr: IpAddr = pattern.parse()?; + Ok(IpPattern::Single(addr)) +} + +fn check_ip(ip: &IpAddr, pattern: &IpPattern) -> bool { + match pattern { + IpPattern::Subnet(subnet) => subnet.contains(ip), + IpPattern::Range(start, end) => start <= ip && ip <= end, + IpPattern::Single(addr) => addr == ip, + } +} + fn project_name_valid(name: &str) -> bool { name.chars().all(|c| c.is_alphanumeric() || c == '-') } @@ -176,8 +221,8 @@ mod tests { fn parse_bare_minimum() -> anyhow::Result<()> { // According to postgresql, only `user` should be required. let options = StartupMessageParams::new([("user", "john_doe")]); - - let creds = ClientCredentials::parse(&options, None, None)?; + let peer_addr = SocketAddr::from(([127, 0, 0, 1], 1234)); + let creds = ClientCredentials::parse(&options, None, None, peer_addr)?; assert_eq!(creds.user, "john_doe"); assert_eq!(creds.project, None); @@ -191,8 +236,8 @@ mod tests { ("database", "world"), // should be ignored ("foo", "bar"), // should be ignored ]); - - let creds = ClientCredentials::parse(&options, None, None)?; + let peer_addr = SocketAddr::from(([127, 0, 0, 1], 1234)); + let creds = ClientCredentials::parse(&options, None, None, peer_addr)?; assert_eq!(creds.user, "john_doe"); assert_eq!(creds.project, None); @@ -206,7 +251,8 @@ mod tests { let sni = Some("foo.localhost"); let common_names = Some(["localhost".into()].into()); - let creds = ClientCredentials::parse(&options, sni, common_names)?; + let peer_addr = SocketAddr::from(([127, 0, 0, 1], 1234)); + let creds = ClientCredentials::parse(&options, sni, common_names, peer_addr)?; assert_eq!(creds.user, "john_doe"); assert_eq!(creds.project.as_deref(), Some("foo")); assert_eq!(creds.cache_key, "foo"); @@ -221,7 +267,8 @@ mod tests { ("options", "-ckey=1 project=bar -c geqo=off"), ]); - let creds = ClientCredentials::parse(&options, None, None)?; + let peer_addr = SocketAddr::from(([127, 0, 0, 1], 1234)); + let creds = ClientCredentials::parse(&options, None, None, peer_addr)?; assert_eq!(creds.user, "john_doe"); assert_eq!(creds.project.as_deref(), Some("bar")); @@ -235,7 +282,8 @@ mod tests { ("options", "-ckey=1 endpoint=bar -c geqo=off"), ]); - let creds = ClientCredentials::parse(&options, None, None)?; + let peer_addr = SocketAddr::from(([127, 0, 0, 1], 1234)); + let creds = ClientCredentials::parse(&options, None, None, peer_addr)?; assert_eq!(creds.user, "john_doe"); assert_eq!(creds.project.as_deref(), Some("bar")); @@ -252,7 +300,8 @@ mod tests { ), ]); - let creds = ClientCredentials::parse(&options, None, None)?; + let peer_addr = SocketAddr::from(([127, 0, 0, 1], 1234)); + let creds = ClientCredentials::parse(&options, None, None, peer_addr)?; assert_eq!(creds.user, "john_doe"); assert!(creds.project.is_none()); @@ -266,7 +315,8 @@ mod tests { ("options", "-ckey=1 endpoint=bar project=foo -c geqo=off"), ]); - let creds = ClientCredentials::parse(&options, None, None)?; + let peer_addr = SocketAddr::from(([127, 0, 0, 1], 1234)); + let creds = ClientCredentials::parse(&options, None, None, peer_addr)?; assert_eq!(creds.user, "john_doe"); assert!(creds.project.is_none()); @@ -280,7 +330,8 @@ mod tests { let sni = Some("baz.localhost"); let common_names = Some(["localhost".into()].into()); - let creds = ClientCredentials::parse(&options, sni, common_names)?; + let peer_addr = SocketAddr::from(([127, 0, 0, 1], 1234)); + let creds = ClientCredentials::parse(&options, sni, common_names, peer_addr)?; assert_eq!(creds.user, "john_doe"); assert_eq!(creds.project.as_deref(), Some("baz")); @@ -293,12 +344,14 @@ mod tests { let common_names = Some(["a.com".into(), "b.com".into()].into()); let sni = Some("p1.a.com"); - let creds = ClientCredentials::parse(&options, sni, common_names)?; + let peer_addr = SocketAddr::from(([127, 0, 0, 1], 1234)); + let creds = ClientCredentials::parse(&options, sni, common_names, peer_addr)?; assert_eq!(creds.project.as_deref(), Some("p1")); let common_names = Some(["a.com".into(), "b.com".into()].into()); let sni = Some("p1.b.com"); - let creds = ClientCredentials::parse(&options, sni, common_names)?; + let peer_addr = SocketAddr::from(([127, 0, 0, 1], 1234)); + let creds = ClientCredentials::parse(&options, sni, common_names, peer_addr)?; assert_eq!(creds.project.as_deref(), Some("p1")); Ok(()) @@ -312,7 +365,9 @@ mod tests { let sni = Some("second.localhost"); let common_names = Some(["localhost".into()].into()); - let err = ClientCredentials::parse(&options, sni, common_names).expect_err("should fail"); + let peer_addr = SocketAddr::from(([127, 0, 0, 1], 1234)); + let err = ClientCredentials::parse(&options, sni, common_names, peer_addr) + .expect_err("should fail"); match err { InconsistentProjectNames { domain, option } => { assert_eq!(option, "first"); @@ -329,7 +384,9 @@ mod tests { let sni = Some("project.localhost"); let common_names = Some(["example.com".into()].into()); - let err = ClientCredentials::parse(&options, sni, common_names).expect_err("should fail"); + let peer_addr = SocketAddr::from(([127, 0, 0, 1], 1234)); + let err = ClientCredentials::parse(&options, sni, common_names, peer_addr) + .expect_err("should fail"); match err { UnknownCommonName { cn } => { assert_eq!(cn, "localhost"); @@ -347,7 +404,8 @@ mod tests { let sni = Some("project.localhost"); let common_names = Some(["localhost".into()].into()); - let creds = ClientCredentials::parse(&options, sni, common_names)?; + let peer_addr = SocketAddr::from(([127, 0, 0, 1], 1234)); + let creds = ClientCredentials::parse(&options, sni, common_names, peer_addr)?; assert_eq!(creds.project.as_deref(), Some("project")); assert_eq!( creds.cache_key, @@ -356,4 +414,91 @@ mod tests { Ok(()) } + + #[test] + fn test_check_peer_addr_is_in_list() { + let peer_addr = IpAddr::from([127, 0, 0, 1]); + assert!(check_peer_addr_is_in_list(&peer_addr, &vec![])); + assert!(check_peer_addr_is_in_list( + &peer_addr, + &vec!["127.0.0.1".into()] + )); + assert!(!check_peer_addr_is_in_list( + &peer_addr, + &vec!["8.8.8.8".into()] + )); + // If there is an incorrect address, it will be skipped. + assert!(check_peer_addr_is_in_list( + &peer_addr, + &vec!["88.8.8".into(), "127.0.0.1".into()] + )); + } + #[test] + fn test_parse_ip_v4() -> anyhow::Result<()> { + let peer_addr = IpAddr::from([127, 0, 0, 1]); + // Ok + assert_eq!(parse_ip_pattern("127.0.0.1")?, IpPattern::Single(peer_addr)); + assert_eq!( + parse_ip_pattern("127.0.0.1/31")?, + IpPattern::Subnet(ipnet::IpNet::new(peer_addr, 31)?) + ); + assert_eq!( + parse_ip_pattern("0.0.0.0-200.0.1.2")?, + IpPattern::Range(IpAddr::from([0, 0, 0, 0]), IpAddr::from([200, 0, 1, 2])) + ); + + // Error + assert!(parse_ip_pattern("300.0.1.2").is_err()); + assert!(parse_ip_pattern("30.1.2").is_err()); + assert!(parse_ip_pattern("127.0.0.1/33").is_err()); + assert!(parse_ip_pattern("127.0.0.1-127.0.3").is_err()); + assert!(parse_ip_pattern("1234.0.0.1-127.0.3.0").is_err()); + Ok(()) + } + + #[test] + fn test_check_ipv4() -> anyhow::Result<()> { + let peer_addr = IpAddr::from([127, 0, 0, 1]); + let peer_addr_next = IpAddr::from([127, 0, 0, 2]); + let peer_addr_prev = IpAddr::from([127, 0, 0, 0]); + // Success + assert!(check_ip(&peer_addr, &IpPattern::Single(peer_addr))); + assert!(check_ip( + &peer_addr, + &IpPattern::Subnet(ipnet::IpNet::new(peer_addr_prev, 31)?) + )); + assert!(check_ip( + &peer_addr, + &IpPattern::Subnet(ipnet::IpNet::new(peer_addr_next, 30)?) + )); + assert!(check_ip( + &peer_addr, + &IpPattern::Range(IpAddr::from([0, 0, 0, 0]), IpAddr::from([200, 0, 1, 2])) + )); + assert!(check_ip( + &peer_addr, + &IpPattern::Range(peer_addr, peer_addr) + )); + + // Not success + assert!(!check_ip(&peer_addr, &IpPattern::Single(peer_addr_prev))); + assert!(!check_ip( + &peer_addr, + &IpPattern::Subnet(ipnet::IpNet::new(peer_addr_next, 31)?) + )); + assert!(!check_ip( + &peer_addr, + &IpPattern::Range(IpAddr::from([0, 0, 0, 0]), peer_addr_prev) + )); + assert!(!check_ip( + &peer_addr, + &IpPattern::Range(peer_addr_next, IpAddr::from([128, 0, 0, 0])) + )); + // There is no check that for range start <= end. But it's fine as long as for all this cases the result is false. + assert!(!check_ip( + &peer_addr, + &IpPattern::Range(peer_addr, peer_addr_prev) + )); + Ok(()) + } } diff --git a/proxy/src/bin/proxy.rs b/proxy/src/bin/proxy.rs index d90ac86a82..7457e26867 100644 --- a/proxy/src/bin/proxy.rs +++ b/proxy/src/bin/proxy.rs @@ -1,8 +1,11 @@ use futures::future::Either; use proxy::auth; use proxy::config::AuthenticationConfig; +use proxy::config::CacheOptions; use proxy::config::HttpConfig; use proxy::console; +use proxy::console::provider::AllowedIpsCache; +use proxy::console::provider::NodeInfoCache; use proxy::http; use proxy::rate_limiter::RateLimiterConfig; use proxy::usage_metrics; @@ -113,6 +116,12 @@ struct ProxyCliArgs { initial_limit: usize, #[clap(flatten)] aimd_config: proxy::rate_limiter::AimdConfig, + /// cache for `allowed_ips` (use `size=0` to disable) + #[clap(long, default_value = config::CacheOptions::DEFAULT_OPTIONS_NODE_INFO)] + allowed_ips_cache: String, + /// disable ip check for http requests. If it is too time consuming, it could be turned off. + #[clap(long, default_value_t = false, value_parser = clap::builder::BoolishValueParser::new(), action = clap::ArgAction::Set)] + disable_ip_check_for_http: bool, } #[tokio::main] @@ -241,11 +250,24 @@ fn build_config(args: &ProxyCliArgs) -> anyhow::Result<&'static ProxyConfig> { let auth_backend = match &args.auth_backend { AuthBackend::Console => { - let config::CacheOptions { size, ttl } = args.wake_compute_cache.parse()?; + let wake_compute_cache_config: CacheOptions = args.wake_compute_cache.parse()?; + let allowed_ips_cache_config: CacheOptions = args.allowed_ips_cache.parse()?; - info!("Using NodeInfoCache (wake_compute) with size={size} ttl={ttl:?}"); + info!("Using NodeInfoCache (wake_compute) with options={wake_compute_cache_config:?}"); + info!("Using AllowedIpsCache (wake_compute) with options={allowed_ips_cache_config:?}"); let caches = Box::leak(Box::new(console::caches::ApiCaches { - node_info: console::caches::NodeInfoCache::new("node_info_cache", size, ttl), + node_info: NodeInfoCache::new( + "node_info_cache", + wake_compute_cache_config.size, + wake_compute_cache_config.ttl, + true, + ), + allowed_ips: AllowedIpsCache::new( + "allowed_ips_cache", + allowed_ips_cache_config.size, + allowed_ips_cache_config.ttl, + false, + ), })); let config::WakeComputeLockOptions { @@ -292,6 +314,7 @@ fn build_config(args: &ProxyCliArgs) -> anyhow::Result<&'static ProxyConfig> { http_config, authentication_config, require_client_ip: args.require_client_ip, + disable_ip_check_for_http: args.disable_ip_check_for_http, })); Ok(config) diff --git a/proxy/src/cache.rs b/proxy/src/cache.rs index a9d6793bbd..f54f360b01 100644 --- a/proxy/src/cache.rs +++ b/proxy/src/cache.rs @@ -55,7 +55,7 @@ pub mod timed_lru { /// * Whenever a new entry is inserted, the least recently accessed one is evicted. /// The cache also keeps track of entry's insertion time (`created_at`) and TTL (`expires_at`). /// - /// * When the entry is about to be retrieved, we check its expiration timestamp. + /// * If `update_ttl_on_retrieval` is `true`. When the entry is about to be retrieved, we check its expiration timestamp. /// If the entry has expired, we remove it from the cache; Otherwise we bump the /// expiration timestamp (e.g. +5mins) and change its place in LRU list to prolong /// its existence. @@ -79,6 +79,8 @@ pub mod timed_lru { /// Default time-to-live of a single entry. ttl: Duration, + + update_ttl_on_retrieval: bool, } impl Cache for TimedLru { @@ -99,11 +101,17 @@ pub mod timed_lru { impl TimedLru { /// Construct a new LRU cache with timed entries. - pub fn new(name: &'static str, capacity: usize, ttl: Duration) -> Self { + pub fn new( + name: &'static str, + capacity: usize, + ttl: Duration, + update_ttl_on_retrieval: bool, + ) -> Self { Self { name, cache: LruCache::new(capacity).into(), ttl, + update_ttl_on_retrieval, } } @@ -165,7 +173,9 @@ pub mod timed_lru { let (created_at, expires_at) = (entry.created_at, entry.expires_at); // Update the deadline and the entry's position in the LRU list. - raw_entry.get_mut().expires_at = deadline; + if self.update_ttl_on_retrieval { + raw_entry.get_mut().expires_at = deadline; + } raw_entry.to_back(); drop(cache); // drop lock before logging diff --git a/proxy/src/config.rs b/proxy/src/config.rs index 89b432df92..182d71f9be 100644 --- a/proxy/src/config.rs +++ b/proxy/src/config.rs @@ -19,6 +19,7 @@ pub struct ProxyConfig { pub http_config: HttpConfig, pub authentication_config: AuthenticationConfig, pub require_client_ip: bool, + pub disable_ip_check_for_http: bool, } #[derive(Debug)] @@ -298,6 +299,7 @@ impl CertResolver { } /// Helper for cmdline cache options parsing. +#[derive(Debug)] pub struct CacheOptions { /// Max number of entries. pub size: usize, diff --git a/proxy/src/console.rs b/proxy/src/console.rs index 6da627389e..07bc807950 100644 --- a/proxy/src/console.rs +++ b/proxy/src/console.rs @@ -6,7 +6,7 @@ pub mod messages; /// Wrappers for console APIs and their mocks. pub mod provider; -pub use provider::{errors, Api, AuthInfo, CachedNodeInfo, ConsoleReqExtra, NodeInfo}; +pub use provider::{errors, Api, AuthSecret, CachedNodeInfo, ConsoleReqExtra, NodeInfo}; /// Various cache-related types. pub mod caches { diff --git a/proxy/src/console/provider.rs b/proxy/src/console/provider.rs index 54bcd1f081..a525de8e53 100644 --- a/proxy/src/console/provider.rs +++ b/proxy/src/console/provider.rs @@ -204,7 +204,7 @@ pub struct ConsoleReqExtra<'a> { } /// Auth secret which is managed by the cloud. -pub enum AuthInfo { +pub enum AuthSecret { /// Md5 hash of user's password. Md5([u8; 16]), @@ -212,6 +212,13 @@ pub enum AuthInfo { Scram(scram::ServerSecret), } +#[derive(Default)] +pub struct AuthInfo { + pub secret: Option, + /// List of IP addresses allowed for the autorization. + pub allowed_ips: Vec, +} + /// Info for establishing a connection to a compute node. /// This is what we get after auth succeeded, but not before! #[derive(Clone)] @@ -230,6 +237,7 @@ pub struct NodeInfo { pub type NodeInfoCache = TimedLru, NodeInfo>; pub type CachedNodeInfo = timed_lru::Cached<&'static NodeInfoCache>; +pub type AllowedIpsCache = TimedLru, Arc>>; /// This will allocate per each call, but the http requests alone /// already require a few allocations, so it should be fine. @@ -240,7 +248,13 @@ pub trait Api { &self, extra: &ConsoleReqExtra<'_>, creds: &ClientCredentials, - ) -> Result, errors::GetAuthInfoError>; + ) -> Result; + + async fn get_allowed_ips( + &self, + extra: &ConsoleReqExtra<'_>, + creds: &ClientCredentials, + ) -> Result>, errors::GetAuthInfoError>; /// Wake up the compute node and return the corresponding connection info. async fn wake_compute( @@ -254,6 +268,8 @@ pub trait Api { pub struct ApiCaches { /// Cache for the `wake_compute` API method. pub node_info: NodeInfoCache, + /// Cache for the `get_allowed_ips`. TODO(anna): use notifications listener instead. + pub allowed_ips: TimedLru, Arc>>, } /// Various caches for [`console`](super). diff --git a/proxy/src/console/provider/mock.rs b/proxy/src/console/provider/mock.rs index 750a2d141e..4cc68f0ac1 100644 --- a/proxy/src/console/provider/mock.rs +++ b/proxy/src/console/provider/mock.rs @@ -1,14 +1,16 @@ //! Mock console backend which relies on a user-provided postgres instance. +use std::sync::Arc; + use super::{ errors::{ApiError, GetAuthInfoError, WakeComputeError}, - AuthInfo, CachedNodeInfo, ConsoleReqExtra, NodeInfo, + AuthInfo, AuthSecret, CachedNodeInfo, ConsoleReqExtra, NodeInfo, }; use crate::{auth::ClientCredentials, compute, error::io_error, scram, url::ApiUrl}; use async_trait::async_trait; use futures::TryFutureExt; use thiserror::Error; -use tokio_postgres::config::SslMode; +use tokio_postgres::{config::SslMode, Client}; use tracing::{error, info, info_span, warn, Instrument}; #[derive(Debug, Error)] @@ -46,8 +48,8 @@ impl Api { async fn do_get_auth_info( &self, creds: &ClientCredentials<'_>, - ) -> Result, GetAuthInfoError> { - async { + ) -> Result { + let (secret, allowed_ips) = async { // Perhaps we could persist this connection, but then we'd have to // write more code for reopening it if it got closed, which doesn't // seem worth it. @@ -55,32 +57,48 @@ impl Api { tokio_postgres::connect(self.endpoint.as_str(), tokio_postgres::NoTls).await?; tokio::spawn(connection); - let query = "select rolpassword from pg_catalog.pg_authid where rolname = $1"; - let rows = client.query(query, &[&creds.user]).await?; - - // We can get at most one row, because `rolname` is unique. - let row = match rows.first() { - Some(row) => row, - // This means that the user doesn't exist, so there can be no secret. - // However, this is still a *valid* outcome which is very similar - // to getting `404 Not found` from the Neon console. + let secret = match get_execute_postgres_query( + &client, + "select rolpassword from pg_catalog.pg_authid where rolname = $1", + &[&creds.user], + "rolpassword", + ) + .await? + { + Some(entry) => { + info!("got a secret: {entry}"); // safe since it's not a prod scenario + let secret = scram::ServerSecret::parse(&entry).map(AuthSecret::Scram); + secret.or_else(|| parse_md5(&entry).map(AuthSecret::Md5)) + } None => { warn!("user '{}' does not exist", creds.user); - return Ok(None); + None } }; + let allowed_ips = match get_execute_postgres_query( + &client, + "select allowed_ips from neon_control_plane.endpoints where endpoint_id = $1", + &[&creds.project.clone().unwrap_or_default().as_str()], + "allowed_ips", + ) + .await? + { + Some(s) => { + info!("got allowed_ips: {s}"); + s.split(',').map(String::from).collect() + } + None => vec![], + }; - let entry = row - .try_get("rolpassword") - .map_err(MockApiError::PasswordNotSet)?; - - info!("got a secret: {entry}"); // safe since it's not a prod scenario - let secret = scram::ServerSecret::parse(entry).map(AuthInfo::Scram); - Ok(secret.or_else(|| parse_md5(entry).map(AuthInfo::Md5))) + Ok((secret, allowed_ips)) } - .map_err(crate::error::log_error) + .map_err(crate::error::log_error::) .instrument(info_span!("postgres", url = self.endpoint.as_str())) - .await + .await?; + Ok(AuthInfo { + secret, + allowed_ips, + }) } async fn do_wake_compute(&self) -> Result { @@ -100,6 +118,27 @@ impl Api { } } +async fn get_execute_postgres_query( + client: &Client, + query: &str, + params: &[&(dyn tokio_postgres::types::ToSql + Sync)], + idx: &str, +) -> Result, GetAuthInfoError> { + let rows = client.query(query, params).await?; + + // We can get at most one row, because `rolname` is unique. + let row = match rows.first() { + Some(row) => row, + // This means that the user doesn't exist, so there can be no secret. + // However, this is still a *valid* outcome which is very similar + // to getting `404 Not found` from the Neon console. + None => return Ok(None), + }; + + let entry = row.try_get(idx).map_err(MockApiError::PasswordNotSet)?; + Ok(Some(entry)) +} + #[async_trait] impl super::Api for Api { #[tracing::instrument(skip_all)] @@ -107,10 +146,18 @@ impl super::Api for Api { &self, _extra: &ConsoleReqExtra<'_>, creds: &ClientCredentials, - ) -> Result, GetAuthInfoError> { + ) -> Result { self.do_get_auth_info(creds).await } + async fn get_allowed_ips( + &self, + _extra: &ConsoleReqExtra<'_>, + creds: &ClientCredentials, + ) -> Result>, GetAuthInfoError> { + Ok(Arc::new(self.do_get_auth_info(creds).await?.allowed_ips)) + } + #[tracing::instrument(skip_all)] async fn wake_compute( &self, diff --git a/proxy/src/console/provider/neon.rs b/proxy/src/console/provider/neon.rs index 0dc7c71534..117d0ec190 100644 --- a/proxy/src/console/provider/neon.rs +++ b/proxy/src/console/provider/neon.rs @@ -3,11 +3,17 @@ use super::{ super::messages::{ConsoleError, GetRoleSecret, WakeCompute}, errors::{ApiError, GetAuthInfoError, WakeComputeError}, - ApiCaches, ApiLocks, AuthInfo, CachedNodeInfo, ConsoleReqExtra, NodeInfo, + ApiCaches, ApiLocks, AuthInfo, AuthSecret, CachedNodeInfo, ConsoleReqExtra, NodeInfo, +}; +use crate::{ + auth::ClientCredentials, + compute, http, + proxy::{ALLOWED_IPS_BY_CACHE_OUTCOME, ALLOWED_IPS_NUMBER}, + scram, }; -use crate::{auth::ClientCredentials, compute, http, scram}; use async_trait::async_trait; use futures::TryFutureExt; +use itertools::Itertools; use std::{net::SocketAddr, sync::Arc}; use tokio::time::Instant; use tokio_postgres::config::SslMode; @@ -48,7 +54,7 @@ impl Api { &self, extra: &ConsoleReqExtra<'_>, creds: &ClientCredentials<'_>, - ) -> Result, GetAuthInfoError> { + ) -> Result { let request_id = uuid::Uuid::new_v4().to_string(); async { let request = self @@ -72,16 +78,25 @@ impl Api { Ok(body) => body, // Error 404 is special: it's ok not to have a secret. Err(e) => match e.http_status_code() { - Some(http::StatusCode::NOT_FOUND) => return Ok(None), + Some(http::StatusCode::NOT_FOUND) => return Ok(AuthInfo::default()), _otherwise => return Err(e.into()), }, }; let secret = scram::ServerSecret::parse(&body.role_secret) - .map(AuthInfo::Scram) + .map(AuthSecret::Scram) .ok_or(GetAuthInfoError::BadSecret)?; - - Ok(Some(secret)) + let allowed_ips = body + .allowed_ips + .into_iter() + .flatten() + .map(String::from) + .collect_vec(); + ALLOWED_IPS_NUMBER.observe(allowed_ips.len() as f64); + Ok(AuthInfo { + secret: Some(secret), + allowed_ips, + }) } .map_err(crate::error::log_error) .instrument(info_span!("http", id = request_id)) @@ -148,10 +163,32 @@ impl super::Api for Api { &self, extra: &ConsoleReqExtra<'_>, creds: &ClientCredentials, - ) -> Result, GetAuthInfoError> { + ) -> Result { self.do_get_auth_info(extra, creds).await } + async fn get_allowed_ips( + &self, + extra: &ConsoleReqExtra<'_>, + creds: &ClientCredentials, + ) -> Result>, GetAuthInfoError> { + let key: &str = creds.project().expect("impossible"); + if let Some(allowed_ips) = self.caches.allowed_ips.get(key) { + ALLOWED_IPS_BY_CACHE_OUTCOME + .with_label_values(&["hit"]) + .inc(); + return Ok(Arc::new(allowed_ips.to_vec())); + } + ALLOWED_IPS_BY_CACHE_OUTCOME + .with_label_values(&["miss"]) + .inc(); + let allowed_ips = Arc::new(self.do_get_auth_info(extra, creds).await?.allowed_ips); + self.caches + .allowed_ips + .insert(key.into(), allowed_ips.clone()); + Ok(allowed_ips) + } + #[tracing::instrument(skip_all)] async fn wake_compute( &self, diff --git a/proxy/src/http.rs b/proxy/src/http.rs index 159b949da3..638705d3e9 100644 --- a/proxy/src/http.rs +++ b/proxy/src/http.rs @@ -13,7 +13,7 @@ pub use reqwest_retry::{policies::ExponentialBackoff, RetryTransientMiddleware}; use tokio::time::Instant; use tracing::trace; -use crate::{rate_limiter, url::ApiUrl}; +use crate::{proxy::CONSOLE_REQUEST_LATENCY, rate_limiter, url::ApiUrl}; use reqwest_middleware::RequestBuilder; /// This is the preferred way to create new http clients, @@ -90,7 +90,13 @@ impl Endpoint { /// Execute a [request](reqwest::Request). pub async fn execute(&self, request: Request) -> Result { - self.client.execute(request).await + let path = request.url().path().to_string(); + let start = Instant::now(); + let res = self.client.execute(request).await; + CONSOLE_REQUEST_LATENCY + .with_label_values(&[&path]) + .observe(start.elapsed().as_micros() as f64); + res } } diff --git a/proxy/src/proxy.rs b/proxy/src/proxy.rs index 9560c8546a..2af2dd5562 100644 --- a/proxy/src/proxy.rs +++ b/proxy/src/proxy.rs @@ -24,7 +24,7 @@ use prometheus::{ IntGaugeVec, }; use regex::Regex; -use std::{error::Error, io, ops::ControlFlow, sync::Arc, time::Instant}; +use std::{error::Error, io, net::SocketAddr, ops::ControlFlow, sync::Arc, time::Instant}; use tokio::{ io::{AsyncRead, AsyncWrite, AsyncWriteExt}, time, @@ -110,12 +110,34 @@ static COMPUTE_CONNECTION_LATENCY: Lazy = Lazy::new(|| { .unwrap() }); +pub static CONSOLE_REQUEST_LATENCY: Lazy = Lazy::new(|| { + register_histogram_vec!( + "proxy_console_request_latency", + "Time it took for proxy to establish a connection to the compute endpoint", + // proxy_wake_compute/proxy_get_role_info + &["request"], + // largest bucket = 2^16 * 0.2ms = 13s + exponential_buckets(0.2, 2.0, 16).unwrap(), + ) + .unwrap() +}); + +pub static ALLOWED_IPS_BY_CACHE_OUTCOME: Lazy = Lazy::new(|| { + register_int_counter_vec!( + "proxy_allowed_ips_cache_misses", + "Number of cache hits/misses for allowed ips", + // hit/miss + &["outcome"], + ) + .unwrap() +}); + pub static RATE_LIMITER_ACQUIRE_LATENCY: Lazy = Lazy::new(|| { register_histogram!( "semaphore_control_plane_token_acquire_seconds", "Time it took for proxy to establish a connection to the compute endpoint", - // largest bucket = 2^16 * 0.5ms = 32s - exponential_buckets(0.0005, 2.0, 16).unwrap(), + // largest bucket = 3^16 * 0.00005s = 3.28s + exponential_buckets(0.00005, 3.0, 16).unwrap(), ) .unwrap() }); @@ -138,6 +160,15 @@ pub static NUM_CONNECTION_ACCEPTED_BY_SNI: Lazy = Lazy::new(|| { .unwrap() }); +pub static ALLOWED_IPS_NUMBER: Lazy = Lazy::new(|| { + register_histogram!( + "proxy_allowed_ips_number", + "Number of allowed ips", + vec![0.0, 1.0, 2.0, 3.0, 4.0, 5.0, 10.0, 20.0, 50.0, 100.0], + ) + .unwrap() +}); + pub struct LatencyTimer { // time since the stopwatch was started start: Option, @@ -265,7 +296,7 @@ pub async fn task_main( loop { tokio::select! { accept_result = listener.accept() => { - let (socket, _) = accept_result?; + let (socket, peer_addr) = accept_result?; let session_id = uuid::Uuid::new_v4(); let cancel_map = Arc::clone(&cancel_map); @@ -274,7 +305,9 @@ pub async fn task_main( info!("accepted postgres client connection"); let mut socket = WithClientIp::new(socket); + let mut peer_addr = peer_addr; if let Some(ip) = socket.wait_for_addr().await? { + peer_addr = ip; tracing::Span::current().record("peer_addr", &tracing::field::display(ip)); } else if config.require_client_ip { bail!("missing required client IP"); @@ -285,7 +318,7 @@ pub async fn task_main( .set_nodelay(true) .context("failed to set socket option")?; - handle_client(config, &cancel_map, session_id, socket, ClientMode::Tcp).await + handle_client(config, &cancel_map, session_id, socket, ClientMode::Tcp, peer_addr).await } .instrument(info_span!("handle_client", ?session_id, peer_addr = tracing::field::Empty)) .unwrap_or_else(move |e| { @@ -375,6 +408,7 @@ pub async fn handle_client( session_id: uuid::Uuid, stream: S, mode: ClientMode, + peer_addr: SocketAddr, ) -> anyhow::Result<()> { info!( protocol = mode.protocol_label(), @@ -408,7 +442,7 @@ pub async fn handle_client( let result = config .auth_backend .as_ref() - .map(|_| auth::ClientCredentials::parse(¶ms, hostname, common_names)) + .map(|_| auth::ClientCredentials::parse(¶ms, hostname, common_names, peer_addr)) .transpose(); match result { diff --git a/proxy/src/proxy/tests.rs b/proxy/src/proxy/tests.rs index de9cc0800b..b97c0efce4 100644 --- a/proxy/src/proxy/tests.rs +++ b/proxy/src/proxy/tests.rs @@ -466,6 +466,10 @@ impl TestBackend for TestConnectMechanism { x => panic!("expecting action {:?}, wake_compute is called instead", x), } } + + fn get_allowed_ips(&self) -> Result>, console::errors::GetAuthInfoError> { + unimplemented!("not used in tests") + } } fn helper_create_cached_node_info() -> CachedNodeInfo { diff --git a/proxy/src/serverless.rs b/proxy/src/serverless.rs index 23deda3ae6..45f8132393 100644 --- a/proxy/src/serverless.rs +++ b/proxy/src/serverless.rs @@ -23,6 +23,7 @@ use hyper::{ Body, Method, Request, Response, }; +use std::net::SocketAddr; use std::task::Poll; use std::{future::ready, sync::Arc}; use tls_listener::TlsListener; @@ -102,7 +103,7 @@ pub async fn task_main( let session_id = uuid::Uuid::new_v4(); request_handler( - req, config, conn_pool, cancel_map, session_id, sni_name, + req, config, conn_pool, cancel_map, session_id, sni_name, peer_addr, ) .instrument(info_span!( "serverless", @@ -170,6 +171,7 @@ async fn request_handler( cancel_map: Arc, session_id: uuid::Uuid, sni_hostname: Option, + peer_addr: SocketAddr, ) -> Result, ApiError> { let host = request .headers() @@ -187,9 +189,15 @@ async fn request_handler( tokio::spawn( async move { - if let Err(e) = - websocket::serve_websocket(websocket, config, &cancel_map, session_id, host) - .await + if let Err(e) = websocket::serve_websocket( + websocket, + config, + &cancel_map, + session_id, + host, + peer_addr, + ) + .await { error!(session_id = ?session_id, "error in websocket connection: {e:#}"); } @@ -205,6 +213,7 @@ async fn request_handler( sni_hostname, conn_pool, session_id, + peer_addr, &config.http_config, ) .await diff --git a/proxy/src/serverless/conn_pool.rs b/proxy/src/serverless/conn_pool.rs index b753bc8918..2072cadc3a 100644 --- a/proxy/src/serverless/conn_pool.rs +++ b/proxy/src/serverless/conn_pool.rs @@ -8,7 +8,7 @@ use pbkdf2::{ Params, Pbkdf2, }; use pq_proto::StartupMessageParams; -use std::{collections::HashMap, sync::Arc}; +use std::{collections::HashMap, net::SocketAddr, sync::Arc}; use std::{ fmt, task::{ready, Poll}, @@ -21,7 +21,8 @@ use tokio::time; use tokio_postgres::{AsyncMessage, ReadyForQueryStatus}; use crate::{ - auth, console, + auth::{self, check_peer_addr_is_in_list}, + console, proxy::{ neon_options, LatencyTimer, NUM_DB_CONNECTIONS_CLOSED_COUNTER, NUM_DB_CONNECTIONS_OPENED_COUNTER, @@ -144,6 +145,7 @@ impl GlobalConnPool { conn_info: &ConnInfo, force_new: bool, session_id: uuid::Uuid, + peer_addr: SocketAddr, ) -> anyhow::Result { let mut client: Option = None; let mut latency_timer = LatencyTimer::new("http"); @@ -203,6 +205,7 @@ impl GlobalConnPool { conn_id, session_id, latency_timer, + peer_addr, ) .await } else { @@ -225,6 +228,7 @@ impl GlobalConnPool { conn_id, session_id, latency_timer, + peer_addr, ) .await }; @@ -401,6 +405,7 @@ async fn connect_to_compute( conn_id: uuid::Uuid, session_id: uuid::Uuid, latency_timer: LatencyTimer, + peer_addr: SocketAddr, ) -> anyhow::Result { let tls = config.tls_config.as_ref(); let common_names = tls.and_then(|tls| tls.common_names.clone()); @@ -411,12 +416,13 @@ async fn connect_to_compute( ("application_name", APP_NAME), ("options", conn_info.options.as_deref().unwrap_or("")), ]); - - let creds = config - .auth_backend - .as_ref() - .map(|_| auth::ClientCredentials::parse(¶ms, Some(&conn_info.hostname), common_names)) - .transpose()?; + let creds = auth::ClientCredentials::parse( + ¶ms, + Some(&conn_info.hostname), + common_names, + peer_addr, + )?; + let backend = config.auth_backend.as_ref().map(|_| creds); let console_options = neon_options(¶ms); @@ -425,8 +431,14 @@ async fn connect_to_compute( application_name: Some(APP_NAME), options: console_options.as_deref(), }; - - let node_info = creds + // TODO(anna): this is a bit hacky way, consider using console notification listener. + if !config.disable_ip_check_for_http { + let allowed_ips = backend.get_allowed_ips(&extra).await?; + if !check_peer_addr_is_in_list(&peer_addr.ip(), &allowed_ips) { + return Err(auth::AuthError::ip_address_not_allowed().into()); + } + } + let node_info = backend .wake_compute(&extra) .await? .context("missing cache entry from wake_compute")?; @@ -439,7 +451,7 @@ async fn connect_to_compute( }, node_info, &extra, - &creds, + &backend, latency_timer, ) .await diff --git a/proxy/src/serverless/sql_over_http.rs b/proxy/src/serverless/sql_over_http.rs index 2df2be1d3d..25b96668de 100644 --- a/proxy/src/serverless/sql_over_http.rs +++ b/proxy/src/serverless/sql_over_http.rs @@ -1,3 +1,4 @@ +use std::net::SocketAddr; use std::sync::Arc; use anyhow::bail; @@ -201,11 +202,19 @@ pub async fn handle( sni_hostname: Option, conn_pool: Arc, session_id: uuid::Uuid, + peer_addr: SocketAddr, config: &'static HttpConfig, ) -> Result, ApiError> { let result = tokio::time::timeout( config.timeout, - handle_inner(config, request, sni_hostname, conn_pool, session_id), + handle_inner( + config, + request, + sni_hostname, + conn_pool, + session_id, + peer_addr, + ), ) .await; let mut response = match result { @@ -292,6 +301,7 @@ async fn handle_inner( sni_hostname: Option, conn_pool: Arc, session_id: uuid::Uuid, + peer_addr: SocketAddr, ) -> anyhow::Result> { NUM_CONNECTIONS_ACCEPTED_COUNTER .with_label_values(&["http"]) @@ -351,7 +361,9 @@ async fn handle_inner( let body = hyper::body::to_bytes(request.into_body()).await?; let payload: Payload = serde_json::from_slice(&body)?; - let mut client = conn_pool.get(&conn_info, !allow_pool, session_id).await?; + let mut client = conn_pool + .get(&conn_info, !allow_pool, session_id, peer_addr) + .await?; let mut response = Response::builder() .status(StatusCode::OK) diff --git a/proxy/src/serverless/websocket.rs b/proxy/src/serverless/websocket.rs index 86141ab64f..8fb9a3dee4 100644 --- a/proxy/src/serverless/websocket.rs +++ b/proxy/src/serverless/websocket.rs @@ -11,6 +11,7 @@ use hyper_tungstenite::{tungstenite::Message, HyperWebsocket, WebSocketStream}; use pin_project_lite::pin_project; use std::{ + net::SocketAddr, pin::Pin, task::{ready, Context, Poll}, }; @@ -132,6 +133,7 @@ pub async fn serve_websocket( cancel_map: &CancelMap, session_id: uuid::Uuid, hostname: Option, + peer_addr: SocketAddr, ) -> anyhow::Result<()> { let websocket = websocket.await?; handle_client( @@ -140,6 +142,7 @@ pub async fn serve_websocket( session_id, WebSocketRw::new(websocket), ClientMode::Websockets { hostname }, + peer_addr, ) .await?; Ok(()) diff --git a/test_runner/fixtures/neon_fixtures.py b/test_runner/fixtures/neon_fixtures.py index 23a36ad6c9..862aab84dc 100644 --- a/test_runner/fixtures/neon_fixtures.py +++ b/test_runner/fixtures/neon_fixtures.py @@ -2390,6 +2390,10 @@ def static_proxy( # For simplicity, we use the same user for both `--auth-endpoint` and `safe_psql` vanilla_pg.start() vanilla_pg.safe_psql("create user proxy with login superuser password 'password'") + vanilla_pg.safe_psql("CREATE SCHEMA IF NOT EXISTS neon_control_plane") + vanilla_pg.safe_psql( + "CREATE TABLE neon_control_plane.endpoints (endpoint_id VARCHAR(255) PRIMARY KEY, allowed_ips VARCHAR(255))" + ) proxy_port = port_distributor.get_port() mgmt_port = port_distributor.get_port() diff --git a/test_runner/regress/test_proxy_allowed_ips.py b/test_runner/regress/test_proxy_allowed_ips.py new file mode 100644 index 0000000000..f533579811 --- /dev/null +++ b/test_runner/regress/test_proxy_allowed_ips.py @@ -0,0 +1,74 @@ +import psycopg2 +import pytest +from fixtures.neon_fixtures import ( + NeonProxy, + VanillaPostgres, +) + +TABLE_NAME = "neon_control_plane.endpoints" + + +# Proxy uses the same logic for psql and websockets. +@pytest.mark.asyncio +async def test_proxy_psql_allowed_ips(static_proxy: NeonProxy, vanilla_pg: VanillaPostgres): + # Shouldn't be able to connect to this project + vanilla_pg.safe_psql( + f"INSERT INTO {TABLE_NAME} (endpoint_id, allowed_ips) VALUES ('private-project', '8.8.8.8')" + ) + # Should be able to connect to this project + vanilla_pg.safe_psql( + f"INSERT INTO {TABLE_NAME} (endpoint_id, allowed_ips) VALUES ('generic-project', '::1,127.0.0.1')" + ) + + def check_cannot_connect(**kwargs): + with pytest.raises(psycopg2.Error) as exprinfo: + static_proxy.safe_psql(**kwargs) + text = str(exprinfo.value).strip() + assert "This IP address is not allowed to connect" in text + + # no SNI, deprecated `options=project` syntax (before we had several endpoint in project) + check_cannot_connect(query="select 1", sslsni=0, options="project=private-project") + + # no SNI, new `options=endpoint` syntax + check_cannot_connect(query="select 1", sslsni=0, options="endpoint=private-project") + + # with SNI + check_cannot_connect(query="select 1", host="private-project.localtest.me") + + # no SNI, deprecated `options=project` syntax (before we had several endpoint in project) + out = static_proxy.safe_psql(query="select 1", sslsni=0, options="project=generic-project") + assert out[0][0] == 1 + + # no SNI, new `options=endpoint` syntax + out = static_proxy.safe_psql(query="select 1", sslsni=0, options="endpoint=generic-project") + assert out[0][0] == 1 + + # with SNI + out = static_proxy.safe_psql(query="select 1", host="generic-project.localtest.me") + assert out[0][0] == 1 + + +@pytest.mark.asyncio +async def test_proxy_http_allowed_ips(static_proxy: NeonProxy, vanilla_pg: VanillaPostgres): + static_proxy.safe_psql("create user http_auth with password 'http' superuser") + + # Shouldn't be able to connect to this project + vanilla_pg.safe_psql( + f"INSERT INTO {TABLE_NAME} (endpoint_id, allowed_ips) VALUES ('proxy', '8.8.8.8')" + ) + + def query(status: int, query: str, *args): + static_proxy.http_query( + query, + args, + user="http_auth", + password="http", + expected_code=status, + ) + + query(400, "select 1;") # ip address is not allowed + # Should be able to connect to this project + vanilla_pg.safe_psql( + f"UPDATE {TABLE_NAME} SET allowed_ips = '8.8.8.8,127.0.0.1' WHERE endpoint_id = 'proxy'" + ) + query(200, "select 1;") # should work now From 8ec6033ed8da9565891e0dbec7de9fbb04dd17dc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arpad=20M=C3=BCller?= Date: Thu, 30 Nov 2023 14:30:17 +0100 Subject: [PATCH 19/38] Pageserver disaster recovery RFC (#5248) Enable the pageserver to recover from data corruption events by implementing a feature to re-apply historic WAL records in parallel to the already occurring WAL replay. The feature is outside of the user-visible backup and history story, and only serves as a second-level backup for the case that there is a bug in the pageservers that corrupted the served pages. The RFC proposes the addition of two new features: * recover a broken branch from WAL (downtime is allowed) * a test recovery system to recover random branches to make sure recovery works --- .../027-pageserver-wal-disaster-recovery.md | 205 ++++++++++++++++++ 1 file changed, 205 insertions(+) create mode 100644 docs/rfcs/027-pageserver-wal-disaster-recovery.md diff --git a/docs/rfcs/027-pageserver-wal-disaster-recovery.md b/docs/rfcs/027-pageserver-wal-disaster-recovery.md new file mode 100644 index 0000000000..15ebd72bfe --- /dev/null +++ b/docs/rfcs/027-pageserver-wal-disaster-recovery.md @@ -0,0 +1,205 @@ +# Name + +Created on: 2023-09-08 +Author: Arpad Müller + +## Summary + +Enable the pageserver to recover from data corruption events by implementing +a feature to re-apply historic WAL records in parallel to the already occurring +WAL replay. + +The feature is outside of the user-visible backup and history story, and only +serves as a second-level backup for the case that there is a bug in the +pageservers that corrupted the served pages. + +The RFC proposes the addition of two new features: +* recover a broken branch from WAL (downtime is allowed) +* a test recovery system to recover random branches to make sure recovery works + +## Motivation + +The historic WAL is currently stored in S3 even after it has been replayed by +the pageserver and thus been integrated into the pageserver's storage system. +This is done to defend from data corruption failures inside the pageservers. + +However, application of this WAL in the disaster recovery setting is currently +very manual and we want to automate this to make it easier. + +### Use cases + +There are various use cases for this feature, like: + +* The main motivation is replaying in the instance of pageservers corrupting + data. +* We might want to, beyond the user-visible history features, through our + support channels and upon customer request, in select instances, recover + historic versions beyond the range of history that we officially support. +* Running the recovery process in the background for random tenant timelines + to figure out if there was a corruption of data (we would compare with what + the pageserver stores for the "official" timeline). +* Using the WAL to arrive at historic pages we can then back up to S3 so that + WAL itself can be discarded, or at least not used for future replays. + Again, this sounds a lot like what the pageserver is already doing, but the + point is to provide a fallback to the service provided by the pageserver. + +## Design + +### Design constraints + +The main design constraint is that the feature needs to be *simple* enough that +the number of bugs are as low, and reliability as high as possible: the main +goal of this endeavour is to achieve higher correctness than the pageserver. + +For the background process, we cannot afford a downtime of the timeline that is +being cloned, as we don't want to restrict ourselves to offline tenants only. +In the scenario where we want to recover from disasters or roll back to a +historic lsn through support staff, downtimes are more affordable, and +inevitable if the original had been subject to the corruption. Ideally, the +two code paths would share code, so the solution would be designed for not +requiring downtimes. + +### API endpoint changes + +This RFC proposes two API endpoint changes in the safekeeper and the +pageserver. + +Remember, the pageserver timeline API creation endpoint is to this URL: + +``` +/v1/tenant/{tenant_id}/timeline/ +``` + +Where `{tenant_id}` is the ID of the tenant the timeline is created for, +and specified as part of the URL. The timeline ID is passed via the POST +request body as the only required parameter `new_timeline_id`. + +This proposal adds one optional parameter called +`existing_initdb_timeline_id` to the request's json body. If the parameter +is not specified, behaviour should be as existing, so the pageserver runs +initdb. +If the parameter is specified, it is expected to point to a timeline ID. +In fact that ID might match `new_timeline_id`, what's important is that +S3 storage contains a matching initdb under the URL matching the given +tenant and timeline. + +Having both `ancestor_timeline_id` and `existing_initdb_timeline_id` +specified is illegal and will yield in an HTTP error. This feature is +only meant for the "main" branch that doesn't have any ancestors +of its own, as only here initdb is relevant. + +For the safekeeper, we propose the addition of the following copy endpoint: + +``` +/v1/tenant/{tenant_id}/timeline/{source_timeline_id}/copy +``` +it is meant for POST requests with json, and the two URL parameters +`tenant_id` and `source_timeline_id`. The json request body contains +the two required parameters `target_timeline_id` and `until_lsn`. + +After invoking, the copy endpoint starts a copy process of the WAL from +the source ID to the target ID. The lsn is updated according to the +progress of the API call. + +### Higher level features + +We want the API changes to support the following higher level features: + +* recovery-after-corruption DR of the main timeline of a tenant. This + feature allows for downtime. +* test DR of the main timeline into a special copy timeline. this feature + is meant to run against selected production tenants in the background, + without the user noticing, so it does not allow for downtime. + +The recovery-after-corruption DR only needs the pageserver changes. +It works as follows: + +* delete the timeline from the pageservers via timeline deletion API +* re-create it via timeline creation API (same ID as before) and set + `existing_initdb_timeline_id` to the same timeline ID + +The test DR requires also the copy primitive and works as follows: + +* copy the WAL of the timeline to a new place +* create a new timeline for the tenant + +## Non Goals + +At the danger of being repetitive, the main goal of this feature is to be a +backup method, so reliability is very important. This implies that other +aspects like performance or space reduction are less important. + +### Corrupt WAL + +The process suggested by this RFC assumes that the WAL is free of corruption. +In some instances, corruption can make it into WAL, like for example when +higher level components like postgres or the application first read corrupt +data, and then execute a write with data derived from that earlier read. That +written data might then contain the corruption. + +Common use cases can hit this quite easily. For example, an application reads +some counter, increments it, and then writes the new counter value to the +database. +On a lower level, the compute might put FPIs (Full Page Images) into the WAL, +which have corrupt data for rows unrelated to the write operation at hand. + +Separating corrupt writes from non-corrupt ones is a hard problem in general, +and if the application was involved in making the corrupt write, a recovery +would also involve the application. Therefore, corruption that has made it into +the WAL is outside of the scope of this feature. However, the WAL replay can be +issued to right before the point in time where the corruption occured. Then the +data loss is isolated to post-corruption writes only. + +## Impacted components (e.g. pageserver, safekeeper, console, etc) + +Most changes would happen to the pageservers. +For the higher level features, maybe other components like the console would +be involved. + +We need to make sure that the shadow timelines are not subject to the usual +limits and billing we apply to existing timelines. + +## Proposed implementation + +The first problem to keep in mind is the reproducability of `initdb`. +So an initial step would be to upload `initdb` snapshots to S3. + +After that, we'd have the endpoint spawn a background process which +performs the replay of the WAL to that new timeline. This process should +follow the existing workflows as closely as possible, just using the +WAL records of a different timeline. + +The timeline created will be in a special state that solely looks for WAL +entries of the timeline it is trying to copy. Once the target LSN is reached, +it turns into a normal timeline that also accepts writes to its own +timeline ID. + +### Scalability + +For now we want to run this entire process on a single node, and as +it is by nature linear, it's hard to parallelize. However, for the +verification workloads, we can easily start the WAL replay in parallel +for different points in time. This is valuable especially for tenants +with large WAL records. + +Compare this with the tricks to make addition circuits execute with +lower latency by making them perform the addition for both possible +values of the carry bit, and then, in a second step, taking the +result for the carry bit that was actually obtained. + +The other scalability dimension to consider is the WAL length, which +is a growing question as tenants accumulate changes. There are +possible approaches to this, including creating snapshots of the +page files and uploading them to S3, but if we do this for every single +branch, we lose the cheap branching property. + +### Implementation by component + +The proposed changes for the various components of the neon architecture +are written up in this notion page: + +https://www.notion.so/neondatabase/Pageserver-disaster-recovery-one-pager-4ecfb5df16ce4f6bbfc3817ed1a6cbb2 + +### Unresolved questions + +none known (outside of the mentioned ones). From 0c87d1866b05f62e882df9a6bf9d215ff5b5aaa2 Mon Sep 17 00:00:00 2001 From: Conrad Ludgate Date: Thu, 30 Nov 2023 13:43:21 +0000 Subject: [PATCH 20/38] proxy: fix wake_compute error prop (#5989) ## Problem fixes #5654 - WakeComputeErrors occuring during a connect_to_compute got propagated as IO errors, which get forwarded to the user as "Couldn't connect to compute node" with no helpful message. ## Summary of changes Handle WakeComputeError during ConnectionError properly --- proxy/src/compute.rs | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/proxy/src/compute.rs b/proxy/src/compute.rs index 0741ad0623..c838c8fc38 100644 --- a/proxy/src/compute.rs +++ b/proxy/src/compute.rs @@ -1,9 +1,6 @@ use crate::{ - auth::parse_endpoint_param, - cancellation::CancelClosure, - console::errors::WakeComputeError, - error::{io_error, UserFacingError}, - proxy::is_neon_param, + auth::parse_endpoint_param, cancellation::CancelClosure, console::errors::WakeComputeError, + error::UserFacingError, proxy::is_neon_param, }; use futures::{FutureExt, TryFutureExt}; use itertools::Itertools; @@ -28,12 +25,9 @@ pub enum ConnectionError { #[error("{COULD_NOT_CONNECT}: {0}")] TlsError(#[from] native_tls::Error), -} -impl From for ConnectionError { - fn from(value: WakeComputeError) -> Self { - io_error(value).into() - } + #[error("{COULD_NOT_CONNECT}: {0}")] + WakeComputeError(#[from] WakeComputeError), } impl UserFacingError for ConnectionError { @@ -46,6 +40,7 @@ impl UserFacingError for ConnectionError { Some(err) => err.message().to_owned(), None => err.to_string(), }, + WakeComputeError(err) => err.to_string_client(), _ => COULD_NOT_CONNECT.to_owned(), } } From 5d3c3636fcab796919587d1ab219a6afa690dfbb Mon Sep 17 00:00:00 2001 From: John Spray Date: Thu, 30 Nov 2023 13:44:14 +0000 Subject: [PATCH 21/38] tests: add a log allow list entry in `test_timeline_deletion_with_files_stuck_in_upload_queue` (#5981) Test failure seen here: https://neon-github-public-dev.s3.amazonaws.com/reports/pr-5860/7032903218/index.html#suites/837740b64a53e769572c4ed7b7a7eeeb/c0f1c79a70a3b9ab ``` E AssertionError: assert not [(302, '2023-11-29T13:23:51.046801Z ERROR request{method=PUT path=/v1/tenant/f6b845de60cb0e92f4426e0d6af1d2ea/timeline/69a8c98004abe71a281cff8642a45274/checkpoint request_id=eca33d8a-7af2-46e7-92ab-c28629feb42c}: Error processing HTTP request: InternalServerError(queue is in state Stopped\n')] ``` This appears to be a legitimate log: the test is issuing checkpoint requests in the background, and deleting (therefore shutting down) a timeline. --- test_runner/regress/test_remote_storage.py | 1 + 1 file changed, 1 insertion(+) diff --git a/test_runner/regress/test_remote_storage.py b/test_runner/regress/test_remote_storage.py index 4da8dd957d..ccf28cae8c 100644 --- a/test_runner/regress/test_remote_storage.py +++ b/test_runner/regress/test_remote_storage.py @@ -588,6 +588,7 @@ def test_timeline_deletion_with_files_stuck_in_upload_queue( env.pageserver.allowed_errors.extend( [ ".* ERROR .*Error processing HTTP request: InternalServerError\\(The timeline or pageserver is shutting down", + ".* ERROR .*queue is in state Stopped.*", ".* ERROR .*[Cc]ould not flush frozen layer.*", ] ) From 3bb1030f5df96f56adf8982f6880f9e715f0a920 Mon Sep 17 00:00:00 2001 From: Christian Schwarz Date: Thu, 30 Nov 2023 15:07:41 +0100 Subject: [PATCH 22/38] walingest: refactor if-cascade on `decoded.xl_rmid` into match statement (#5974) refs https://github.com/neondatabase/neon/issues/5962 --------- Co-authored-by: Joonas Koivunen --- pageserver/src/walingest.rs | 483 ++++++++++++++++++------------------ 1 file changed, 240 insertions(+), 243 deletions(-) diff --git a/pageserver/src/walingest.rs b/pageserver/src/walingest.rs index a9173b41e9..4189200d5c 100644 --- a/pageserver/src/walingest.rs +++ b/pageserver/src/walingest.rs @@ -98,261 +98,258 @@ impl<'a> WalIngest<'a> { self.checkpoint_modified = true; } - // Heap AM records need some special handling, because they modify VM pages - // without registering them with the standard mechanism. - if decoded.xl_rmid == pg_constants::RM_HEAP_ID - || decoded.xl_rmid == pg_constants::RM_HEAP2_ID - { - self.ingest_heapam_record(&mut buf, modification, decoded, ctx) - .await?; - } - if decoded.xl_rmid == pg_constants::RM_NEON_ID { - self.ingest_neonrmgr_record(&mut buf, modification, decoded, ctx) - .await?; - } - // Handle other special record types - if decoded.xl_rmid == pg_constants::RM_SMGR_ID - && (decoded.xl_info & pg_constants::XLR_RMGR_INFO_MASK) - == pg_constants::XLOG_SMGR_CREATE - { - let create = XlSmgrCreate::decode(&mut buf); - self.ingest_xlog_smgr_create(modification, &create, ctx) - .await?; - } else if decoded.xl_rmid == pg_constants::RM_SMGR_ID - && (decoded.xl_info & pg_constants::XLR_RMGR_INFO_MASK) - == pg_constants::XLOG_SMGR_TRUNCATE - { - let truncate = XlSmgrTruncate::decode(&mut buf); - self.ingest_xlog_smgr_truncate(modification, &truncate, ctx) - .await?; - } else if decoded.xl_rmid == pg_constants::RM_DBASE_ID { - debug!( - "handle RM_DBASE_ID for Postgres version {:?}", - self.timeline.pg_version - ); - if self.timeline.pg_version == 14 { - if (decoded.xl_info & pg_constants::XLR_RMGR_INFO_MASK) - == postgres_ffi::v14::bindings::XLOG_DBASE_CREATE - { - let createdb = XlCreateDatabase::decode(&mut buf); - debug!("XLOG_DBASE_CREATE v14"); + match decoded.xl_rmid { + pg_constants::RM_HEAP_ID | pg_constants::RM_HEAP2_ID => { + // Heap AM records need some special handling, because they modify VM pages + // without registering them with the standard mechanism. + self.ingest_heapam_record(&mut buf, modification, decoded, ctx) + .await?; + } + pg_constants::RM_NEON_ID => { + self.ingest_neonrmgr_record(&mut buf, modification, decoded, ctx) + .await?; + } + // Handle other special record types + pg_constants::RM_SMGR_ID => { + let info = decoded.xl_info & pg_constants::XLR_RMGR_INFO_MASK; - self.ingest_xlog_dbase_create(modification, &createdb, ctx) + if info == pg_constants::XLOG_SMGR_CREATE { + let create = XlSmgrCreate::decode(&mut buf); + self.ingest_xlog_smgr_create(modification, &create, ctx) .await?; - } else if (decoded.xl_info & pg_constants::XLR_RMGR_INFO_MASK) - == postgres_ffi::v14::bindings::XLOG_DBASE_DROP - { - let dropdb = XlDropDatabase::decode(&mut buf); - for tablespace_id in dropdb.tablespace_ids { - trace!("Drop db {}, {}", tablespace_id, dropdb.db_id); - modification - .drop_dbdir(tablespace_id, dropdb.db_id, ctx) - .await?; - } - } - } else if self.timeline.pg_version == 15 { - if (decoded.xl_info & pg_constants::XLR_RMGR_INFO_MASK) - == postgres_ffi::v15::bindings::XLOG_DBASE_CREATE_WAL_LOG - { - debug!("XLOG_DBASE_CREATE_WAL_LOG: noop"); - } else if (decoded.xl_info & pg_constants::XLR_RMGR_INFO_MASK) - == postgres_ffi::v15::bindings::XLOG_DBASE_CREATE_FILE_COPY - { - // The XLOG record was renamed between v14 and v15, - // but the record format is the same. - // So we can reuse XlCreateDatabase here. - debug!("XLOG_DBASE_CREATE_FILE_COPY"); - let createdb = XlCreateDatabase::decode(&mut buf); - self.ingest_xlog_dbase_create(modification, &createdb, ctx) + } else if info == pg_constants::XLOG_SMGR_TRUNCATE { + let truncate = XlSmgrTruncate::decode(&mut buf); + self.ingest_xlog_smgr_truncate(modification, &truncate, ctx) .await?; - } else if (decoded.xl_info & pg_constants::XLR_RMGR_INFO_MASK) - == postgres_ffi::v15::bindings::XLOG_DBASE_DROP - { - let dropdb = XlDropDatabase::decode(&mut buf); - for tablespace_id in dropdb.tablespace_ids { - trace!("Drop db {}, {}", tablespace_id, dropdb.db_id); - modification - .drop_dbdir(tablespace_id, dropdb.db_id, ctx) - .await?; - } - } - } else if self.timeline.pg_version == 16 { - if (decoded.xl_info & pg_constants::XLR_RMGR_INFO_MASK) - == postgres_ffi::v16::bindings::XLOG_DBASE_CREATE_WAL_LOG - { - debug!("XLOG_DBASE_CREATE_WAL_LOG: noop"); - } else if (decoded.xl_info & pg_constants::XLR_RMGR_INFO_MASK) - == postgres_ffi::v16::bindings::XLOG_DBASE_CREATE_FILE_COPY - { - // The XLOG record was renamed between v14 and v15, - // but the record format is the same. - // So we can reuse XlCreateDatabase here. - debug!("XLOG_DBASE_CREATE_FILE_COPY"); - let createdb = XlCreateDatabase::decode(&mut buf); - self.ingest_xlog_dbase_create(modification, &createdb, ctx) - .await?; - } else if (decoded.xl_info & pg_constants::XLR_RMGR_INFO_MASK) - == postgres_ffi::v16::bindings::XLOG_DBASE_DROP - { - let dropdb = XlDropDatabase::decode(&mut buf); - for tablespace_id in dropdb.tablespace_ids { - trace!("Drop db {}, {}", tablespace_id, dropdb.db_id); - modification - .drop_dbdir(tablespace_id, dropdb.db_id, ctx) - .await?; - } } } - } else if decoded.xl_rmid == pg_constants::RM_TBLSPC_ID { - trace!("XLOG_TBLSPC_CREATE/DROP is not handled yet"); - } else if decoded.xl_rmid == pg_constants::RM_CLOG_ID { - let info = decoded.xl_info & !pg_constants::XLR_INFO_MASK; - if info == pg_constants::CLOG_ZEROPAGE { - let pageno = buf.get_u32_le(); - let segno = pageno / pg_constants::SLRU_PAGES_PER_SEGMENT; - let rpageno = pageno % pg_constants::SLRU_PAGES_PER_SEGMENT; - self.put_slru_page_image( - modification, - SlruKind::Clog, - segno, - rpageno, - ZERO_PAGE.clone(), - ctx, - ) - .await?; - } else { - assert!(info == pg_constants::CLOG_TRUNCATE); - let xlrec = XlClogTruncate::decode(&mut buf); - self.ingest_clog_truncate_record(modification, &xlrec, ctx) - .await?; - } - } else if decoded.xl_rmid == pg_constants::RM_XACT_ID { - let info = decoded.xl_info & pg_constants::XLOG_XACT_OPMASK; - if info == pg_constants::XLOG_XACT_COMMIT || info == pg_constants::XLOG_XACT_ABORT { - let parsed_xact = - XlXactParsedRecord::decode(&mut buf, decoded.xl_xid, decoded.xl_info); - self.ingest_xact_record( - modification, - &parsed_xact, - info == pg_constants::XLOG_XACT_COMMIT, - ctx, - ) - .await?; - } else if info == pg_constants::XLOG_XACT_COMMIT_PREPARED - || info == pg_constants::XLOG_XACT_ABORT_PREPARED - { - let parsed_xact = - XlXactParsedRecord::decode(&mut buf, decoded.xl_xid, decoded.xl_info); - self.ingest_xact_record( - modification, - &parsed_xact, - info == pg_constants::XLOG_XACT_COMMIT_PREPARED, - ctx, - ) - .await?; - // Remove twophase file. see RemoveTwoPhaseFile() in postgres code - trace!( - "Drop twophaseFile for xid {} parsed_xact.xid {} here at {}", - decoded.xl_xid, - parsed_xact.xid, - lsn, - ); - modification - .drop_twophase_file(parsed_xact.xid, ctx) - .await?; - } else if info == pg_constants::XLOG_XACT_PREPARE { - modification - .put_twophase_file(decoded.xl_xid, Bytes::copy_from_slice(&buf[..]), ctx) - .await?; - } - } else if decoded.xl_rmid == pg_constants::RM_MULTIXACT_ID { - let info = decoded.xl_info & pg_constants::XLR_RMGR_INFO_MASK; + pg_constants::RM_DBASE_ID => { + let info = decoded.xl_info & pg_constants::XLR_RMGR_INFO_MASK; + debug!(%info, pg_version=%self.timeline.pg_version, "handle RM_DBASE_ID"); - if info == pg_constants::XLOG_MULTIXACT_ZERO_OFF_PAGE { - let pageno = buf.get_u32_le(); - let segno = pageno / pg_constants::SLRU_PAGES_PER_SEGMENT; - let rpageno = pageno % pg_constants::SLRU_PAGES_PER_SEGMENT; - self.put_slru_page_image( - modification, - SlruKind::MultiXactOffsets, - segno, - rpageno, - ZERO_PAGE.clone(), - ctx, - ) - .await?; - } else if info == pg_constants::XLOG_MULTIXACT_ZERO_MEM_PAGE { - let pageno = buf.get_u32_le(); - let segno = pageno / pg_constants::SLRU_PAGES_PER_SEGMENT; - let rpageno = pageno % pg_constants::SLRU_PAGES_PER_SEGMENT; - self.put_slru_page_image( - modification, - SlruKind::MultiXactMembers, - segno, - rpageno, - ZERO_PAGE.clone(), - ctx, - ) - .await?; - } else if info == pg_constants::XLOG_MULTIXACT_CREATE_ID { - let xlrec = XlMultiXactCreate::decode(&mut buf); - self.ingest_multixact_create_record(modification, &xlrec)?; - } else if info == pg_constants::XLOG_MULTIXACT_TRUNCATE_ID { - let xlrec = XlMultiXactTruncate::decode(&mut buf); - self.ingest_multixact_truncate_record(modification, &xlrec, ctx) + if self.timeline.pg_version == 14 { + if info == postgres_ffi::v14::bindings::XLOG_DBASE_CREATE { + let createdb = XlCreateDatabase::decode(&mut buf); + debug!("XLOG_DBASE_CREATE v14"); + + self.ingest_xlog_dbase_create(modification, &createdb, ctx) + .await?; + } else if info == postgres_ffi::v14::bindings::XLOG_DBASE_DROP { + let dropdb = XlDropDatabase::decode(&mut buf); + for tablespace_id in dropdb.tablespace_ids { + trace!("Drop db {}, {}", tablespace_id, dropdb.db_id); + modification + .drop_dbdir(tablespace_id, dropdb.db_id, ctx) + .await?; + } + } + } else if self.timeline.pg_version == 15 { + if info == postgres_ffi::v15::bindings::XLOG_DBASE_CREATE_WAL_LOG { + debug!("XLOG_DBASE_CREATE_WAL_LOG: noop"); + } else if info == postgres_ffi::v15::bindings::XLOG_DBASE_CREATE_FILE_COPY { + // The XLOG record was renamed between v14 and v15, + // but the record format is the same. + // So we can reuse XlCreateDatabase here. + debug!("XLOG_DBASE_CREATE_FILE_COPY"); + let createdb = XlCreateDatabase::decode(&mut buf); + self.ingest_xlog_dbase_create(modification, &createdb, ctx) + .await?; + } else if info == postgres_ffi::v15::bindings::XLOG_DBASE_DROP { + let dropdb = XlDropDatabase::decode(&mut buf); + for tablespace_id in dropdb.tablespace_ids { + trace!("Drop db {}, {}", tablespace_id, dropdb.db_id); + modification + .drop_dbdir(tablespace_id, dropdb.db_id, ctx) + .await?; + } + } + } else if self.timeline.pg_version == 16 { + if info == postgres_ffi::v16::bindings::XLOG_DBASE_CREATE_WAL_LOG { + debug!("XLOG_DBASE_CREATE_WAL_LOG: noop"); + } else if info == postgres_ffi::v16::bindings::XLOG_DBASE_CREATE_FILE_COPY { + // The XLOG record was renamed between v14 and v15, + // but the record format is the same. + // So we can reuse XlCreateDatabase here. + debug!("XLOG_DBASE_CREATE_FILE_COPY"); + let createdb = XlCreateDatabase::decode(&mut buf); + self.ingest_xlog_dbase_create(modification, &createdb, ctx) + .await?; + } else if info == postgres_ffi::v16::bindings::XLOG_DBASE_DROP { + let dropdb = XlDropDatabase::decode(&mut buf); + for tablespace_id in dropdb.tablespace_ids { + trace!("Drop db {}, {}", tablespace_id, dropdb.db_id); + modification + .drop_dbdir(tablespace_id, dropdb.db_id, ctx) + .await?; + } + } + } + } + pg_constants::RM_TBLSPC_ID => { + trace!("XLOG_TBLSPC_CREATE/DROP is not handled yet"); + } + pg_constants::RM_CLOG_ID => { + let info = decoded.xl_info & !pg_constants::XLR_INFO_MASK; + + if info == pg_constants::CLOG_ZEROPAGE { + let pageno = buf.get_u32_le(); + let segno = pageno / pg_constants::SLRU_PAGES_PER_SEGMENT; + let rpageno = pageno % pg_constants::SLRU_PAGES_PER_SEGMENT; + self.put_slru_page_image( + modification, + SlruKind::Clog, + segno, + rpageno, + ZERO_PAGE.clone(), + ctx, + ) .await?; - } - } else if decoded.xl_rmid == pg_constants::RM_RELMAP_ID { - let xlrec = XlRelmapUpdate::decode(&mut buf); - self.ingest_relmap_page(modification, &xlrec, decoded, ctx) - .await?; - } else if decoded.xl_rmid == pg_constants::RM_XLOG_ID { - let info = decoded.xl_info & pg_constants::XLR_RMGR_INFO_MASK; - if info == pg_constants::XLOG_NEXTOID { - let next_oid = buf.get_u32_le(); - if self.checkpoint.nextOid != next_oid { - self.checkpoint.nextOid = next_oid; - self.checkpoint_modified = true; + } else { + assert!(info == pg_constants::CLOG_TRUNCATE); + let xlrec = XlClogTruncate::decode(&mut buf); + self.ingest_clog_truncate_record(modification, &xlrec, ctx) + .await?; } - } else if info == pg_constants::XLOG_CHECKPOINT_ONLINE - || info == pg_constants::XLOG_CHECKPOINT_SHUTDOWN - { - let mut checkpoint_bytes = [0u8; SIZEOF_CHECKPOINT]; - buf.copy_to_slice(&mut checkpoint_bytes); - let xlog_checkpoint = CheckPoint::decode(&checkpoint_bytes)?; - trace!( - "xlog_checkpoint.oldestXid={}, checkpoint.oldestXid={}", - xlog_checkpoint.oldestXid, - self.checkpoint.oldestXid - ); - if (self - .checkpoint - .oldestXid - .wrapping_sub(xlog_checkpoint.oldestXid) as i32) - < 0 + } + pg_constants::RM_XACT_ID => { + let info = decoded.xl_info & pg_constants::XLOG_XACT_OPMASK; + + if info == pg_constants::XLOG_XACT_COMMIT || info == pg_constants::XLOG_XACT_ABORT { + let parsed_xact = + XlXactParsedRecord::decode(&mut buf, decoded.xl_xid, decoded.xl_info); + self.ingest_xact_record( + modification, + &parsed_xact, + info == pg_constants::XLOG_XACT_COMMIT, + ctx, + ) + .await?; + } else if info == pg_constants::XLOG_XACT_COMMIT_PREPARED + || info == pg_constants::XLOG_XACT_ABORT_PREPARED { - self.checkpoint.oldestXid = xlog_checkpoint.oldestXid; - self.checkpoint_modified = true; - } - } - } else if decoded.xl_rmid == pg_constants::RM_LOGICALMSG_ID { - let info = decoded.xl_info & pg_constants::XLR_RMGR_INFO_MASK; - if info == pg_constants::XLOG_LOGICAL_MESSAGE { - let xlrec = XlLogicalMessage::decode(&mut buf); - let prefix = std::str::from_utf8(&buf[0..xlrec.prefix_size - 1])?; - let message = &buf[xlrec.prefix_size..xlrec.prefix_size + xlrec.message_size]; - if prefix == "neon-test" { - // This is a convenient way to make the WAL ingestion pause at - // particular point in the WAL. For more fine-grained control, - // we could peek into the message and only pause if it contains - // a particular string, for example, but this is enough for now. - crate::failpoint_support::sleep_millis_async!( - "wal-ingest-logical-message-sleep" + let parsed_xact = + XlXactParsedRecord::decode(&mut buf, decoded.xl_xid, decoded.xl_info); + self.ingest_xact_record( + modification, + &parsed_xact, + info == pg_constants::XLOG_XACT_COMMIT_PREPARED, + ctx, + ) + .await?; + // Remove twophase file. see RemoveTwoPhaseFile() in postgres code + trace!( + "Drop twophaseFile for xid {} parsed_xact.xid {} here at {}", + decoded.xl_xid, + parsed_xact.xid, + lsn, ); - } else if let Some(path) = prefix.strip_prefix("neon-file:") { - modification.put_file(path, message, ctx).await?; + modification + .drop_twophase_file(parsed_xact.xid, ctx) + .await?; + } else if info == pg_constants::XLOG_XACT_PREPARE { + modification + .put_twophase_file(decoded.xl_xid, Bytes::copy_from_slice(&buf[..]), ctx) + .await?; } } + pg_constants::RM_MULTIXACT_ID => { + let info = decoded.xl_info & pg_constants::XLR_RMGR_INFO_MASK; + + if info == pg_constants::XLOG_MULTIXACT_ZERO_OFF_PAGE { + let pageno = buf.get_u32_le(); + let segno = pageno / pg_constants::SLRU_PAGES_PER_SEGMENT; + let rpageno = pageno % pg_constants::SLRU_PAGES_PER_SEGMENT; + self.put_slru_page_image( + modification, + SlruKind::MultiXactOffsets, + segno, + rpageno, + ZERO_PAGE.clone(), + ctx, + ) + .await?; + } else if info == pg_constants::XLOG_MULTIXACT_ZERO_MEM_PAGE { + let pageno = buf.get_u32_le(); + let segno = pageno / pg_constants::SLRU_PAGES_PER_SEGMENT; + let rpageno = pageno % pg_constants::SLRU_PAGES_PER_SEGMENT; + self.put_slru_page_image( + modification, + SlruKind::MultiXactMembers, + segno, + rpageno, + ZERO_PAGE.clone(), + ctx, + ) + .await?; + } else if info == pg_constants::XLOG_MULTIXACT_CREATE_ID { + let xlrec = XlMultiXactCreate::decode(&mut buf); + self.ingest_multixact_create_record(modification, &xlrec)?; + } else if info == pg_constants::XLOG_MULTIXACT_TRUNCATE_ID { + let xlrec = XlMultiXactTruncate::decode(&mut buf); + self.ingest_multixact_truncate_record(modification, &xlrec, ctx) + .await?; + } + } + pg_constants::RM_RELMAP_ID => { + let xlrec = XlRelmapUpdate::decode(&mut buf); + self.ingest_relmap_page(modification, &xlrec, decoded, ctx) + .await?; + } + pg_constants::RM_XLOG_ID => { + let info = decoded.xl_info & pg_constants::XLR_RMGR_INFO_MASK; + + if info == pg_constants::XLOG_NEXTOID { + let next_oid = buf.get_u32_le(); + if self.checkpoint.nextOid != next_oid { + self.checkpoint.nextOid = next_oid; + self.checkpoint_modified = true; + } + } else if info == pg_constants::XLOG_CHECKPOINT_ONLINE + || info == pg_constants::XLOG_CHECKPOINT_SHUTDOWN + { + let mut checkpoint_bytes = [0u8; SIZEOF_CHECKPOINT]; + buf.copy_to_slice(&mut checkpoint_bytes); + let xlog_checkpoint = CheckPoint::decode(&checkpoint_bytes)?; + trace!( + "xlog_checkpoint.oldestXid={}, checkpoint.oldestXid={}", + xlog_checkpoint.oldestXid, + self.checkpoint.oldestXid + ); + if (self + .checkpoint + .oldestXid + .wrapping_sub(xlog_checkpoint.oldestXid) as i32) + < 0 + { + self.checkpoint.oldestXid = xlog_checkpoint.oldestXid; + self.checkpoint_modified = true; + } + } + } + pg_constants::RM_LOGICALMSG_ID => { + let info = decoded.xl_info & pg_constants::XLR_RMGR_INFO_MASK; + + if info == pg_constants::XLOG_LOGICAL_MESSAGE { + let xlrec = XlLogicalMessage::decode(&mut buf); + let prefix = std::str::from_utf8(&buf[0..xlrec.prefix_size - 1])?; + let message = &buf[xlrec.prefix_size..xlrec.prefix_size + xlrec.message_size]; + if prefix == "neon-test" { + // This is a convenient way to make the WAL ingestion pause at + // particular point in the WAL. For more fine-grained control, + // we could peek into the message and only pause if it contains + // a particular string, for example, but this is enough for now. + crate::failpoint_support::sleep_millis_async!( + "wal-ingest-logical-message-sleep" + ); + } else if let Some(path) = prefix.strip_prefix("neon-file:") { + modification.put_file(path, message, ctx).await?; + } + } + } + _x => { + // TODO: should probably log & fail here instead of blindly + // doing something without understanding the protocol + } } // Iterate through all the blocks that the record modifies, and From 57ae9cd07f97445ac55a3f42ac44b28f5b51e184 Mon Sep 17 00:00:00 2001 From: John Spray Date: Thu, 30 Nov 2023 14:22:07 +0000 Subject: [PATCH 23/38] pageserver: add `flush_ms` and document `/location_config` API (#5860) - During migration of tenants, it is useful for callers to `/location_conf` to flush a tenant's layers while transitioning to AttachedStale: this optimization reduces the redundant WAL replay work that the tenant's new attached pageserver will have to do. Test coverage for this will come as part of the larger tests for live migration in #5745 #5842 - Flushing is controlled with `flush_ms` query parameter: it is the caller's job to decide how long they want to wait for a flush to complete. If flush is not complete within the time limit, the pageserver proceeds to succeed anyway: flushing is only an optimization. - Add swagger definitions for all this: the location_config API is the primary interface for driving tenant migration as described in docs/rfcs/028-pageserver-migration.md, and will eventually replace the various /attach /detach /load /ignore APIs. --------- Co-authored-by: Joonas Koivunen --- control_plane/src/pageserver.rs | 26 +++--- control_plane/src/tenant_migration.rs | 10 +-- pageserver/src/http/openapi_spec.yml | 118 ++++++++++++++++++++++++++ pageserver/src/http/routes.rs | 4 +- pageserver/src/tenant.rs | 64 ++++++++++++++ pageserver/src/tenant/mgr.rs | 48 +++++++++-- 6 files changed, 246 insertions(+), 24 deletions(-) diff --git a/control_plane/src/pageserver.rs b/control_plane/src/pageserver.rs index 237df48543..0a55c90e8f 100644 --- a/control_plane/src/pageserver.rs +++ b/control_plane/src/pageserver.rs @@ -11,6 +11,7 @@ use std::io::{BufReader, Write}; use std::num::NonZeroU64; use std::path::PathBuf; use std::process::{Child, Command}; +use std::time::Duration; use std::{io, result}; use anyhow::{bail, Context}; @@ -522,19 +523,24 @@ impl PageServerNode { &self, tenant_id: TenantId, config: LocationConfig, + flush_ms: Option, ) -> anyhow::Result<()> { let req_body = TenantLocationConfigRequest { tenant_id, config }; - self.http_request( - Method::PUT, - format!( - "{}/tenant/{}/location_config", - self.http_base_url, tenant_id - ), - )? - .json(&req_body) - .send()? - .error_from_body()?; + let path = format!( + "{}/tenant/{}/location_config", + self.http_base_url, tenant_id + ); + let path = if let Some(flush_ms) = flush_ms { + format!("{}?flush_ms={}", path, flush_ms.as_millis()) + } else { + path + }; + + self.http_request(Method::PUT, path)? + .json(&req_body) + .send()? + .error_from_body()?; Ok(()) } diff --git a/control_plane/src/tenant_migration.rs b/control_plane/src/tenant_migration.rs index 42780db85c..c0c44e279f 100644 --- a/control_plane/src/tenant_migration.rs +++ b/control_plane/src/tenant_migration.rs @@ -117,7 +117,7 @@ pub fn migrate_tenant( println!("🔁 Already attached to {origin_ps_id}, freshening..."); let gen = attachment_service.attach_hook(tenant_id, dest_ps.conf.id)?; let dest_conf = build_location_config(LocationConfigMode::AttachedSingle, gen, None); - dest_ps.location_config(tenant_id, dest_conf)?; + dest_ps.location_config(tenant_id, dest_conf, None)?; println!("✅ Migration complete"); return Ok(()); } @@ -126,7 +126,7 @@ pub fn migrate_tenant( let stale_conf = build_location_config(LocationConfigMode::AttachedStale, Some(*generation), None); - origin_ps.location_config(tenant_id, stale_conf)?; + origin_ps.location_config(tenant_id, stale_conf, Some(Duration::from_secs(10)))?; baseline_lsns = Some(get_lsns(tenant_id, &origin_ps)?); } @@ -135,7 +135,7 @@ pub fn migrate_tenant( let dest_conf = build_location_config(LocationConfigMode::AttachedMulti, gen, None); println!("🔁 Attaching to pageserver {}", dest_ps.conf.id); - dest_ps.location_config(tenant_id, dest_conf)?; + dest_ps.location_config(tenant_id, dest_conf, None)?; if let Some(baseline) = baseline_lsns { println!("🕑 Waiting for LSN to catch up..."); @@ -181,7 +181,7 @@ pub fn migrate_tenant( "💤 Switching to secondary mode on pageserver {}", other_ps.conf.id ); - other_ps.location_config(tenant_id, secondary_conf)?; + other_ps.location_config(tenant_id, secondary_conf, None)?; } println!( @@ -189,7 +189,7 @@ pub fn migrate_tenant( dest_ps.conf.id ); let dest_conf = build_location_config(LocationConfigMode::AttachedSingle, gen, None); - dest_ps.location_config(tenant_id, dest_conf)?; + dest_ps.location_config(tenant_id, dest_conf, None)?; println!("✅ Migration complete"); diff --git a/pageserver/src/http/openapi_spec.yml b/pageserver/src/http/openapi_spec.yml index 4d455243f0..2e418f4d8f 100644 --- a/pageserver/src/http/openapi_spec.yml +++ b/pageserver/src/http/openapi_spec.yml @@ -624,6 +624,99 @@ paths: $ref: "#/components/schemas/ServiceUnavailableError" + /v1/tenant/{tenant_id}/location_config: + parameters: + - name: tenant_id + in: path + required: true + schema: + type: string + format: hex + - name: flush_ms + in: query + required: false + schema: + type: integer + put: + description: | + Configures a _tenant location_, that is how a particular pageserver handles + a particular tenant. This includes _attached_ tenants, i.e. those ingesting WAL + and page service requests, and _secondary_ tenants, i.e. those which are just keeping + a warm cache in anticipation of transitioning to attached state in the future. + + This is a declarative, idempotent API: there are not separate endpoints + for different tenant location configurations. Rather, this single endpoint accepts + a description of the desired location configuration, and makes whatever changes + are required to reach that state. + + In imperative terms, this API is used to attach and detach tenants, and + to transition tenants to and from secondary mode. + + This is a synchronous API: there is no 202 response. State transitions should always + be fast (milliseconds), with the exception of requests setting `flush_ms`, in which case + the caller controls the runtime of the request. + + In some state transitions, it makes sense to flush dirty data to remote storage: this includes transitions + to AttachedStale and Detached. Flushing is never necessary for correctness, but is an + important optimization when doing migrations. The `flush_ms` parameter controls whether + flushing should be attempted, and how much time is allowed for flushing. If the time limit expires, + the requested transition will continue without waiting for any outstanding data to flush. Callers + should use a duration which is substantially less than their HTTP client's request + timeout. It is safe to supply flush_ms irrespective of the request body: in state transitions + where flushing doesn't make sense, the server will ignore it. + + It is safe to retry requests, but if one receives a 409 or 503 response, it is not + useful to retry aggressively: there is probably an existing request still ongoing. + requestBody: + required: false + content: + application/json: + schema: + $ref: "#/components/schemas/TenantLocationConfigRequest" + responses: + "200": + description: Tenant is now in requested state + "503": + description: Tenant's state cannot be changed right now. Wait a few seconds and retry. + content: + application/json: + schema: + $ref: "#/components/schemas/Error" + "401": + description: Unauthorized Error + content: + application/json: + schema: + $ref: "#/components/schemas/UnauthorizedError" + "403": + description: Forbidden Error + content: + application/json: + schema: + $ref: "#/components/schemas/ForbiddenError" + "409": + description: | + The tenant is already known to Pageserver in some way, + and hence this `/attach` call has been rejected. + + Some examples of how this can happen: + - tenant was created on this pageserver + - tenant attachment was started by an earlier call to `/attach`. + + Callers should poll the tenant status's `attachment_status` field, + like for status 202. See the longer description for `POST /attach` + for details. + content: + application/json: + schema: + $ref: "#/components/schemas/ConflictError" + "500": + description: Generic operation error + content: + application/json: + schema: + $ref: "#/components/schemas/Error" + /v1/tenant/{tenant_id}/detach: parameters: - name: tenant_id @@ -1274,6 +1367,31 @@ components: tenant_id: type: string format: hex + TenantLocationConfigRequest: + type: object + required: + - tenant_id + properties: + tenant_id: + type: string + format: hex + mode: + type: string + enum: ["AttachedSingle", "AttachedMulti", "AttachedStale", "Secondary", "Detached"] + description: Mode of functionality that this pageserver will run in for this tenant. + generation: + type: integer + description: Attachment generation number, mandatory when `mode` is an attached state + secondary_conf: + $ref: '#/components/schemas/SecondaryConfig' + tenant_conf: + $ref: '#/components/schemas/TenantConfig' + SecondaryConfig: + type: object + properties: + warm: + type: boolean + description: Whether to poll remote storage for layers to download. If false, secondary locations don't download anything. TenantConfig: type: object properties: diff --git a/pageserver/src/http/routes.rs b/pageserver/src/http/routes.rs index 9cb411c95c..37159be95c 100644 --- a/pageserver/src/http/routes.rs +++ b/pageserver/src/http/routes.rs @@ -4,6 +4,7 @@ use std::collections::HashMap; use std::str::FromStr; use std::sync::Arc; +use std::time::Duration; use anyhow::{anyhow, Context, Result}; use enumset::EnumSet; @@ -1158,6 +1159,7 @@ async fn put_tenant_location_config_handler( let tenant_shard_id: TenantShardId = parse_request_param(&request, "tenant_shard_id")?; let request_data: TenantLocationConfigRequest = json_request(&mut request).await?; + let flush = parse_query_param(&request, "flush_ms")?.map(Duration::from_millis); check_permission(&request, Some(tenant_shard_id.tenant_id))?; let ctx = RequestContext::new(TaskKind::MgmtRequest, DownloadBehavior::Warn); @@ -1190,7 +1192,7 @@ async fn put_tenant_location_config_handler( state .tenant_manager - .upsert_location(tenant_shard_id, location_conf, &ctx) + .upsert_location(tenant_shard_id, location_conf, flush, &ctx) .await // TODO: badrequest assumes the caller was asking for something unreasonable, but in // principle we might have hit something like concurrent API calls to the same tenant, diff --git a/pageserver/src/tenant.rs b/pageserver/src/tenant.rs index 2a63f193e3..08066a612d 100644 --- a/pageserver/src/tenant.rs +++ b/pageserver/src/tenant.rs @@ -15,7 +15,9 @@ use anyhow::{bail, Context}; use bytes::Bytes; use camino::{Utf8Path, Utf8PathBuf}; use enumset::EnumSet; +use futures::stream::FuturesUnordered; use futures::FutureExt; +use futures::StreamExt; use pageserver_api::models::TimelineState; use pageserver_api::shard::TenantShardId; use remote_storage::DownloadError; @@ -32,6 +34,7 @@ use utils::completion; use utils::crashsafe::path_with_suffix_extension; use utils::fs_ext; use utils::sync::gate::Gate; +use utils::sync::gate::GateGuard; use self::config::AttachedLocationConfig; use self::config::AttachmentMode; @@ -51,6 +54,7 @@ use self::timeline::TimelineResources; use crate::config::PageServerConf; use crate::context::{DownloadBehavior, RequestContext}; use crate::deletion_queue::DeletionQueueClient; +use crate::deletion_queue::DeletionQueueError; use crate::import_datadir; use crate::is_uninit_mark; use crate::metrics::TENANT_ACTIVATION; @@ -3282,6 +3286,66 @@ impl Tenant { pub fn cached_synthetic_size(&self) -> u64 { self.cached_synthetic_tenant_size.load(Ordering::Relaxed) } + + /// Flush any in-progress layers, schedule uploads, and wait for uploads to complete. + /// + /// This function can take a long time: callers should wrap it in a timeout if calling + /// from an external API handler. + /// + /// Cancel-safety: cancelling this function may leave I/O running, but such I/O is + /// still bounded by tenant/timeline shutdown. + #[tracing::instrument(skip_all)] + pub(crate) async fn flush_remote(&self) -> anyhow::Result<()> { + let timelines = self.timelines.lock().unwrap().clone(); + + async fn flush_timeline(_gate: GateGuard, timeline: Arc) -> anyhow::Result<()> { + tracing::info!(timeline_id=%timeline.timeline_id, "Flushing..."); + timeline.freeze_and_flush().await?; + tracing::info!(timeline_id=%timeline.timeline_id, "Waiting for uploads..."); + if let Some(client) = &timeline.remote_client { + client.wait_completion().await?; + } + + Ok(()) + } + + // We do not use a JoinSet for these tasks, because we don't want them to be + // aborted when this function's future is cancelled: they should stay alive + // holding their GateGuard until they complete, to ensure their I/Os complete + // before Timeline shutdown completes. + let mut results = FuturesUnordered::new(); + + for (_timeline_id, timeline) in timelines { + // Run each timeline's flush in a task holding the timeline's gate: this + // means that if this function's future is cancelled, the Timeline shutdown + // will still wait for any I/O in here to complete. + let gate = match timeline.gate.enter() { + Ok(g) => g, + Err(_) => continue, + }; + let jh = tokio::task::spawn(async move { flush_timeline(gate, timeline).await }); + results.push(jh); + } + + while let Some(r) = results.next().await { + if let Err(e) = r { + if !e.is_cancelled() && !e.is_panic() { + tracing::error!("unexpected join error: {e:?}"); + } + } + } + + // The flushes we did above were just writes, but the Tenant might have had + // pending deletions as well from recent compaction/gc: we want to flush those + // as well. This requires flushing the global delete queue. This is cheap + // because it's typically a no-op. + match self.deletion_queue_client.flush_execute().await { + Ok(_) => {} + Err(DeletionQueueError::ShuttingDown) => {} + } + + Ok(()) + } } fn remove_timeline_and_uninit_mark( diff --git a/pageserver/src/tenant/mgr.rs b/pageserver/src/tenant/mgr.rs index e94d29327e..f34d62ba53 100644 --- a/pageserver/src/tenant/mgr.rs +++ b/pageserver/src/tenant/mgr.rs @@ -29,7 +29,9 @@ use crate::control_plane_client::{ use crate::deletion_queue::DeletionQueueClient; use crate::metrics::TENANT_MANAGER as METRICS; use crate::task_mgr::{self, TaskKind}; -use crate::tenant::config::{AttachmentMode, LocationConf, LocationMode, TenantConfOpt}; +use crate::tenant::config::{ + AttachedLocationConfig, AttachmentMode, LocationConf, LocationMode, TenantConfOpt, +}; use crate::tenant::delete::DeleteTenantFlow; use crate::tenant::span::debug_assert_current_span_has_tenant_id; use crate::tenant::{create_tenant_files, AttachedTenantConf, SpawnMode, Tenant, TenantState}; @@ -754,6 +756,7 @@ pub(crate) async fn create_tenant( ctx: &RequestContext, ) -> Result, TenantMapInsertError> { let location_conf = LocationConf::attached_single(tenant_conf, generation); + info!("Creating tenant at location {location_conf:?}"); let slot_guard = tenant_map_acquire_slot(&tenant_shard_id, TenantSlotAcquireMode::MustNotExist)?; @@ -861,6 +864,7 @@ impl TenantManager { &self, tenant_shard_id: TenantShardId, new_location_config: LocationConf, + flush: Option, ctx: &RequestContext, ) -> Result<(), anyhow::Error> { debug_assert_current_span_has_tenant_id(); @@ -869,7 +873,7 @@ impl TenantManager { // Special case fast-path for updates to Tenant: if our upsert is only updating configuration, // then we do not need to set the slot to InProgress, we can just call into the // existng tenant. - { + let modify_tenant = { let locked = self.tenants.read().unwrap(); let peek_slot = tenant_map_peek_slot(&locked, &tenant_shard_id, TenantSlotPeekMode::Write)?; @@ -880,22 +884,50 @@ impl TenantManager { // take our fast path and just provide the updated configuration // to the tenant. tenant.set_new_location_config(AttachedTenantConf::try_from( - new_location_config, + new_location_config.clone(), )?); - // Persist the new config in the background, to avoid holding up any - // locks while we do so. - // TODO - - return Ok(()); + Some(tenant.clone()) } else { // Different generations, fall through to general case + None } } _ => { // Not an Attached->Attached transition, fall through to general case + None } } + }; + + // Fast-path continued: having dropped out of the self.tenants lock, do the async + // phase of waiting for flush, before returning. + if let Some(tenant) = modify_tenant { + // Transition to AttachedStale means we may well hold a valid generation + // still, and have been requested to go stale as part of a migration. If + // the caller set `flush`, then flush to remote storage. + if let LocationMode::Attached(AttachedLocationConfig { + generation: _, + attach_mode: AttachmentMode::Stale, + }) = &new_location_config.mode + { + if let Some(flush_timeout) = flush { + match tokio::time::timeout(flush_timeout, tenant.flush_remote()).await { + Ok(Err(e)) => { + return Err(e); + } + Ok(Ok(_)) => return Ok(()), + Err(_) => { + tracing::warn!( + timeout_ms = flush_timeout.as_millis(), + "Timed out waiting for flush to remote storage, proceeding anyway." + ) + } + } + } + } + + return Ok(()); } // General case for upserts to TenantsMap, excluding the case above: we will substitute an From eba3bfc57e1a12e820321eafad1695e79d4c7ef5 Mon Sep 17 00:00:00 2001 From: Joonas Koivunen Date: Thu, 30 Nov 2023 17:48:40 +0200 Subject: [PATCH 24/38] test: python needs thread safety as well (#5992) we have test cases which launch processes from threads, and they capture output assuming this counter is thread-safe. at least according to my understanding this operation in python requires a lock to be thread-safe. --- test_runner/fixtures/utils.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/test_runner/fixtures/utils.py b/test_runner/fixtures/utils.py index 6e857766e5..1ec18b9f74 100644 --- a/test_runner/fixtures/utils.py +++ b/test_runner/fixtures/utils.py @@ -138,17 +138,19 @@ def subprocess_capture( _global_counter = 0 +_global_counter_lock = threading.Lock() def global_counter() -> int: - """A really dumb global counter. + """A really dumb but thread-safe global counter. This is useful for giving output files a unique number, so if we run the same command multiple times we can keep their output separate. """ - global _global_counter - _global_counter += 1 - return _global_counter + global _global_counter, _global_counter_lock + with _global_counter_lock: + _global_counter += 1 + return _global_counter def print_gc_result(row: Dict[str, Any]): From 3657a3c76e8b794997caf01baa80044626c82049 Mon Sep 17 00:00:00 2001 From: Anna Khanova <32508607+khanova@users.noreply.github.com> Date: Thu, 30 Nov 2023 17:33:54 +0100 Subject: [PATCH 25/38] Proxy fix metrics record (#5996) ## Problem Some latency metrics are recorded in inconsistent way. ## Summary of changes Make sure that everything is recorded in seconds. --- proxy/src/http.rs | 2 +- proxy/src/proxy.rs | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/proxy/src/http.rs b/proxy/src/http.rs index 638705d3e9..09423eca77 100644 --- a/proxy/src/http.rs +++ b/proxy/src/http.rs @@ -95,7 +95,7 @@ impl Endpoint { let res = self.client.execute(request).await; CONSOLE_REQUEST_LATENCY .with_label_values(&[&path]) - .observe(start.elapsed().as_micros() as f64); + .observe(start.elapsed().as_secs_f64()); res } } diff --git a/proxy/src/proxy.rs b/proxy/src/proxy.rs index 2af2dd5562..c4bea13f7f 100644 --- a/proxy/src/proxy.rs +++ b/proxy/src/proxy.rs @@ -117,7 +117,7 @@ pub static CONSOLE_REQUEST_LATENCY: Lazy = Lazy::new(|| { // proxy_wake_compute/proxy_get_role_info &["request"], // largest bucket = 2^16 * 0.2ms = 13s - exponential_buckets(0.2, 2.0, 16).unwrap(), + exponential_buckets(0.0002, 2.0, 16).unwrap(), ) .unwrap() }); @@ -136,7 +136,7 @@ pub static RATE_LIMITER_ACQUIRE_LATENCY: Lazy = Lazy::new(|| { register_histogram!( "semaphore_control_plane_token_acquire_seconds", "Time it took for proxy to establish a connection to the compute endpoint", - // largest bucket = 3^16 * 0.00005s = 3.28s + // largest bucket = 3^16 * 0.00005ms = 2.15s exponential_buckets(0.00005, 3.0, 16).unwrap(), ) .unwrap() From b451e75dc619bca87aa2e3b4a585aedd1526c6b5 Mon Sep 17 00:00:00 2001 From: Joonas Koivunen Date: Thu, 30 Nov 2023 19:31:03 +0200 Subject: [PATCH 26/38] test: include cmdline in captured output (#5977) aiming for faster to understand a bunch of `.stdout` and `.stderr` files, see example echo_1.stdout differences: ``` +# echo foobar abbacd + foobar abbacd ``` it can be disabled and is disabled in this PR for some tests; use `pg_bin.run_capture(..., with_command_header=False)` for that. as a bonus this cleans up the echoed newlines from s3_scrubber output which are also saved to file but echoed to test log. Co-authored-by: Alexander Bayandin --- test_runner/fixtures/neon_fixtures.py | 11 +++++-- test_runner/fixtures/utils.py | 17 +++++++++-- test_runner/regress/test_fullbackup.py | 11 ++++--- test_runner/regress/test_import.py | 39 ++++++++++++++++-------- test_runner/regress/test_wal_acceptor.py | 7 ++--- 5 files changed, 60 insertions(+), 25 deletions(-) diff --git a/test_runner/fixtures/neon_fixtures.py b/test_runner/fixtures/neon_fixtures.py index 862aab84dc..188851c39f 100644 --- a/test_runner/fixtures/neon_fixtures.py +++ b/test_runner/fixtures/neon_fixtures.py @@ -1858,7 +1858,8 @@ class PgBin: command: List[str], env: Optional[Env] = None, cwd: Optional[str] = None, - **kwargs: Any, + with_command_header=True, + **popen_kwargs: Any, ) -> str: """ Run one of the postgres binaries, with stderr and stdout redirected to a file. @@ -1871,7 +1872,13 @@ class PgBin: log.info(f"Running command '{' '.join(command)}'") env = self._build_env(env) base_path, _, _ = subprocess_capture( - self.log_dir, command, env=env, cwd=cwd, check=True, **kwargs + self.log_dir, + command, + env=env, + cwd=cwd, + check=True, + with_command_header=with_command_header, + **popen_kwargs, ) return base_path diff --git a/test_runner/fixtures/utils.py b/test_runner/fixtures/utils.py index 1ec18b9f74..cda788b2a4 100644 --- a/test_runner/fixtures/utils.py +++ b/test_runner/fixtures/utils.py @@ -49,7 +49,8 @@ def subprocess_capture( echo_stdout=False, capture_stdout=False, timeout=None, - **kwargs: Any, + with_command_header=True, + **popen_kwargs: Any, ) -> Tuple[str, Optional[str], int]: """Run a process and bifurcate its output to files and the `log` logger @@ -86,13 +87,23 @@ def subprocess_capture( self.captured = "" def run(self): + first = with_command_header for line in self.in_file: + if first: + # do this only after receiving any input so that we can + # keep deleting empty files, or leave it out completly if + # it was unwanted (using the file as input later for example) + first = False + # prefix the files with the command line so that we can + # later understand which file is for what command + self.out_file.write((f"# {' '.join(cmd)}\n\n").encode("utf-8")) + # Only bother decoding if we are going to do something more than stream to a file if self.echo or self.capture: string = line.decode(encoding="utf-8", errors="replace") if self.echo: - log.info(string) + log.info(string.strip()) if self.capture: self.captured += string @@ -107,7 +118,7 @@ def subprocess_capture( p = subprocess.Popen( cmd, - **kwargs, + **popen_kwargs, stdout=subprocess.PIPE, stderr=subprocess.PIPE, ) diff --git a/test_runner/regress/test_fullbackup.py b/test_runner/regress/test_fullbackup.py index 214f1f33a8..a456c06862 100644 --- a/test_runner/regress/test_fullbackup.py +++ b/test_runner/regress/test_fullbackup.py @@ -20,6 +20,7 @@ def test_fullbackup( pg_bin: PgBin, port_distributor: PortDistributor, pg_distrib_dir: Path, + test_output_dir: Path, ): env = neon_env_builder.init_start() @@ -49,10 +50,12 @@ def test_fullbackup( restored_dir_path = env.repo_dir / "restored_datadir" os.mkdir(restored_dir_path, 0o750) query = f"fullbackup {env.initial_tenant} {timeline} {lsn}" - cmd = ["psql", "--no-psqlrc", env.pageserver.connstr(), "-c", query] - result_basepath = pg_bin.run_capture(cmd, env=psql_env) - tar_output_file = result_basepath + ".stdout" - subprocess_capture(env.repo_dir, ["tar", "-xf", tar_output_file, "-C", str(restored_dir_path)]) + tar_output_file = test_output_dir / "fullbackup.tar" + cmd = ["psql", "--no-psqlrc", env.pageserver.connstr(), "-c", query, "-o", str(tar_output_file)] + pg_bin.run_capture(cmd, env=psql_env) + subprocess_capture( + env.repo_dir, ["tar", "-xf", str(tar_output_file), "-C", str(restored_dir_path)] + ) # HACK # fullbackup returns neon specific pg_control and first WAL segment diff --git a/test_runner/regress/test_import.py b/test_runner/regress/test_import.py index d357bd0ee4..8da5f1eec2 100644 --- a/test_runner/regress/test_import.py +++ b/test_runner/regress/test_import.py @@ -163,7 +163,9 @@ def test_import_from_vanilla(test_output_dir, pg_bin, vanilla_pg, neon_env_build assert endpoint.safe_psql("select count(*) from t") == [(300000,)] -def test_import_from_pageserver_small(pg_bin: PgBin, neon_env_builder: NeonEnvBuilder): +def test_import_from_pageserver_small( + pg_bin: PgBin, neon_env_builder: NeonEnvBuilder, test_output_dir: Path +): neon_env_builder.enable_pageserver_remote_storage(RemoteStorageKind.LOCAL_FS) env = neon_env_builder.init_start() @@ -177,7 +179,7 @@ def test_import_from_pageserver_small(pg_bin: PgBin, neon_env_builder: NeonEnvBu num_rows = 3000 lsn = _generate_data(num_rows, endpoint) - _import(num_rows, lsn, env, pg_bin, timeline, env.pg_distrib_dir) + _import(num_rows, lsn, env, pg_bin, timeline, env.pg_distrib_dir, test_output_dir) @pytest.mark.timeout(1800) @@ -185,7 +187,9 @@ def test_import_from_pageserver_small(pg_bin: PgBin, neon_env_builder: NeonEnvBu # the test back after finding the failure cause. # @pytest.mark.skipif(os.environ.get('BUILD_TYPE') == "debug", reason="only run with release build") @pytest.mark.skip("See https://github.com/neondatabase/neon/issues/2255") -def test_import_from_pageserver_multisegment(pg_bin: PgBin, neon_env_builder: NeonEnvBuilder): +def test_import_from_pageserver_multisegment( + pg_bin: PgBin, neon_env_builder: NeonEnvBuilder, test_output_dir: Path +): neon_env_builder.enable_pageserver_remote_storage(RemoteStorageKind.LOCAL_FS) env = neon_env_builder.init_start() @@ -205,7 +209,9 @@ def test_import_from_pageserver_multisegment(pg_bin: PgBin, neon_env_builder: Ne log.info(f"timeline logical size = {logical_size / (1024 ** 2)}MB") assert logical_size > 1024**3 # = 1GB - tar_output_file = _import(num_rows, lsn, env, pg_bin, timeline, env.pg_distrib_dir) + tar_output_file = _import( + num_rows, lsn, env, pg_bin, timeline, env.pg_distrib_dir, test_output_dir + ) # Check if the backup data contains multiple segment files cnt_seg_files = 0 @@ -246,7 +252,8 @@ def _import( pg_bin: PgBin, timeline: TimelineId, pg_distrib_dir: Path, -) -> str: + test_output_dir: Path, +) -> Path: """Test importing backup data to the pageserver. Args: @@ -263,9 +270,9 @@ def _import( # Get a fullbackup from pageserver query = f"fullbackup { env.initial_tenant} {timeline} {lsn}" - cmd = ["psql", "--no-psqlrc", env.pageserver.connstr(), "-c", query] - result_basepath = pg_bin.run_capture(cmd, env=psql_env) - tar_output_file = result_basepath + ".stdout" + tar_output_file = test_output_dir / "fullbackup.tar" + cmd = ["psql", "--no-psqlrc", env.pageserver.connstr(), "-c", query, "-o", str(tar_output_file)] + pg_bin.run_capture(cmd, env=psql_env) # Stop the first pageserver instance, erase all its data env.endpoints.stop_all() @@ -299,7 +306,7 @@ def _import( "--base-lsn", str(lsn), "--base-tarfile", - os.path.join(tar_output_file), + str(tar_output_file), "--pg-version", env.pg_version, ] @@ -315,9 +322,17 @@ def _import( # Take another fullbackup query = f"fullbackup { tenant} {timeline} {lsn}" - cmd = ["psql", "--no-psqlrc", env.pageserver.connstr(), "-c", query] - result_basepath = pg_bin.run_capture(cmd, env=psql_env) - new_tar_output_file = result_basepath + ".stdout" + new_tar_output_file = test_output_dir / "fullbackup-new.tar" + cmd = [ + "psql", + "--no-psqlrc", + env.pageserver.connstr(), + "-c", + query, + "-o", + str(new_tar_output_file), + ] + pg_bin.run_capture(cmd, env=psql_env) # Check it's the same as the first fullbackup # TODO pageserver should be checking checksum diff --git a/test_runner/regress/test_wal_acceptor.py b/test_runner/regress/test_wal_acceptor.py index 05c60eb102..b7eaaf39bc 100644 --- a/test_runner/regress/test_wal_acceptor.py +++ b/test_runner/regress/test_wal_acceptor.py @@ -1,6 +1,5 @@ import filecmp import os -import pathlib import random import shutil import signal @@ -639,7 +638,7 @@ class ProposerPostgres(PgProtocol): def __init__( self, pgdata_dir: str, - pg_bin, + pg_bin: PgBin, tenant_id: TenantId, timeline_id: TimelineId, listen_addr: str, @@ -665,7 +664,7 @@ class ProposerPostgres(PgProtocol): def create_dir_config(self, safekeepers: str): """Create dir and config for running --sync-safekeepers""" - pathlib.Path(self.pg_data_dir_path()).mkdir(exist_ok=True) + Path(self.pg_data_dir_path()).mkdir(exist_ok=True) with open(self.config_file_path(), "w") as f: cfg = [ "synchronous_standby_names = 'walproposer'\n", @@ -691,7 +690,7 @@ class ProposerPostgres(PgProtocol): "PGDATA": self.pg_data_dir_path(), } - basepath = self.pg_bin.run_capture(command, env) + basepath = self.pg_bin.run_capture(command, env, with_command_header=False) log.info(f"postgres --sync-safekeepers output: {basepath}") From f39fca0049c36cff5ed2c4b890b08f86fa56c15b Mon Sep 17 00:00:00 2001 From: Conrad Ludgate Date: Thu, 30 Nov 2023 20:52:30 +0000 Subject: [PATCH 27/38] proxy: chore: replace strings with SmolStr (#5786) ## Problem no problem ## Summary of changes replaces boxstr with arcstr as it's cheaper to clone. mild perf improvement. probably should look into other smallstring optimsations tbh, they will likely be even better. The longest endpoint name I was able to construct is something like `ep-weathered-wildflower-12345678` which is 32 bytes. Most string optimisations top out at 23 bytes --- Cargo.lock | 10 ++++++++++ Cargo.toml | 1 + proxy/Cargo.toml | 1 + proxy/src/auth/backend/link.rs | 2 +- proxy/src/bin/pg_sni_router.rs | 2 +- proxy/src/console/messages.rs | 9 +++++---- proxy/src/console/provider.rs | 2 +- proxy/src/console/provider/neon.rs | 2 +- proxy/src/proxy.rs | 8 ++++---- proxy/src/serverless/conn_pool.rs | 25 +++++++++++++------------ proxy/src/serverless/sql_over_http.rs | 10 +++++----- proxy/src/usage_metrics.rs | 9 +++++---- 12 files changed, 48 insertions(+), 33 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 6546590f6c..5639665758 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3643,6 +3643,7 @@ dependencies = [ "serde", "serde_json", "sha2", + "smol_str", "socket2 0.5.3", "sync_wrapper", "task-local-extensions", @@ -4709,6 +4710,15 @@ version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "62bb4feee49fdd9f707ef802e22365a35de4b7b299de4763d44bfea899442ff9" +[[package]] +name = "smol_str" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "74212e6bbe9a4352329b2f68ba3130c15a3f26fe88ff22dbdc6cdd58fa85e99c" +dependencies = [ + "serde", +] + [[package]] name = "socket2" version = "0.4.9" diff --git a/Cargo.toml b/Cargo.toml index cbcb25359d..ba8b49c0e0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -132,6 +132,7 @@ serde_assert = "0.5.0" sha2 = "0.10.2" signal-hook = "0.3" smallvec = "1.11" +smol_str = { version = "0.2.0", features = ["serde"] } socket2 = "0.5" strum = "0.24" strum_macros = "0.24" diff --git a/proxy/Cargo.toml b/proxy/Cargo.toml index 0822718bae..48c8604d86 100644 --- a/proxy/Cargo.toml +++ b/proxy/Cargo.toml @@ -69,6 +69,7 @@ webpki-roots.workspace = true x509-parser.workspace = true native-tls.workspace = true postgres-native-tls.workspace = true +smol_str.workspace = true workspace_hack.workspace = true tokio-util.workspace = true diff --git a/proxy/src/auth/backend/link.rs b/proxy/src/auth/backend/link.rs index da43cf11c4..3a77d7e5ca 100644 --- a/proxy/src/auth/backend/link.rs +++ b/proxy/src/auth/backend/link.rs @@ -106,7 +106,7 @@ pub(super) async fn authenticate( reported_auth_ok: true, value: NodeInfo { config, - aux: db_info.aux.into(), + aux: db_info.aux, allow_self_signed_compute: false, // caller may override }, }) diff --git a/proxy/src/bin/pg_sni_router.rs b/proxy/src/bin/pg_sni_router.rs index 2b859fc2db..bedbdbcc83 100644 --- a/proxy/src/bin/pg_sni_router.rs +++ b/proxy/src/bin/pg_sni_router.rs @@ -284,5 +284,5 @@ async fn handle_client( let client = tokio::net::TcpStream::connect(destination).await?; let metrics_aux: MetricsAuxInfo = Default::default(); - proxy::proxy::proxy_pass(tls_stream, client, &metrics_aux).await + proxy::proxy::proxy_pass(tls_stream, client, metrics_aux).await } diff --git a/proxy/src/console/messages.rs b/proxy/src/console/messages.rs index e5f1615b14..837379b21f 100644 --- a/proxy/src/console/messages.rs +++ b/proxy/src/console/messages.rs @@ -1,4 +1,5 @@ use serde::Deserialize; +use smol_str::SmolStr; use std::fmt; /// Generic error response with human-readable description. @@ -88,11 +89,11 @@ impl fmt::Debug for DatabaseInfo { /// Various labels for prometheus metrics. /// Also known as `ProxyMetricsAuxInfo` in the console. -#[derive(Debug, Deserialize, Default)] +#[derive(Debug, Deserialize, Clone, Default)] pub struct MetricsAuxInfo { - pub endpoint_id: Box, - pub project_id: Box, - pub branch_id: Box, + pub endpoint_id: SmolStr, + pub project_id: SmolStr, + pub branch_id: SmolStr, } impl MetricsAuxInfo { diff --git a/proxy/src/console/provider.rs b/proxy/src/console/provider.rs index a525de8e53..e735b9f66c 100644 --- a/proxy/src/console/provider.rs +++ b/proxy/src/console/provider.rs @@ -229,7 +229,7 @@ pub struct NodeInfo { pub config: compute::ConnCfg, /// Labels for proxy's metrics. - pub aux: Arc, + pub aux: MetricsAuxInfo, /// Whether we should accept self-signed certificates (for testing) pub allow_self_signed_compute: bool, diff --git a/proxy/src/console/provider/neon.rs b/proxy/src/console/provider/neon.rs index 117d0ec190..7828a7d7e4 100644 --- a/proxy/src/console/provider/neon.rs +++ b/proxy/src/console/provider/neon.rs @@ -144,7 +144,7 @@ impl Api { let node = NodeInfo { config, - aux: body.aux.into(), + aux: body.aux, allow_self_signed_compute: false, }; diff --git a/proxy/src/proxy.rs b/proxy/src/proxy.rs index c4bea13f7f..36d01f9acc 100644 --- a/proxy/src/proxy.rs +++ b/proxy/src/proxy.rs @@ -877,11 +877,11 @@ async fn prepare_client_connection( pub async fn proxy_pass( client: impl AsyncRead + AsyncWrite + Unpin, compute: impl AsyncRead + AsyncWrite + Unpin, - aux: &MetricsAuxInfo, + aux: MetricsAuxInfo, ) -> anyhow::Result<()> { let usage = USAGE_METRICS.register(Ids { - endpoint_id: aux.endpoint_id.to_string(), - branch_id: aux.branch_id.to_string(), + endpoint_id: aux.endpoint_id.clone(), + branch_id: aux.branch_id.clone(), }); let m_sent = NUM_BYTES_PROXIED_COUNTER.with_label_values(&["tx"]); @@ -1032,7 +1032,7 @@ impl Client<'_, S> { // immediately after opening the connection. let (stream, read_buf) = stream.into_inner(); node.stream.write_all(&read_buf).await?; - proxy_pass(stream, node.stream, &aux).await + proxy_pass(stream, node.stream, aux).await } } diff --git a/proxy/src/serverless/conn_pool.rs b/proxy/src/serverless/conn_pool.rs index 2072cadc3a..ca7a9ad0a0 100644 --- a/proxy/src/serverless/conn_pool.rs +++ b/proxy/src/serverless/conn_pool.rs @@ -8,6 +8,7 @@ use pbkdf2::{ Params, Pbkdf2, }; use pq_proto::StartupMessageParams; +use smol_str::SmolStr; use std::{collections::HashMap, net::SocketAddr, sync::Arc}; use std::{ fmt, @@ -41,16 +42,16 @@ const MAX_CONNS_PER_ENDPOINT: usize = 20; #[derive(Debug, Clone)] pub struct ConnInfo { - pub username: String, - pub dbname: String, - pub hostname: String, - pub password: String, - pub options: Option, + pub username: SmolStr, + pub dbname: SmolStr, + pub hostname: SmolStr, + pub password: SmolStr, + pub options: Option, } impl ConnInfo { // hm, change to hasher to avoid cloning? - pub fn db_and_user(&self) -> (String, String) { + pub fn db_and_user(&self) -> (SmolStr, SmolStr) { (self.dbname.clone(), self.username.clone()) } } @@ -70,7 +71,7 @@ struct ConnPoolEntry { // Per-endpoint connection pool, (dbname, username) -> DbUserConnPool // Number of open connections is limited by the `max_conns_per_endpoint`. pub struct EndpointConnPool { - pools: HashMap<(String, String), DbUserConnPool>, + pools: HashMap<(SmolStr, SmolStr), DbUserConnPool>, total_conns: usize, } @@ -95,7 +96,7 @@ pub struct GlobalConnPool { // // That should be a fairly conteded map, so return reference to the per-endpoint // pool as early as possible and release the lock. - global_pool: DashMap>>, + global_pool: DashMap>>, /// [`DashMap::len`] iterates over all inner pools and acquires a read lock on each. /// That seems like far too much effort, so we're using a relaxed increment counter instead. @@ -327,7 +328,7 @@ impl GlobalConnPool { Ok(()) } - fn get_or_create_endpoint_pool(&self, endpoint: &String) -> Arc> { + fn get_or_create_endpoint_pool(&self, endpoint: &SmolStr) -> Arc> { // fast path if let Some(pool) = self.global_pool.get(endpoint) { return pool.clone(); @@ -468,7 +469,7 @@ async fn connect_to_compute_once( let (client, mut connection) = config .user(&conn_info.username) - .password(&conn_info.password) + .password(&*conn_info.password) .dbname(&conn_info.dbname) .connect_timeout(timeout) .connect(tokio_postgres::NoTls) @@ -482,8 +483,8 @@ async fn connect_to_compute_once( info!(%conn_info, %session, "new connection"); }); let ids = Ids { - endpoint_id: node_info.aux.endpoint_id.to_string(), - branch_id: node_info.aux.branch_id.to_string(), + endpoint_id: node_info.aux.endpoint_id.clone(), + branch_id: node_info.aux.branch_id.clone(), }; tokio::spawn( diff --git a/proxy/src/serverless/sql_over_http.rs b/proxy/src/serverless/sql_over_http.rs index 25b96668de..6c337a837c 100644 --- a/proxy/src/serverless/sql_over_http.rs +++ b/proxy/src/serverless/sql_over_http.rs @@ -182,16 +182,16 @@ fn get_conn_info( for (key, value) in pairs { if key == "options" { - options = Some(value.to_string()); + options = Some(value.into()); break; } } Ok(ConnInfo { - username: username.to_owned(), - dbname: dbname.to_owned(), - hostname: hostname.to_owned(), - password: password.to_owned(), + username: username.into(), + dbname: dbname.into(), + hostname: hostname.into(), + password: password.into(), options, }) } diff --git a/proxy/src/usage_metrics.rs b/proxy/src/usage_metrics.rs index 180b5f7199..789a4c680c 100644 --- a/proxy/src/usage_metrics.rs +++ b/proxy/src/usage_metrics.rs @@ -6,6 +6,7 @@ use consumption_metrics::{idempotency_key, Event, EventChunk, EventType, CHUNK_S use dashmap::{mapref::entry::Entry, DashMap}; use once_cell::sync::Lazy; use serde::{Deserialize, Serialize}; +use smol_str::SmolStr; use std::{ convert::Infallible, sync::{ @@ -29,8 +30,8 @@ const DEFAULT_HTTP_REPORTING_TIMEOUT: Duration = Duration::from_secs(60); /// because we enrich the event with project_id in the control-plane endpoint. #[derive(Eq, Hash, PartialEq, Serialize, Deserialize, Debug, Clone)] pub struct Ids { - pub endpoint_id: String, - pub branch_id: String, + pub endpoint_id: SmolStr, + pub branch_id: SmolStr, } #[derive(Debug)] @@ -290,8 +291,8 @@ mod tests { // register a new counter let counter = metrics.register(Ids { - endpoint_id: "e1".to_string(), - branch_id: "b1".to_string(), + endpoint_id: "e1".into(), + branch_id: "b1".into(), }); // the counter should be observed despite 0 egress From 384277354607a98312f61c5e23e053ac8cabbfd1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arpad=20M=C3=BCller?= Date: Thu, 30 Nov 2023 22:01:25 +0100 Subject: [PATCH 28/38] Correct RFC number for Pageserver WAL DR RFC (#5997) When I opened #5248, 27 was an unused RFC number. Since then, two RFCs have been merged, so now 27 is taken. 29 is free though, so move it there. --- ...saster-recovery.md => 029-pageserver-wal-disaster-recovery.md} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename docs/rfcs/{027-pageserver-wal-disaster-recovery.md => 029-pageserver-wal-disaster-recovery.md} (100%) diff --git a/docs/rfcs/027-pageserver-wal-disaster-recovery.md b/docs/rfcs/029-pageserver-wal-disaster-recovery.md similarity index 100% rename from docs/rfcs/027-pageserver-wal-disaster-recovery.md rename to docs/rfcs/029-pageserver-wal-disaster-recovery.md From b71b8ecfc2f557773cc6d1221ba5255a03a25e8a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arpad=20M=C3=BCller?= Date: Thu, 30 Nov 2023 22:32:04 +0100 Subject: [PATCH 29/38] Add existing_initdb_timeline_id param to timeline creation (#5912) This PR adds an `existing_initdb_timeline_id` option to timeline creation APIs, taking an optional timeline ID. Follow-up of #5390. If the `existing_initdb_timeline_id` option is specified via the HTTP API, the pageserver downloads the existing initdb archive from the given timeline ID and extracts it, instead of running initdb itself. --------- Co-authored-by: Christian Schwarz --- control_plane/src/bin/neon_local.rs | 3 + control_plane/src/pageserver.rs | 2 + libs/pageserver_api/src/models.rs | 2 + pageserver/src/http/openapi_spec.yml | 3 + pageserver/src/http/routes.rs | 1 + pageserver/src/import_datadir.rs | 13 ++- pageserver/src/tenant.rs | 79 +++++++++++++------ .../src/tenant/remote_timeline_client.rs | 13 ++- .../tenant/remote_timeline_client/download.rs | 75 ++++++++++++++++-- pageserver/src/walingest.rs | 2 +- test_runner/fixtures/pageserver/http.py | 4 + test_runner/fixtures/pageserver/utils.py | 25 ++++-- test_runner/regress/test_remote_storage.py | 7 +- test_runner/regress/test_tenant_delete.py | 2 + test_runner/regress/test_tenants.py | 9 ++- test_runner/regress/test_timeline_delete.py | 8 +- test_runner/regress/test_wal_restore.py | 41 ++++++++++ 17 files changed, 245 insertions(+), 44 deletions(-) diff --git a/control_plane/src/bin/neon_local.rs b/control_plane/src/bin/neon_local.rs index f7442c02c7..8d53a6a658 100644 --- a/control_plane/src/bin/neon_local.rs +++ b/control_plane/src/bin/neon_local.rs @@ -415,6 +415,7 @@ fn handle_tenant(tenant_match: &ArgMatches, env: &mut local_env::LocalEnv) -> an None, None, Some(pg_version), + None, )?; let new_timeline_id = timeline_info.timeline_id; let last_record_lsn = timeline_info.last_record_lsn; @@ -495,6 +496,7 @@ fn handle_timeline(timeline_match: &ArgMatches, env: &mut local_env::LocalEnv) - None, None, Some(pg_version), + None, )?; let new_timeline_id = timeline_info.timeline_id; @@ -582,6 +584,7 @@ fn handle_timeline(timeline_match: &ArgMatches, env: &mut local_env::LocalEnv) - start_lsn, Some(ancestor_timeline_id), None, + None, )?; let new_timeline_id = timeline_info.timeline_id; diff --git a/control_plane/src/pageserver.rs b/control_plane/src/pageserver.rs index 0a55c90e8f..96a41874fd 100644 --- a/control_plane/src/pageserver.rs +++ b/control_plane/src/pageserver.rs @@ -565,6 +565,7 @@ impl PageServerNode { ancestor_start_lsn: Option, ancestor_timeline_id: Option, pg_version: Option, + existing_initdb_timeline_id: Option, ) -> anyhow::Result { // If timeline ID was not specified, generate one let new_timeline_id = new_timeline_id.unwrap_or(TimelineId::generate()); @@ -578,6 +579,7 @@ impl PageServerNode { ancestor_start_lsn, ancestor_timeline_id, pg_version, + existing_initdb_timeline_id, }) .send()? .error_from_body()? diff --git a/libs/pageserver_api/src/models.rs b/libs/pageserver_api/src/models.rs index 60c508037e..e6236f91e3 100644 --- a/libs/pageserver_api/src/models.rs +++ b/libs/pageserver_api/src/models.rs @@ -179,6 +179,8 @@ pub struct TimelineCreateRequest { #[serde(default)] pub ancestor_timeline_id: Option, #[serde(default)] + pub existing_initdb_timeline_id: Option, + #[serde(default)] pub ancestor_start_lsn: Option, pub pg_version: Option, } diff --git a/pageserver/src/http/openapi_spec.yml b/pageserver/src/http/openapi_spec.yml index 2e418f4d8f..237109abfe 100644 --- a/pageserver/src/http/openapi_spec.yml +++ b/pageserver/src/http/openapi_spec.yml @@ -1028,6 +1028,9 @@ paths: format: hex pg_version: type: integer + existing_initdb_timeline_id: + type: string + format: hex responses: "201": description: TimelineInfo diff --git a/pageserver/src/http/routes.rs b/pageserver/src/http/routes.rs index 37159be95c..7ab56ccab2 100644 --- a/pageserver/src/http/routes.rs +++ b/pageserver/src/http/routes.rs @@ -441,6 +441,7 @@ async fn timeline_create_handler( request_data.ancestor_timeline_id.map(TimelineId::from), request_data.ancestor_start_lsn, request_data.pg_version.unwrap_or(crate::DEFAULT_PG_VERSION), + request_data.existing_initdb_timeline_id, state.broker_client.clone(), &ctx, ) diff --git a/pageserver/src/import_datadir.rs b/pageserver/src/import_datadir.rs index 770458e02e..452cd73f76 100644 --- a/pageserver/src/import_datadir.rs +++ b/pageserver/src/import_datadir.rs @@ -7,12 +7,13 @@ use std::pin::Pin; use std::task::{self, Poll}; use anyhow::{bail, ensure, Context, Result}; +use async_compression::tokio::bufread::ZstdDecoder; use async_compression::{tokio::write::ZstdEncoder, zstd::CParameter, Level}; use bytes::Bytes; use camino::Utf8Path; use futures::StreamExt; use nix::NixPath; -use tokio::io::{AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt}; +use tokio::io::{AsyncBufRead, AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt}; use tokio_tar::Archive; use tokio_tar::Builder; use tokio_tar::HeaderMode; @@ -732,3 +733,13 @@ pub async fn create_tar_zst(pgdata_path: &Utf8Path) -> Result> { } Ok(compressed.buf) } + +pub async fn extract_tar_zst( + pgdata_path: &Utf8Path, + tar_zst: impl AsyncBufRead + Unpin, +) -> Result<()> { + let tar = Box::pin(ZstdDecoder::new(tar_zst)); + let mut archive = Archive::new(tar); + archive.unpack(pgdata_path).await?; + Ok(()) +} diff --git a/pageserver/src/tenant.rs b/pageserver/src/tenant.rs index 08066a612d..bf063664b4 100644 --- a/pageserver/src/tenant.rs +++ b/pageserver/src/tenant.rs @@ -24,6 +24,7 @@ use remote_storage::DownloadError; use remote_storage::GenericRemoteStorage; use std::fmt; use storage_broker::BrokerClientChannel; +use tokio::io::BufReader; use tokio::runtime::Handle; use tokio::sync::watch; use tokio::task::JoinSet; @@ -1558,12 +1559,14 @@ impl Tenant { /// /// If the caller specified the timeline ID to use (`new_timeline_id`), and timeline with /// the same timeline ID already exists, returns CreateTimelineError::AlreadyExists. + #[allow(clippy::too_many_arguments)] pub async fn create_timeline( &self, new_timeline_id: TimelineId, ancestor_timeline_id: Option, mut ancestor_start_lsn: Option, pg_version: u32, + load_existing_initdb: Option, broker_client: storage_broker::BrokerClientChannel, ctx: &RequestContext, ) -> Result, CreateTimelineError> { @@ -1638,7 +1641,7 @@ impl Tenant { .await? } None => { - self.bootstrap_timeline(new_timeline_id, pg_version, ctx) + self.bootstrap_timeline(new_timeline_id, pg_version, load_existing_initdb, ctx) .await? } }; @@ -2951,6 +2954,7 @@ impl Tenant { &self, timeline_id: TimelineId, pg_version: u32, + load_existing_initdb: Option, ctx: &RequestContext, ) -> anyhow::Result> { let timeline_uninit_mark = { @@ -2973,8 +2977,6 @@ impl Tenant { format!("Failed to remove already existing initdb directory: {pgdata_path}") })?; } - // Init temporarily repo to get bootstrap data, this creates a directory in the `initdb_path` path - run_initdb(self.conf, &pgdata_path, pg_version, &self.cancel).await?; // this new directory is very temporary, set to remove it immediately after bootstrap, we don't need it scopeguard::defer! { if let Err(e) = fs::remove_dir_all(&pgdata_path) { @@ -2982,31 +2984,58 @@ impl Tenant { error!("Failed to remove temporary initdb directory '{pgdata_path}': {e}"); } } - let pgdata_lsn = import_datadir::get_lsn_from_controlfile(&pgdata_path)?.align(); + if let Some(existing_initdb_timeline_id) = load_existing_initdb { + let Some(storage) = &self.remote_storage else { + bail!("no storage configured but load_existing_initdb set to {existing_initdb_timeline_id}"); + }; + let (initdb_tar_zst_path, initdb_tar_zst) = + self::remote_timeline_client::download_initdb_tar_zst( + self.conf, + storage, + &self.tenant_shard_id, + &existing_initdb_timeline_id, + ) + .await + .context("download initdb tar")?; + let buf_read = Box::pin(BufReader::new(initdb_tar_zst)); + import_datadir::extract_tar_zst(&pgdata_path, buf_read) + .await + .context("extract initdb tar")?; - // Upload the created data dir to S3 - if let Some(storage) = &self.remote_storage { - let pgdata_zstd = import_datadir::create_tar_zst(&pgdata_path).await?; - let pgdata_zstd = Bytes::from(pgdata_zstd); - backoff::retry( - || async { - self::remote_timeline_client::upload_initdb_dir( - storage, - &self.tenant_shard_id.tenant_id, - &timeline_id, - pgdata_zstd.clone(), - ) + if initdb_tar_zst_path.exists() { + tokio::fs::remove_file(&initdb_tar_zst_path) .await - }, - |_| false, - 3, - u32::MAX, - "persist_initdb_tar_zst", - // TODO: use a cancellation token (https://github.com/neondatabase/neon/issues/5066) - backoff::Cancel::new(CancellationToken::new(), || unreachable!()), - ) - .await?; + .context("tempfile removal")?; + } + } else { + // Init temporarily repo to get bootstrap data, this creates a directory in the `initdb_path` path + run_initdb(self.conf, &pgdata_path, pg_version, &self.cancel).await?; + + // Upload the created data dir to S3 + if let Some(storage) = &self.remote_storage { + let pgdata_zstd = import_datadir::create_tar_zst(&pgdata_path).await?; + let pgdata_zstd = Bytes::from(pgdata_zstd); + backoff::retry( + || async { + self::remote_timeline_client::upload_initdb_dir( + storage, + &self.tenant_shard_id.tenant_id, + &timeline_id, + pgdata_zstd.clone(), + ) + .await + }, + |_| false, + 3, + u32::MAX, + "persist_initdb_tar_zst", + // TODO: use a cancellation token (https://github.com/neondatabase/neon/issues/5066) + backoff::Cancel::new(CancellationToken::new(), || unreachable!()), + ) + .await?; + } } + let pgdata_lsn = import_datadir::get_lsn_from_controlfile(&pgdata_path)?.align(); // Import the contents of the data directory at the initial checkpoint // LSN, and any WAL after that. diff --git a/pageserver/src/tenant/remote_timeline_client.rs b/pageserver/src/tenant/remote_timeline_client.rs index 183ee19a40..5b649a420c 100644 --- a/pageserver/src/tenant/remote_timeline_client.rs +++ b/pageserver/src/tenant/remote_timeline_client.rs @@ -188,6 +188,7 @@ use anyhow::Context; use camino::Utf8Path; use chrono::{NaiveDateTime, Utc}; +pub(crate) use download::download_initdb_tar_zst; use pageserver_api::shard::{ShardIndex, TenantShardId}; use scopeguard::ScopeGuard; use tokio_util::sync::CancellationToken; @@ -1077,7 +1078,17 @@ impl RemoteTimelineClient { let remaining_layers: Vec = remaining .into_iter() - .filter(|p| p!= &latest_index) + .filter(|p| { + if p == &latest_index { + return false; + } + if let Some(name) = p.object_name() { + if name == INITDB_PATH { + return false; + } + } + true + }) .inspect(|path| { if let Some(name) = path.object_name() { info!(%name, "deleting a file not referenced from index_part.json"); diff --git a/pageserver/src/tenant/remote_timeline_client/download.rs b/pageserver/src/tenant/remote_timeline_client/download.rs index 1e9dcfe76a..3b5fe4b207 100644 --- a/pageserver/src/tenant/remote_timeline_client/download.rs +++ b/pageserver/src/tenant/remote_timeline_client/download.rs @@ -8,11 +8,12 @@ use std::future::Future; use std::time::Duration; use anyhow::{anyhow, Context}; -use camino::Utf8Path; +use camino::{Utf8Path, Utf8PathBuf}; use pageserver_api::shard::TenantShardId; -use tokio::fs; -use tokio::io::AsyncWriteExt; +use tokio::fs::{self, File, OpenOptions}; +use tokio::io::{AsyncSeekExt, AsyncWriteExt}; use tokio_util::sync::CancellationToken; +use tracing::warn; use utils::{backoff, crashsafe}; use crate::config::PageServerConf; @@ -20,14 +21,15 @@ use crate::tenant::remote_timeline_client::{remote_layer_path, remote_timelines_ use crate::tenant::storage_layer::LayerFileName; use crate::tenant::timeline::span::debug_assert_current_span_has_tenant_and_timeline_id; use crate::tenant::Generation; +use crate::TEMP_FILE_SUFFIX; use remote_storage::{DownloadError, GenericRemoteStorage, ListingMode}; use utils::crashsafe::path_with_suffix_extension; use utils::id::TimelineId; use super::index::{IndexPart, LayerFileMetadata}; use super::{ - parse_remote_index_path, remote_index_path, FAILED_DOWNLOAD_WARN_THRESHOLD, - FAILED_REMOTE_OP_RETRIES, + parse_remote_index_path, remote_index_path, remote_initdb_archive_path, + FAILED_DOWNLOAD_WARN_THRESHOLD, FAILED_REMOTE_OP_RETRIES, INITDB_PATH, }; static MAX_DOWNLOAD_DURATION: Duration = Duration::from_secs(120); @@ -374,6 +376,69 @@ pub(super) async fn download_index_part( } } +pub(crate) async fn download_initdb_tar_zst( + conf: &'static PageServerConf, + storage: &GenericRemoteStorage, + tenant_shard_id: &TenantShardId, + timeline_id: &TimelineId, +) -> Result<(Utf8PathBuf, File), DownloadError> { + debug_assert_current_span_has_tenant_and_timeline_id(); + + let remote_path = remote_initdb_archive_path(&tenant_shard_id.tenant_id, timeline_id); + + let timeline_path = conf.timelines_path(tenant_shard_id); + + if !timeline_path.exists() { + tokio::fs::create_dir_all(&timeline_path) + .await + .with_context(|| format!("timeline dir creation {timeline_path}")) + .map_err(DownloadError::Other)?; + } + let temp_path = timeline_path.join(format!("{INITDB_PATH}-{timeline_id}.{TEMP_FILE_SUFFIX}")); + + let file = download_retry( + || async { + let mut file = OpenOptions::new() + .create(true) + .truncate(true) + .read(true) + .write(true) + .open(&temp_path) + .await + .with_context(|| format!("tempfile creation {temp_path}")) + .map_err(DownloadError::Other)?; + + let mut download = storage.download(&remote_path).await?; + + tokio::io::copy(&mut download.download_stream, &mut file) + .await + .with_context(|| format!("download initdb.tar.zst at {remote_path:?}")) + .map_err(DownloadError::Other)?; + + file.seek(std::io::SeekFrom::Start(0)) + .await + .with_context(|| format!("rewinding initdb.tar.zst at: {remote_path:?}")) + .map_err(DownloadError::Other)?; + + Ok(file) + }, + &format!("download {remote_path}"), + ) + .await + .map_err(|e| { + if temp_path.exists() { + // Do a best-effort attempt at deleting the temporary file upon encountering an error. + // We don't have async here nor do we want to pile on any extra errors. + if let Err(e) = std::fs::remove_file(&temp_path) { + warn!("error deleting temporary file {temp_path}: {e}"); + } + } + e + })?; + + Ok((temp_path, file)) +} + /// Helper function to handle retries for a download operation. /// /// Remote operations can fail due to rate limits (IAM, S3), spurious network diff --git a/pageserver/src/walingest.rs b/pageserver/src/walingest.rs index 4189200d5c..d6e01d01be 100644 --- a/pageserver/src/walingest.rs +++ b/pageserver/src/walingest.rs @@ -2115,7 +2115,7 @@ mod tests { .load() .await; let tline = tenant - .bootstrap_timeline(TIMELINE_ID, pg_version, &ctx) + .bootstrap_timeline(TIMELINE_ID, pg_version, None, &ctx) .await .unwrap(); diff --git a/test_runner/fixtures/pageserver/http.py b/test_runner/fixtures/pageserver/http.py index 0e00d4a7de..76aa40122f 100644 --- a/test_runner/fixtures/pageserver/http.py +++ b/test_runner/fixtures/pageserver/http.py @@ -362,12 +362,16 @@ class PageserverHttpClient(requests.Session): new_timeline_id: TimelineId, ancestor_timeline_id: Optional[TimelineId] = None, ancestor_start_lsn: Optional[Lsn] = None, + existing_initdb_timeline_id: Optional[TimelineId] = None, **kwargs, ) -> Dict[Any, Any]: body: Dict[str, Any] = { "new_timeline_id": str(new_timeline_id), "ancestor_start_lsn": str(ancestor_start_lsn) if ancestor_start_lsn else None, "ancestor_timeline_id": str(ancestor_timeline_id) if ancestor_timeline_id else None, + "existing_initdb_timeline_id": str(existing_initdb_timeline_id) + if existing_initdb_timeline_id + else None, } if pg_version != PgVersion.NOT_SET: body["pg_version"] = int(pg_version) diff --git a/test_runner/fixtures/pageserver/utils.py b/test_runner/fixtures/pageserver/utils.py index 007ff387f4..e7b78cfb9a 100644 --- a/test_runner/fixtures/pageserver/utils.py +++ b/test_runner/fixtures/pageserver/utils.py @@ -1,7 +1,7 @@ import time -from typing import TYPE_CHECKING, Any, Dict, Optional +from typing import TYPE_CHECKING, Any, Dict, List, Optional -from mypy_boto3_s3.type_defs import ListObjectsV2OutputTypeDef +from mypy_boto3_s3.type_defs import ListObjectsV2OutputTypeDef, ObjectTypeDef from fixtures.log_helper import log from fixtures.pageserver.http import PageserverApiException, PageserverHttpClient @@ -235,10 +235,14 @@ if TYPE_CHECKING: from fixtures.neon_fixtures import NeonEnvBuilder -def assert_prefix_empty(neon_env_builder: "NeonEnvBuilder", prefix: Optional[str] = None): +def assert_prefix_empty( + neon_env_builder: "NeonEnvBuilder", + prefix: Optional[str] = None, + allowed_postfix: Optional[str] = None, +): response = list_prefix(neon_env_builder, prefix) keys = response["KeyCount"] - objects = response.get("Contents", []) + objects: List[ObjectTypeDef] = response.get("Contents", []) common_prefixes = response.get("CommonPrefixes", []) remote_storage = neon_env_builder.pageserver_remote_storage @@ -261,7 +265,18 @@ def assert_prefix_empty(neon_env_builder: "NeonEnvBuilder", prefix: Optional[str f"contradicting ListObjectsV2 response with KeyCount={keys} and Contents={objects}, CommonPrefixes={common_prefixes}" ) - assert keys == 0, f"remote dir with prefix {prefix} is not empty after deletion: {objects}" + filtered_count = 0 + if allowed_postfix is None: + filtered_count = len(objects) + else: + for _obj in objects: + key: str = str(response.get("Key", [])) + if not (allowed_postfix.endswith(key)): + filtered_count += 1 + + assert ( + filtered_count == 0 + ), f"remote dir with prefix {prefix} is not empty after deletion: {objects}" def assert_prefix_not_empty(neon_env_builder: "NeonEnvBuilder", prefix: Optional[str] = None): diff --git a/test_runner/regress/test_remote_storage.py b/test_runner/regress/test_remote_storage.py index ccf28cae8c..9c2bb2db11 100644 --- a/test_runner/regress/test_remote_storage.py +++ b/test_runner/regress/test_remote_storage.py @@ -603,7 +603,12 @@ def test_timeline_deletion_with_files_stuck_in_upload_queue( assert isinstance(env.pageserver_remote_storage, LocalFsStorage) remote_timeline_path = env.pageserver_remote_storage.timeline_path(tenant_id, timeline_id) - assert not list(remote_timeline_path.iterdir()) + filtered = [ + path + for path in remote_timeline_path.iterdir() + if not (path.name.endswith("initdb.tar.zst")) + ] + assert len(filtered) == 0 # timeline deletion should kill ongoing uploads, so, the metric will be gone assert get_queued_count(file_kind="index", op_kind="upload") is None diff --git a/test_runner/regress/test_tenant_delete.py b/test_runner/regress/test_tenant_delete.py index 167c8355cc..9bafa60d18 100644 --- a/test_runner/regress/test_tenant_delete.py +++ b/test_runner/regress/test_tenant_delete.py @@ -197,6 +197,7 @@ def test_delete_tenant_exercise_crash_safety_failpoints( # So by ignoring these instead of waiting for empty upload queue # we execute more distinct code paths. '.*stopping left-over name="remote upload".*', + ".*Failed to load index_part from remote storage, failed creation?.*", ] ) @@ -285,6 +286,7 @@ def test_delete_tenant_exercise_crash_safety_failpoints( str(tenant_id), ) ), + allowed_postfix="initdb.tar.zst", ) diff --git a/test_runner/regress/test_tenants.py b/test_runner/regress/test_tenants.py index 090d586721..a31f410f94 100644 --- a/test_runner/regress/test_tenants.py +++ b/test_runner/regress/test_tenants.py @@ -290,10 +290,13 @@ def test_pageserver_with_empty_tenants( env = neon_env_builder.init_start() - env.pageserver.allowed_errors.append( - ".*marking .* as locally complete, while it doesnt exist in remote index.*" + env.pageserver.allowed_errors.extend( + [ + ".*marking .* as locally complete, while it doesnt exist in remote index.*", + ".*Failed to load index_part from remote storage, failed creation?.*", + ".*load failed.*list timelines directory.*", + ] ) - env.pageserver.allowed_errors.append(".*load failed.*list timelines directory.*") client = env.pageserver.http_client() diff --git a/test_runner/regress/test_timeline_delete.py b/test_runner/regress/test_timeline_delete.py index 2e1fcd38fe..51ad971cdb 100644 --- a/test_runner/regress/test_timeline_delete.py +++ b/test_runner/regress/test_timeline_delete.py @@ -230,6 +230,9 @@ def test_delete_timeline_exercise_crash_safety_failpoints( env.pageserver.allowed_errors.append(".*Timeline dir entry become invalid.*") # In one of the branches we poll for tenant to become active. Polls can generate this log message: env.pageserver.allowed_errors.append(f".*Tenant {env.initial_tenant} is not active*") + env.pageserver.allowed_errors.append( + ".*Failed to load index_part from remote storage, failed creation?.*" + ) ps_http.configure_failpoints((failpoint, "return")) @@ -308,8 +311,9 @@ def test_delete_timeline_exercise_crash_safety_failpoints( ) timeline_dir = env.pageserver.timeline_dir(env.initial_tenant, timeline_id) - # Check local is empty - assert not timeline_dir.exists() + if failpoint != "timeline-delete-after-index-delete": + # Check local is empty + assert (not timeline_dir.exists()) or len(os.listdir(timeline_dir)) == 0 # Check no delete mark present assert not (timeline_dir.parent / f"{timeline_id}.___deleted").exists() diff --git a/test_runner/regress/test_wal_restore.py b/test_runner/regress/test_wal_restore.py index b039b36255..4a9ffeee4b 100644 --- a/test_runner/regress/test_wal_restore.py +++ b/test_runner/regress/test_wal_restore.py @@ -1,6 +1,7 @@ import sys import tarfile import tempfile +import time from pathlib import Path import pytest @@ -125,3 +126,43 @@ def test_wal_restore_initdb( ) log.info(f"original lsn: {original_lsn}, restored lsn: {restored_lsn}") assert restored.safe_psql("select count(*) from t", user="cloud_admin") == [(300000,)] + + +def test_wal_restore_http( + neon_env_builder: NeonEnvBuilder, + test_output_dir: Path, +): + env = neon_env_builder.init_start() + endpoint = env.endpoints.create_start("main") + endpoint.safe_psql("create table t as select generate_series(1,300000)") + tenant_id = env.initial_tenant + timeline_id = env.initial_timeline + + ps_client = env.pageserver.http_client() + + # shut down the endpoint and delete the timeline from the pageserver + endpoint.stop() + + assert isinstance(env.pageserver_remote_storage, LocalFsStorage) + + test_output_dir / "initdb.tar.zst" + + (env.pageserver_remote_storage.timeline_path(tenant_id, timeline_id) / "initdb.tar.zst") + + ps_client.timeline_delete(tenant_id, timeline_id) + time.sleep(2) + + # verify that it is indeed deleted + # TODO + + # issue the restoration command + ps_client.timeline_create( + tenant_id=tenant_id, + new_timeline_id=timeline_id, + existing_initdb_timeline_id=timeline_id, + pg_version=env.pg_version, + ) + + # the table is back now! + restored = env.endpoints.create_start("main") + assert restored.safe_psql("select count(*) from t", user="cloud_admin") == [(300000,)] From f784e59b12a045e8a38613fe384315336e8f75ce Mon Sep 17 00:00:00 2001 From: Vadim Kharitonov Date: Fri, 1 Dec 2023 00:12:52 +0100 Subject: [PATCH 30/38] Update timescaledb to 2.13.0 (#5975) TimescaleDB has released 2.13.0. This version is compatible with Postgres16 --- Dockerfile.compute-node | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/Dockerfile.compute-node b/Dockerfile.compute-node index 8109091e76..425f163e8b 100644 --- a/Dockerfile.compute-node +++ b/Dockerfile.compute-node @@ -387,18 +387,10 @@ COPY --from=pg-build /usr/local/pgsql/ /usr/local/pgsql/ ARG PG_VERSION ENV PATH "/usr/local/pgsql/bin:$PATH" -RUN case "${PG_VERSION}" in \ - "v14" | "v15") \ - export TIMESCALEDB_VERSION=2.10.1 \ - export TIMESCALEDB_CHECKSUM=6fca72a6ed0f6d32d2b3523951ede73dc5f9b0077b38450a029a5f411fdb8c73 \ - ;; \ - *) \ - echo "TimescaleDB not supported on this PostgreSQL version. See https://github.com/timescale/timescaledb/issues/5752" && exit 0;; \ - esac && \ - apt-get update && \ +RUN apt-get update && \ apt-get install -y cmake && \ - wget https://github.com/timescale/timescaledb/archive/refs/tags/${TIMESCALEDB_VERSION}.tar.gz -O timescaledb.tar.gz && \ - echo "${TIMESCALEDB_CHECKSUM} timescaledb.tar.gz" | sha256sum --check && \ + wget https://github.com/timescale/timescaledb/archive/refs/tags/2.13.0.tar.gz -O timescaledb.tar.gz && \ + echo "584a351c7775f0e067eaa0e7277ea88cab9077cc4c455cbbf09a5d9723dce95d timescaledb.tar.gz" | sha256sum --check && \ mkdir timescaledb-src && cd timescaledb-src && tar xvzf ../timescaledb.tar.gz --strip-components=1 -C . && \ ./bootstrap -DSEND_TELEMETRY_DEFAULT:BOOL=OFF -DUSE_TELEMETRY:BOOL=OFF -DAPACHE_ONLY:BOOL=ON -DCMAKE_BUILD_TYPE=Release && \ cd build && \ From 1ce1c82d78676257d007bbae667109cc2ab1a634 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arpad=20M=C3=BCller?= Date: Fri, 1 Dec 2023 11:58:06 +0100 Subject: [PATCH 31/38] Clean up local state if index_part.json request gives 404 (#6009) If `index_part.json` is (verifiably) not present on remote storage, we should regard the timeline as inexistent. This lets `clean_up_timelines` purge the partial local disk state, which is important in the case of incomplete creations leaving behind state that hinders retries. For incomplete deletions, we also want the timeline's local disk content be gone completely. The PR removes the allowed warnings added by #5390 and #5912, as we now are only supposed to issue info level messages. It also adds a reproducer for #6007, by parametrizing the `test_timeline_init_break_before_checkpoint_recreate` test added by #5390. If one reverts the .rs changes, the "cannot create its uninit mark file" log line occurs once one comments out the failing checks for the local disk state being actually empty. Closes #6007 --------- Co-authored-by: Joonas Koivunen --- pageserver/src/tenant.rs | 25 +++++++++++----- test_runner/regress/test_broken_timeline.py | 33 +++++++++++++-------- test_runner/regress/test_tenant_delete.py | 1 - test_runner/regress/test_tenants.py | 1 - test_runner/regress/test_timeline_delete.py | 10 +++---- 5 files changed, 42 insertions(+), 28 deletions(-) diff --git a/pageserver/src/tenant.rs b/pageserver/src/tenant.rs index bf063664b4..12610b1c89 100644 --- a/pageserver/src/tenant.rs +++ b/pageserver/src/tenant.rs @@ -796,20 +796,31 @@ impl Tenant { let mut timeline_ancestors = HashMap::new(); let mut existent_timelines = HashSet::new(); for (timeline_id, preload) in preload.timelines { - // In this context a timeline "exists" if it has any content in remote storage: this will - // be our cue to not delete any corresponding local directory - existent_timelines.insert(timeline_id); - let index_part = match preload.index_part { Ok(i) => { debug!("remote index part exists for timeline {timeline_id}"); + // We found index_part on the remote, this is the standard case. + existent_timelines.insert(timeline_id); i } + Err(DownloadError::NotFound) => { + // There is no index_part on the remote. We only get here + // if there is some prefix for the timeline in the remote storage. + // This can e.g. be the initdb.tar.zst archive, maybe a + // remnant from a prior incomplete creation or deletion attempt. + // Delete the local directory as the deciding criterion for a + // timeline's existence is presence of index_part. + info!(%timeline_id, "index_part not found on remote"); + continue; + } Err(e) => { - // Timeline creation is not atomic: we might upload a layer but no index_part. We expect - // that the creation will be retried by the control plane and eventually result in - // a valid loadable state. + // Some (possibly ephemeral) error happened during index_part download. + // Pretend the timeline exists to not delete the timeline directory, + // as it might be a temporary issue and we don't want to re-download + // everything after it resolves. warn!(%timeline_id, "Failed to load index_part from remote storage, failed creation? ({e})"); + + existent_timelines.insert(timeline_id); continue; } }; diff --git a/test_runner/regress/test_broken_timeline.py b/test_runner/regress/test_broken_timeline.py index 4911fc09d6..84a322039a 100644 --- a/test_runner/regress/test_broken_timeline.py +++ b/test_runner/regress/test_broken_timeline.py @@ -114,7 +114,6 @@ def test_timeline_init_break_before_checkpoint(neon_env_builder: NeonEnvBuilder) [ ".*Failed to process timeline dir contents.*Timeline has no ancestor and no layer files.*", ".*Timeline got dropped without initializing, cleaning its files.*", - ".*Failed to load index_part from remote storage, failed creation?.*", ] ) @@ -144,8 +143,13 @@ def test_timeline_init_break_before_checkpoint(neon_env_builder: NeonEnvBuilder) ), "pageserver should clean its temp timeline files on timeline creation failure" -def test_timeline_init_break_before_checkpoint_recreate(neon_env_builder: NeonEnvBuilder): - env = neon_env_builder.init_start() +# The "exit" case is for a reproducer of issue 6007: an unclean shutdown where we can't do local fs cleanups +@pytest.mark.parametrize("exit_or_return", ["return", "exit"]) +def test_timeline_init_break_before_checkpoint_recreate( + neon_env_builder: NeonEnvBuilder, exit_or_return: str +): + env = neon_env_builder.init_configs() + env.start() pageserver_http = env.pageserver.http_client() env.pageserver.allowed_errors.extend( @@ -156,6 +160,7 @@ def test_timeline_init_break_before_checkpoint_recreate(neon_env_builder: NeonEn ] ) + pageserver_http.tenant_create(env.initial_tenant) tenant_id = env.initial_tenant timelines_dir = env.pageserver.timeline_dir(tenant_id) @@ -166,13 +171,17 @@ def test_timeline_init_break_before_checkpoint_recreate(neon_env_builder: NeonEn timeline_id = TimelineId("1080243c1f76fe3c5147266663c9860b") # Introduce failpoint during timeline init (some intermediate files are on disk), before it's checkpointed. - pageserver_http.configure_failpoints(("before-checkpoint-new-timeline", "return")) - with pytest.raises(Exception, match="before-checkpoint-new-timeline"): - _ = env.neon_cli.create_timeline( - "test_timeline_init_break_before_checkpoint", tenant_id, timeline_id - ) + failpoint = "before-checkpoint-new-timeline" + pattern = failpoint + if exit_or_return == "exit": + # in reality a read error happens, but there are automatic retries which now fail because pageserver is dead + pattern = "Connection aborted." - # Restart the page server + pageserver_http.configure_failpoints((failpoint, exit_or_return)) + with pytest.raises(Exception, match=pattern): + _ = pageserver_http.timeline_create(env.pg_version, tenant_id, timeline_id) + + # Restart the page server (with the failpoint disabled) env.pageserver.restart(immediate=True) # Creating the timeline didn't finish. The other timelines on tenant should still be present and work normally. @@ -186,11 +195,9 @@ def test_timeline_init_break_before_checkpoint_recreate(neon_env_builder: NeonEn timeline_dirs == initial_timeline_dirs ), "pageserver should clean its temp timeline files on timeline creation failure" - # Disable the failpoint again - pageserver_http.configure_failpoints(("before-checkpoint-new-timeline", "off")) # creating the branch should have worked now - new_timeline_id = env.neon_cli.create_timeline( - "test_timeline_init_break_before_checkpoint", tenant_id, timeline_id + new_timeline_id = TimelineId( + pageserver_http.timeline_create(env.pg_version, tenant_id, timeline_id)["timeline_id"] ) assert timeline_id == new_timeline_id diff --git a/test_runner/regress/test_tenant_delete.py b/test_runner/regress/test_tenant_delete.py index 9bafa60d18..89c474286a 100644 --- a/test_runner/regress/test_tenant_delete.py +++ b/test_runner/regress/test_tenant_delete.py @@ -197,7 +197,6 @@ def test_delete_tenant_exercise_crash_safety_failpoints( # So by ignoring these instead of waiting for empty upload queue # we execute more distinct code paths. '.*stopping left-over name="remote upload".*', - ".*Failed to load index_part from remote storage, failed creation?.*", ] ) diff --git a/test_runner/regress/test_tenants.py b/test_runner/regress/test_tenants.py index a31f410f94..c81be41530 100644 --- a/test_runner/regress/test_tenants.py +++ b/test_runner/regress/test_tenants.py @@ -293,7 +293,6 @@ def test_pageserver_with_empty_tenants( env.pageserver.allowed_errors.extend( [ ".*marking .* as locally complete, while it doesnt exist in remote index.*", - ".*Failed to load index_part from remote storage, failed creation?.*", ".*load failed.*list timelines directory.*", ] ) diff --git a/test_runner/regress/test_timeline_delete.py b/test_runner/regress/test_timeline_delete.py index 51ad971cdb..b1a2755394 100644 --- a/test_runner/regress/test_timeline_delete.py +++ b/test_runner/regress/test_timeline_delete.py @@ -230,9 +230,6 @@ def test_delete_timeline_exercise_crash_safety_failpoints( env.pageserver.allowed_errors.append(".*Timeline dir entry become invalid.*") # In one of the branches we poll for tenant to become active. Polls can generate this log message: env.pageserver.allowed_errors.append(f".*Tenant {env.initial_tenant} is not active*") - env.pageserver.allowed_errors.append( - ".*Failed to load index_part from remote storage, failed creation?.*" - ) ps_http.configure_failpoints((failpoint, "return")) @@ -311,9 +308,10 @@ def test_delete_timeline_exercise_crash_safety_failpoints( ) timeline_dir = env.pageserver.timeline_dir(env.initial_tenant, timeline_id) - if failpoint != "timeline-delete-after-index-delete": - # Check local is empty - assert (not timeline_dir.exists()) or len(os.listdir(timeline_dir)) == 0 + + # Check local is empty + assert (not timeline_dir.exists()) or len(os.listdir(timeline_dir)) == 0 + # Check no delete mark present assert not (timeline_dir.parent / f"{timeline_id}.___deleted").exists() From 1c88824ed0e6bfbce02fa92e13ca91d5ab0e37b3 Mon Sep 17 00:00:00 2001 From: Christian Schwarz Date: Fri, 1 Dec 2023 12:52:59 +0100 Subject: [PATCH 32/38] initial logical size calculation: add a bunch of metrics (#5995) These will help us answer questions such as: - when & at what do calculations get started after PS restart? - how often is the api to get current incrementally-computed logical size called, and does it return Exact vs Approximate? I'd also be interested in a histogram of how much wall clock time size calculations take, but, I don't know good bucket sizes, and, logging it would introduce yet another per-timeline log message during startup; don't think that's worth it just yet. Context - https://neondb.slack.com/archives/C033RQ5SPDH/p1701197668789769 - https://github.com/neondatabase/neon/issues/5962 - https://github.com/neondatabase/neon/issues/5963 - https://github.com/neondatabase/neon/pull/5955 - https://github.com/neondatabase/cloud/issues/7408 --- pageserver/src/metrics.rs | 128 ++++++++++++++++++ pageserver/src/tenant/timeline.rs | 14 +- .../src/tenant/timeline/logical_size.rs | 19 ++- 3 files changed, 154 insertions(+), 7 deletions(-) diff --git a/pageserver/src/metrics.rs b/pageserver/src/metrics.rs index c15b1fce63..6aee40b579 100644 --- a/pageserver/src/metrics.rs +++ b/pageserver/src/metrics.rs @@ -403,6 +403,134 @@ static CURRENT_LOGICAL_SIZE: Lazy = Lazy::new(|| { .expect("failed to define current logical size metric") }); +pub(crate) mod initial_logical_size { + use metrics::{register_int_counter, register_int_counter_vec, IntCounter, IntCounterVec}; + use once_cell::sync::Lazy; + + use crate::task_mgr::TaskKind; + + pub(crate) struct StartCalculation(IntCounterVec); + pub(crate) static START_CALCULATION: Lazy = Lazy::new(|| { + StartCalculation( + register_int_counter_vec!( + "pageserver_initial_logical_size_start_calculation", + "Incremented each time we start an initial logical size calculation attempt. \ + The `task_kind` label is for the task kind that caused this attempt.", + &["attempt", "task_kind"] + ) + .unwrap(), + ) + }); + + struct DropCalculation { + first: IntCounter, + retry: IntCounter, + } + + static DROP_CALCULATION: Lazy = Lazy::new(|| { + let vec = register_int_counter_vec!( + "pageserver_initial_logical_size_drop_calculation", + "Incremented each time we abort a started size calculation attmpt.", + &["attempt"] + ) + .unwrap(); + DropCalculation { + first: vec.with_label_values(&["first"]), + retry: vec.with_label_values(&["retry"]), + } + }); + + pub(crate) struct Calculated { + pub(crate) births: IntCounter, + pub(crate) deaths: IntCounter, + } + + pub(crate) static CALCULATED: Lazy = Lazy::new(|| Calculated { + births: register_int_counter!( + "pageserver_initial_logical_size_finish_calculation", + "Incremented every time we finish calculation of initial logical size.\ + If everything is working well, this should happen at most once per Timeline object." + ) + .unwrap(), + deaths: register_int_counter!( + "pageserver_initial_logical_size_drop_finished_calculation", + "Incremented when we drop a finished initial logical size calculation result.\ + Mainly useful to turn pageserver_initial_logical_size_finish_calculation into a gauge." + ) + .unwrap(), + }); + + pub(crate) struct OngoingCalculationGuard { + inc_drop_calculation: Option, + } + + impl StartCalculation { + pub(crate) fn first(&self, causing_task_kind: Option) -> OngoingCalculationGuard { + let task_kind_label: &'static str = + causing_task_kind.map(|k| k.into()).unwrap_or_default(); + self.0.with_label_values(&["first", task_kind_label]); + OngoingCalculationGuard { + inc_drop_calculation: Some(DROP_CALCULATION.first.clone()), + } + } + pub(crate) fn retry(&self, causing_task_kind: Option) -> OngoingCalculationGuard { + let task_kind_label: &'static str = + causing_task_kind.map(|k| k.into()).unwrap_or_default(); + self.0.with_label_values(&["retry", task_kind_label]); + OngoingCalculationGuard { + inc_drop_calculation: Some(DROP_CALCULATION.retry.clone()), + } + } + } + + impl Drop for OngoingCalculationGuard { + fn drop(&mut self) { + if let Some(counter) = self.inc_drop_calculation.take() { + counter.inc(); + } + } + } + + impl OngoingCalculationGuard { + pub(crate) fn calculation_result_saved(mut self) -> FinishedCalculationGuard { + drop(self.inc_drop_calculation.take()); + CALCULATED.births.inc(); + FinishedCalculationGuard { + inc_on_drop: CALCULATED.deaths.clone(), + } + } + } + + pub(crate) struct FinishedCalculationGuard { + inc_on_drop: IntCounter, + } + + impl Drop for FinishedCalculationGuard { + fn drop(&mut self) { + self.inc_on_drop.inc(); + } + } + + pub(crate) struct Calls { + pub(crate) approximate: IntCounter, + pub(crate) exact: IntCounter, + } + + pub(crate) static CALLS: Lazy = Lazy::new(|| { + let vec = register_int_counter_vec!( + "pageserver_initial_logical_size_calls", + "Incremented each time some code asks for incremental logical size.\ + The label records the accuracy of the result.", + &["accuracy"] + ) + .unwrap(); + Calls { + approximate: vec.with_label_values(&["approximate"]), + exact: vec.with_label_values(&["exact"]), + } + }); +} + pub(crate) static TENANT_STATE_METRIC: Lazy = Lazy::new(|| { register_uint_gauge_vec!( "pageserver_tenant_states_count", diff --git a/pageserver/src/tenant/timeline.rs b/pageserver/src/tenant/timeline.rs index 9a7c9a6df3..341ec3e255 100644 --- a/pageserver/src/tenant/timeline.rs +++ b/pageserver/src/tenant/timeline.rs @@ -1812,6 +1812,7 @@ impl Timeline { "spawning logical size computation from context of task kind {:?}", ctx.task_kind() ); + let causing_task_kind = ctx.task_kind(); // We need to start the computation task. // It gets a separate context since it will outlive the request that called this function. let self_clone = Arc::clone(self); @@ -1839,6 +1840,8 @@ impl Timeline { _ = completion::Barrier::maybe_wait(self_clone.initial_logical_size_can_start.clone()) => {} }; + + // hold off background tasks from starting until all timelines get to try at least // once initial logical size calculation; though retry will rarely be useful. // holding off is done because heavier tasks execute blockingly on the same @@ -1846,7 +1849,12 @@ impl Timeline { // // dropping this at every outcome is probably better than trying to cling on to it, // delay will be terminated by a timeout regardless. - let _completion = { self_clone.initial_logical_size_attempt.lock().expect("unexpected initial_logical_size_attempt poisoned").take() }; + let completion = { self_clone.initial_logical_size_attempt.lock().expect("unexpected initial_logical_size_attempt poisoned").take() }; + + let metrics_guard = match &completion { + Some(_) => crate::metrics::initial_logical_size::START_CALCULATION.first(Some(causing_task_kind)), + None => crate::metrics::initial_logical_size::START_CALCULATION.retry(Some(causing_task_kind)), + }; let calculated_size = match self_clone .logical_size_calculation_task(lsn, LogicalSizeCalculationCause::Initial, &background_ctx) @@ -1891,11 +1899,11 @@ impl Timeline { match self_clone .current_logical_size .initial_logical_size - .set(calculated_size) + .set((calculated_size, metrics_guard.calculation_result_saved())) { Ok(()) => (), Err(_what_we_just_attempted_to_set) => { - let existing_size = self_clone + let (existing_size, _) = self_clone .current_logical_size .initial_logical_size .get() diff --git a/pageserver/src/tenant/timeline/logical_size.rs b/pageserver/src/tenant/timeline/logical_size.rs index d9c2bc4cb9..43d8faeb0b 100644 --- a/pageserver/src/tenant/timeline/logical_size.rs +++ b/pageserver/src/tenant/timeline/logical_size.rs @@ -23,7 +23,10 @@ pub(super) struct LogicalSize { /// /// NOTE: size at a given LSN is constant, but after a restart we will calculate /// the initial size at a different LSN. - pub initial_logical_size: OnceCell, + pub initial_logical_size: OnceCell<( + u64, + crate::metrics::initial_logical_size::FinishedCalculationGuard, + )>, /// Semaphore to track ongoing calculation of `initial_logical_size`. pub initial_size_computation: Arc, @@ -78,7 +81,11 @@ impl CurrentLogicalSize { impl LogicalSize { pub(super) fn empty_initial() -> Self { Self { - initial_logical_size: OnceCell::with_value(0), + initial_logical_size: OnceCell::with_value((0, { + crate::metrics::initial_logical_size::START_CALCULATION + .first(None) + .calculation_result_saved() + })), // initial_logical_size already computed, so, don't admit any calculations initial_size_computation: Arc::new(Semaphore::new(0)), initial_part_end: None, @@ -100,12 +107,16 @@ impl LogicalSize { // ^^^ keep this type explicit so that the casts in this function break if // we change the type. match self.initial_logical_size.get() { - Some(initial_size) => { + Some((initial_size, _)) => { + crate::metrics::initial_logical_size::CALLS.exact.inc(); initial_size.checked_add_signed(size_increment) .with_context(|| format!("Overflow during logical size calculation, initial_size: {initial_size}, size_increment: {size_increment}")) .map(CurrentLogicalSize::Exact) } None => { + crate::metrics::initial_logical_size::CALLS + .approximate + .inc(); let non_negative_size_increment = u64::try_from(size_increment).unwrap_or(0); Ok(CurrentLogicalSize::Approximate(non_negative_size_increment)) } @@ -121,7 +132,7 @@ impl LogicalSize { /// available for re-use. This doesn't contain the incremental part. pub(super) fn initialized_size(&self, lsn: Lsn) -> Option { match self.initial_part_end { - Some(v) if v == lsn => self.initial_logical_size.get().copied(), + Some(v) if v == lsn => self.initial_logical_size.get().map(|(s, _)| *s), _ => None, } } From 8cd28e17181124a1a0af122f3550e5a8e7877a8d Mon Sep 17 00:00:00 2001 From: Christian Schwarz Date: Wed, 29 Nov 2023 15:34:46 +0000 Subject: [PATCH 33/38] logical size calculation: make .current_size() infallible (#5999) ... by panicking on overflow; It was made fallible initially due to in-confidence in logical size calculation. However, the error has never happened since I am at Neon. Let's stop worrying about this by converting the overflow check into a panic. --- pageserver/src/tenant/timeline.rs | 8 +++----- pageserver/src/tenant/timeline/logical_size.rs | 8 ++++---- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/pageserver/src/tenant/timeline.rs b/pageserver/src/tenant/timeline.rs index 341ec3e255..2eaf015dd4 100644 --- a/pageserver/src/tenant/timeline.rs +++ b/pageserver/src/tenant/timeline.rs @@ -859,7 +859,7 @@ impl Timeline { self: &Arc, ctx: &RequestContext, ) -> anyhow::Result<(u64, bool)> { - let current_size = self.current_logical_size.current_size()?; + let current_size = self.current_logical_size.current_size(); debug!("Current size: {current_size:?}"); let mut is_exact = true; @@ -2057,16 +2057,14 @@ impl Timeline { // one value while current_logical_size is set to the // other. match logical_size.current_size() { - Ok(CurrentLogicalSize::Exact(new_current_size)) => self + CurrentLogicalSize::Exact(new_current_size) => self .metrics .current_logical_size_gauge .set(new_current_size), - Ok(CurrentLogicalSize::Approximate(_)) => { + CurrentLogicalSize::Approximate(_) => { // don't update the gauge yet, this allows us not to update the gauge back and // forth between the initial size calculation task. } - // this is overflow - Err(e) => error!("Failed to compute current logical size for metrics update: {e:?}"), } } diff --git a/pageserver/src/tenant/timeline/logical_size.rs b/pageserver/src/tenant/timeline/logical_size.rs index 43d8faeb0b..ec93955254 100644 --- a/pageserver/src/tenant/timeline/logical_size.rs +++ b/pageserver/src/tenant/timeline/logical_size.rs @@ -102,23 +102,23 @@ impl LogicalSize { } } - pub(super) fn current_size(&self) -> anyhow::Result { + pub(super) fn current_size(&self) -> CurrentLogicalSize { let size_increment: i64 = self.size_added_after_initial.load(AtomicOrdering::Acquire); // ^^^ keep this type explicit so that the casts in this function break if // we change the type. match self.initial_logical_size.get() { Some((initial_size, _)) => { crate::metrics::initial_logical_size::CALLS.exact.inc(); - initial_size.checked_add_signed(size_increment) + CurrentLogicalSize::Exact(initial_size.checked_add_signed(size_increment) .with_context(|| format!("Overflow during logical size calculation, initial_size: {initial_size}, size_increment: {size_increment}")) - .map(CurrentLogicalSize::Exact) + .unwrap()) } None => { crate::metrics::initial_logical_size::CALLS .approximate .inc(); let non_negative_size_increment = u64::try_from(size_increment).unwrap_or(0); - Ok(CurrentLogicalSize::Approximate(non_negative_size_increment)) + CurrentLogicalSize::Approximate(non_negative_size_increment) } } } From ce1652990d89c0d68aa2b788ebbd12c90e36dbc5 Mon Sep 17 00:00:00 2001 From: Christian Schwarz Date: Wed, 29 Nov 2023 16:55:13 +0000 Subject: [PATCH 34/38] logical size: better represent level of accuracy in the type system (#5999) I would love to not expose the in-accurate value int he mgmt API at all, and in fact control plane doesn't use it [^1]. But our tests do, and I have no desire to change them at this time. [^1]: https://github.com/neondatabase/cloud/pull/8317 --- libs/pageserver_api/src/models.rs | 4 +- pageserver/src/consumption_metrics/metrics.rs | 13 ++--- pageserver/src/http/routes.rs | 14 +++--- pageserver/src/tenant/timeline.rs | 15 +++--- .../src/tenant/timeline/logical_size.rs | 49 +++++++++++++++---- .../walreceiver/walreceiver_connection.rs | 7 +-- 6 files changed, 63 insertions(+), 39 deletions(-) diff --git a/libs/pageserver_api/src/models.rs b/libs/pageserver_api/src/models.rs index e6236f91e3..2234a06501 100644 --- a/libs/pageserver_api/src/models.rs +++ b/libs/pageserver_api/src/models.rs @@ -384,7 +384,9 @@ pub struct TimelineInfo { /// The LSN that we are advertizing to safekeepers pub remote_consistent_lsn_visible: Lsn, - pub current_logical_size: Option, // is None when timeline is Unloaded + pub current_logical_size: u64, + pub current_logical_size_is_accurate: bool, + /// Sum of the size of all layer files. /// If a layer is present in both local FS and S3, it counts only once. pub current_physical_size: Option, // is None when timeline is Unloaded diff --git a/pageserver/src/consumption_metrics/metrics.rs b/pageserver/src/consumption_metrics/metrics.rs index 213f08484c..c6ff91e560 100644 --- a/pageserver/src/consumption_metrics/metrics.rs +++ b/pageserver/src/consumption_metrics/metrics.rs @@ -1,5 +1,4 @@ -use crate::context::RequestContext; -use anyhow::Context; +use crate::{context::RequestContext, tenant::timeline::logical_size::CurrentLogicalSize}; use chrono::{DateTime, Utc}; use consumption_metrics::EventType; use futures::stream::StreamExt; @@ -352,13 +351,11 @@ impl TimelineSnapshot { let current_exact_logical_size = { let span = tracing::info_span!("collect_metrics_iteration", tenant_id = %t.tenant_shard_id.tenant_id, timeline_id = %t.timeline_id); - let res = span - .in_scope(|| t.get_current_logical_size(ctx)) - .context("get_current_logical_size"); - match res? { + let size = span.in_scope(|| t.get_current_logical_size(ctx)); + match size { // Only send timeline logical size when it is fully calculated. - (size, is_exact) if is_exact => Some(size), - (_, _) => None, + CurrentLogicalSize::Exact(ref size) => Some(size.into()), + CurrentLogicalSize::Approximate(_) => None, } }; diff --git a/pageserver/src/http/routes.rs b/pageserver/src/http/routes.rs index 7ab56ccab2..29a1ff52e8 100644 --- a/pageserver/src/http/routes.rs +++ b/pageserver/src/http/routes.rs @@ -338,13 +338,7 @@ async fn build_timeline_info_common( Lsn(0) => None, lsn @ Lsn(_) => Some(lsn), }; - let current_logical_size = match timeline.get_current_logical_size(ctx) { - Ok((size, _)) => Some(size), - Err(err) => { - error!("Timeline info creation failed to get current logical size: {err:?}"); - None - } - }; + let current_logical_size = timeline.get_current_logical_size(ctx); let current_physical_size = Some(timeline.layer_size_sum().await); let state = timeline.current_state(); let remote_consistent_lsn_projected = timeline @@ -368,7 +362,11 @@ async fn build_timeline_info_common( last_record_lsn, prev_record_lsn: Some(timeline.get_prev_record_lsn()), latest_gc_cutoff_lsn: *timeline.get_latest_gc_cutoff_lsn(), - current_logical_size, + current_logical_size: current_logical_size.size_dont_care_about_accuracy(), + current_logical_size_is_accurate: match current_logical_size.accuracy() { + tenant::timeline::logical_size::Accuracy::Approximate => false, + tenant::timeline::logical_size::Accuracy::Exact => true, + }, current_physical_size, current_logical_size_non_incremental: None, timeline_dir_layer_file_size_sum: None, diff --git a/pageserver/src/tenant/timeline.rs b/pageserver/src/tenant/timeline.rs index 2eaf015dd4..6313e7be84 100644 --- a/pageserver/src/tenant/timeline.rs +++ b/pageserver/src/tenant/timeline.rs @@ -2,7 +2,7 @@ pub mod delete; mod eviction_task; mod init; pub mod layer_manager; -mod logical_size; +pub(crate) mod logical_size; pub mod span; pub mod uninit; mod walreceiver; @@ -855,23 +855,20 @@ impl Timeline { /// the initial size calculation has not been run (gets triggered on the first size access). /// /// return size and boolean flag that shows if the size is exact - pub fn get_current_logical_size( + pub(crate) fn get_current_logical_size( self: &Arc, ctx: &RequestContext, - ) -> anyhow::Result<(u64, bool)> { + ) -> logical_size::CurrentLogicalSize { let current_size = self.current_logical_size.current_size(); debug!("Current size: {current_size:?}"); - let mut is_exact = true; - let size = current_size.size(); if let (CurrentLogicalSize::Approximate(_), Some(initial_part_end)) = (current_size, self.current_logical_size.initial_part_end) { - is_exact = false; self.try_spawn_size_init_task(initial_part_end, ctx); } - Ok((size, is_exact)) + current_size } /// Check if more than 'checkpoint_distance' of WAL has been accumulated in @@ -2057,10 +2054,10 @@ impl Timeline { // one value while current_logical_size is set to the // other. match logical_size.current_size() { - CurrentLogicalSize::Exact(new_current_size) => self + CurrentLogicalSize::Exact(ref new_current_size) => self .metrics .current_logical_size_gauge - .set(new_current_size), + .set(new_current_size.into()), CurrentLogicalSize::Approximate(_) => { // don't update the gauge yet, this allows us not to update the gauge back and // forth between the initial size calculation task. diff --git a/pageserver/src/tenant/timeline/logical_size.rs b/pageserver/src/tenant/timeline/logical_size.rs index ec93955254..1f103051ef 100644 --- a/pageserver/src/tenant/timeline/logical_size.rs +++ b/pageserver/src/tenant/timeline/logical_size.rs @@ -59,21 +59,50 @@ pub(super) struct LogicalSize { /// Normalized current size, that the data in pageserver occupies. #[derive(Debug, Clone, Copy)] -pub(super) enum CurrentLogicalSize { +pub(crate) enum CurrentLogicalSize { /// The size is not yet calculated to the end, this is an intermediate result, /// constructed from walreceiver increments and normalized: logical data could delete some objects, hence be negative, /// yet total logical size cannot be below 0. - Approximate(u64), + Approximate(Approximate), // Fully calculated logical size, only other future walreceiver increments are changing it, and those changes are // available for observation without any calculations. - Exact(u64), + Exact(Exact), +} + +#[derive(Debug, Copy, Clone)] +pub(crate) enum Accuracy { + Approximate, + Exact, +} + +#[derive(Debug, Clone, Copy)] +pub(crate) struct Approximate(u64); +#[derive(Debug, Clone, Copy)] +pub(crate) struct Exact(u64); + +impl From<&Approximate> for u64 { + fn from(value: &Approximate) -> Self { + value.0 + } +} + +impl From<&Exact> for u64 { + fn from(val: &Exact) -> Self { + val.0 + } } impl CurrentLogicalSize { - pub(super) fn size(&self) -> u64 { - *match self { - Self::Approximate(size) => size, - Self::Exact(size) => size, + pub(crate) fn size_dont_care_about_accuracy(&self) -> u64 { + match self { + Self::Approximate(size) => size.into(), + Self::Exact(size) => size.into(), + } + } + pub(crate) fn accuracy(&self) -> Accuracy { + match self { + Self::Approximate(_) => Accuracy::Approximate, + Self::Exact(_) => Accuracy::Exact, } } } @@ -109,16 +138,16 @@ impl LogicalSize { match self.initial_logical_size.get() { Some((initial_size, _)) => { crate::metrics::initial_logical_size::CALLS.exact.inc(); - CurrentLogicalSize::Exact(initial_size.checked_add_signed(size_increment) + CurrentLogicalSize::Exact(Exact(initial_size.checked_add_signed(size_increment) .with_context(|| format!("Overflow during logical size calculation, initial_size: {initial_size}, size_increment: {size_increment}")) - .unwrap()) + .unwrap())) } None => { crate::metrics::initial_logical_size::CALLS .approximate .inc(); let non_negative_size_increment = u64::try_from(size_increment).unwrap_or(0); - CurrentLogicalSize::Approximate(non_negative_size_increment) + CurrentLogicalSize::Approximate(Approximate(non_negative_size_increment)) } } } diff --git a/pageserver/src/tenant/timeline/walreceiver/walreceiver_connection.rs b/pageserver/src/tenant/timeline/walreceiver/walreceiver_connection.rs index 2b4aea7596..7045658f24 100644 --- a/pageserver/src/tenant/timeline/walreceiver/walreceiver_connection.rs +++ b/pageserver/src/tenant/timeline/walreceiver/walreceiver_connection.rs @@ -396,11 +396,12 @@ pub(super) async fn handle_walreceiver_connection( // Send the replication feedback message. // Regular standby_status_update fields are put into this message. - let (timeline_logical_size, _) = timeline + let current_timeline_size = timeline .get_current_logical_size(&ctx) - .context("Status update creation failed to get current logical size")?; + // FIXME: https://github.com/neondatabase/neon/issues/5963 + .size_dont_care_about_accuracy(); let status_update = PageserverFeedback { - current_timeline_size: timeline_logical_size, + current_timeline_size, last_received_lsn, disk_consistent_lsn, remote_consistent_lsn, From 7eabfc40eeb3303ca2a0ce041964cce2cbd861b0 Mon Sep 17 00:00:00 2001 From: Alexander Bayandin Date: Fri, 1 Dec 2023 13:26:19 +0000 Subject: [PATCH 35/38] test_runner: use separate directory for each rerun (#6004) ## Problem While investigating https://github.com/neondatabase/neon/issues/5854, we hypothesised that logs/repo-dir from the initial failure might leak into reruns. Use different directories for each run to avoid such a possibility. ## Summary of changes - make each test rerun use different directories - update `pytest-rerunfailure` plugin from 11.1.2 to 13.0 --- poetry.lock | 20 +++++--------------- pyproject.toml | 2 +- test_runner/fixtures/neon_fixtures.py | 5 +++++ 3 files changed, 11 insertions(+), 16 deletions(-) diff --git a/poetry.lock b/poetry.lock index 830f80dc97..a85325b696 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1967,18 +1967,18 @@ pytest = [ [[package]] name = "pytest-rerunfailures" -version = "11.1.2" +version = "13.0" description = "pytest plugin to re-run tests to eliminate flaky failures" optional = false python-versions = ">=3.7" files = [ - {file = "pytest-rerunfailures-11.1.2.tar.gz", hash = "sha256:55611661e873f1cafa384c82f08d07883954f4b76435f4b8a5b470c1954573de"}, - {file = "pytest_rerunfailures-11.1.2-py3-none-any.whl", hash = "sha256:d21fe2e46d9774f8ad95f1aa799544ae95cac3a223477af94aa985adfae92b7e"}, + {file = "pytest-rerunfailures-13.0.tar.gz", hash = "sha256:e132dbe420bc476f544b96e7036edd0a69707574209b6677263c950d19b09199"}, + {file = "pytest_rerunfailures-13.0-py3-none-any.whl", hash = "sha256:34919cb3fcb1f8e5d4b940aa75ccdea9661bade925091873b7c6fa5548333069"}, ] [package.dependencies] packaging = ">=17.1" -pytest = ">=5.3" +pytest = ">=7" [[package]] name = "pytest-split" @@ -2476,16 +2476,6 @@ files = [ {file = "wrapt-1.14.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:8ad85f7f4e20964db4daadcab70b47ab05c7c1cf2a7c1e51087bfaa83831854c"}, {file = "wrapt-1.14.1-cp310-cp310-win32.whl", hash = "sha256:a9a52172be0b5aae932bef82a79ec0a0ce87288c7d132946d645eba03f0ad8a8"}, {file = "wrapt-1.14.1-cp310-cp310-win_amd64.whl", hash = "sha256:6d323e1554b3d22cfc03cd3243b5bb815a51f5249fdcbb86fda4bf62bab9e164"}, - {file = "wrapt-1.14.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:ecee4132c6cd2ce5308e21672015ddfed1ff975ad0ac8d27168ea82e71413f55"}, - {file = "wrapt-1.14.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:2020f391008ef874c6d9e208b24f28e31bcb85ccff4f335f15a3251d222b92d9"}, - {file = "wrapt-1.14.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2feecf86e1f7a86517cab34ae6c2f081fd2d0dac860cb0c0ded96d799d20b335"}, - {file = "wrapt-1.14.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:240b1686f38ae665d1b15475966fe0472f78e71b1b4903c143a842659c8e4cb9"}, - {file = "wrapt-1.14.1-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a9008dad07d71f68487c91e96579c8567c98ca4c3881b9b113bc7b33e9fd78b8"}, - {file = "wrapt-1.14.1-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:6447e9f3ba72f8e2b985a1da758767698efa72723d5b59accefd716e9e8272bf"}, - {file = "wrapt-1.14.1-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:acae32e13a4153809db37405f5eba5bac5fbe2e2ba61ab227926a22901051c0a"}, - {file = "wrapt-1.14.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:49ef582b7a1152ae2766557f0550a9fcbf7bbd76f43fbdc94dd3bf07cc7168be"}, - {file = "wrapt-1.14.1-cp311-cp311-win32.whl", hash = "sha256:358fe87cc899c6bb0ddc185bf3dbfa4ba646f05b1b0b9b5a27c2cb92c2cea204"}, - {file = "wrapt-1.14.1-cp311-cp311-win_amd64.whl", hash = "sha256:26046cd03936ae745a502abf44dac702a5e6880b2b01c29aea8ddf3353b68224"}, {file = "wrapt-1.14.1-cp35-cp35m-manylinux1_i686.whl", hash = "sha256:43ca3bbbe97af00f49efb06e352eae40434ca9d915906f77def219b88e85d907"}, {file = "wrapt-1.14.1-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:6b1a564e6cb69922c7fe3a678b9f9a3c54e72b469875aa8018f18b4d1dd1adf3"}, {file = "wrapt-1.14.1-cp35-cp35m-manylinux2010_i686.whl", hash = "sha256:00b6d4ea20a906c0ca56d84f93065b398ab74b927a7a3dbd470f6fc503f95dc3"}, @@ -2707,4 +2697,4 @@ cffi = ["cffi (>=1.11)"] [metadata] lock-version = "2.0" python-versions = "^3.9" -content-hash = "25ffa9ed98d890a3b85e6036792296a60bb705e8f9eaa1f07336501116a58756" +content-hash = "9f33b4404dbb9803ede5785469241dde1d09132427b87db8928bdbc37ccd6b7a" diff --git a/pyproject.toml b/pyproject.toml index 60cf0e5c96..536efeab56 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -34,7 +34,7 @@ types-psutil = "^5.9.5.12" types-toml = "^0.10.8.6" pytest-httpserver = "^1.0.8" aiohttp = "3.9.0" -pytest-rerunfailures = "^11.1.2" +pytest-rerunfailures = "^13.0" types-pytest-lazy-fixture = "^0.6.3.3" pytest-split = "^0.8.1" zstandard = "^0.21.0" diff --git a/test_runner/fixtures/neon_fixtures.py b/test_runner/fixtures/neon_fixtures.py index 188851c39f..9545dc2dd5 100644 --- a/test_runner/fixtures/neon_fixtures.py +++ b/test_runner/fixtures/neon_fixtures.py @@ -3029,6 +3029,11 @@ def get_test_output_dir(request: FixtureRequest, top_output_dir: Path) -> Path: """Compute the working directory for an individual test.""" test_name = request.node.name test_dir = top_output_dir / test_name.replace("/", "-") + + # We rerun flaky tests multiple times, use a separate directory for each run. + if (suffix := getattr(request.node, "execution_count", None)) is not None: + test_dir = test_dir.parent / f"{test_dir.name}-{suffix}" + log.info(f"get_test_output_dir is {test_dir}") # make mypy happy assert isinstance(test_dir, Path) From 92e7cd40e88825a489c8d3c19c78ef001b59d06c Mon Sep 17 00:00:00 2001 From: Anastasia Lubennikova Date: Fri, 1 Dec 2023 13:40:49 +0000 Subject: [PATCH 36/38] add sql_exporter to vm-image (#5949) expose LFC metrics --- vm-image-spec.yaml | 85 +++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 84 insertions(+), 1 deletion(-) diff --git a/vm-image-spec.yaml b/vm-image-spec.yaml index d0ba742a63..6f0ebe5f66 100644 --- a/vm-image-spec.yaml +++ b/vm-image-spec.yaml @@ -13,6 +13,10 @@ commands: user: nobody sysvInitAction: respawn shell: 'DATA_SOURCE_NAME="user=cloud_admin sslmode=disable dbname=postgres" /bin/postgres_exporter' + - name: sql-exporter + user: nobody + sysvInitAction: respawn + shell: '/bin/sql_exporter -config.file=/etc/sql_exporter.yml' shutdownHook: | su -p postgres --session-command '/usr/local/bin/pg_ctl stop -D /var/db/postgres/compute/pgdata -m fast --wait -t 10' files: @@ -46,6 +50,77 @@ files: } memory {} } + - filename: sql_exporter.yml + content: | + # Configuration for sql_exporter + # Global defaults. + global: + # If scrape_timeout <= 0, no timeout is set unless Prometheus provides one. The default is 10s. + scrape_timeout: 10s + # Subtracted from Prometheus' scrape_timeout to give us some headroom and prevent Prometheus from timing out first. + scrape_timeout_offset: 500ms + # Minimum interval between collector runs: by default (0s) collectors are executed on every scrape. + min_interval: 0s + # Maximum number of open connections to any one target. Metric queries will run concurrently on multiple connections, + # as will concurrent scrapes. + max_connections: 1 + # Maximum number of idle connections to any one target. Unless you use very long collection intervals, this should + # always be the same as max_connections. + max_idle_connections: 1 + # Maximum number of maximum amount of time a connection may be reused. Expired connections may be closed lazily before reuse. + # If 0, connections are not closed due to a connection's age. + max_connection_lifetime: 5m + + # The target to monitor and the collectors to execute on it. + target: + # Data source name always has a URI schema that matches the driver name. In some cases (e.g. MySQL) + # the schema gets dropped or replaced to match the driver expected DSN format. + data_source_name: 'postgresql://cloud_admin@127.0.0.1:5432/postgres?sslmode=disable' + + # Collectors (referenced by name) to execute on the target. + # Glob patterns are supported (see for syntax). + collectors: [neon_collector] + + # Collector files specifies a list of globs. One collector definition is read from each matching file. + # Glob patterns are supported (see for syntax). + collector_files: + - "neon_collector.yml" + - filename: neon_collector.yml + content: | + collector_name: neon_collector + metrics: + - metric_name: lfc_misses + type: gauge + help: 'lfc_misses' + key_labels: + values: [lfc_misses] + query: | + select lfc_value as lfc_misses from neon.neon_lfc_stats where lfc_key='file_cache_misses'; + + - metric_name: lfc_used + type: gauge + help: 'lfc_used' + key_labels: + values: [lfc_used] + query: | + select lfc_value as lfc_used from neon.neon_lfc_stats where lfc_key='file_cache_used'; + + - metric_name: lfc_hits + type: gauge + help: 'lfc_hits' + key_labels: + values: [lfc_hits] + query: | + select lfc_value as lfc_hits from neon.neon_lfc_stats where lfc_key='file_cache_hits'; + + - metric_name: lfc_writes + type: gauge + help: 'lfc_writes' + key_labels: + values: [lfc_writes] + query: | + select lfc_value as lfc_writes from neon.neon_lfc_stats where lfc_key='file_cache_writes'; + build: | # Build cgroup-tools # @@ -82,6 +157,8 @@ build: | FROM quay.io/prometheuscommunity/postgres-exporter:v0.12.0 AS postgres-exporter + FROM burningalchemist/sql_exporter:0.13 AS sql-exporter + # Build pgbouncer # FROM debian:bullseye-slim AS pgbouncer @@ -116,13 +193,19 @@ merge: | COPY cgconfig.conf /etc/cgconfig.conf COPY pgbouncer.ini /etc/pgbouncer.ini + COPY sql_exporter.yml /etc/sql_exporter.yml + COPY neon_collector.yml /etc/neon_collector.yml + RUN set -e \ && chown postgres:postgres /etc/pgbouncer.ini \ && chmod 0644 /etc/pgbouncer.ini \ - && chmod 0644 /etc/cgconfig.conf + && chmod 0644 /etc/cgconfig.conf \ + && chmod 0644 /etc/sql_exporter.yml \ + && chmod 0644 /etc/neon_collector.yml COPY --from=libcgroup-builder /libcgroup-install/bin/* /usr/bin/ COPY --from=libcgroup-builder /libcgroup-install/lib/* /usr/lib/ COPY --from=libcgroup-builder /libcgroup-install/sbin/* /usr/sbin/ COPY --from=postgres-exporter /bin/postgres_exporter /bin/postgres_exporter + COPY --from=sql-exporter /bin/sql_exporter /bin/sql_exporter COPY --from=pgbouncer /usr/local/pgbouncer/bin/pgbouncer /usr/local/bin/pgbouncer From e49c21a3cd21469ab2a6362326cbb85f4c27e7fa Mon Sep 17 00:00:00 2001 From: bojanserafimov Date: Fri, 1 Dec 2023 09:11:41 -0500 Subject: [PATCH 37/38] Speed up rel extend (#5983) --- pageserver/src/walingest.rs | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/pageserver/src/walingest.rs b/pageserver/src/walingest.rs index d6e01d01be..cbb08f7ff1 100644 --- a/pageserver/src/walingest.rs +++ b/pageserver/src/walingest.rs @@ -1437,7 +1437,16 @@ impl<'a> WalIngest<'a> { // record. // TODO: would be nice if to be more explicit about it let last_lsn = modification.lsn; - let old_nblocks = if !self + + // Get current size and put rel creation if rel doesn't exist + // + // NOTE: we check the cache first even though get_rel_exists and get_rel_size would + // check the cache too. This is because eagerly checking the cache results in + // less work overall and 10% better performance. It's more work on cache miss + // but cache miss is rare. + let old_nblocks = if let Some(nblocks) = self.timeline.get_cached_rel_size(&rel, last_lsn) { + nblocks + } else if !self .timeline .get_rel_exists(rel, last_lsn, true, ctx) .await? From fd81945a60dff25fea65a50f2c8cd6e4c955fba4 Mon Sep 17 00:00:00 2001 From: bojanserafimov Date: Fri, 1 Dec 2023 09:16:24 -0500 Subject: [PATCH 38/38] Use TEST_OUTPUT envvar in pageserver (#5984) --- pageserver/src/config.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pageserver/src/config.rs b/pageserver/src/config.rs index 737495d414..13d1fc775b 100644 --- a/pageserver/src/config.rs +++ b/pageserver/src/config.rs @@ -855,7 +855,8 @@ impl PageServerConf { #[cfg(test)] pub fn test_repo_dir(test_name: &str) -> Utf8PathBuf { - Utf8PathBuf::from(format!("../tmp_check/test_{test_name}")) + let test_output_dir = std::env::var("TEST_OUTPUT").unwrap_or("../tmp_check".into()); + Utf8PathBuf::from(format!("{test_output_dir}/test_{test_name}")) } pub fn dummy_conf(repo_dir: Utf8PathBuf) -> Self {