diff --git a/.github/actions/download/action.yml b/.github/actions/download/action.yml index ce26e7825b..01c216b1ac 100644 --- a/.github/actions/download/action.yml +++ b/.github/actions/download/action.yml @@ -26,7 +26,7 @@ runs: TARGET: ${{ inputs.path }} ARCHIVE: /tmp/downloads/${{ inputs.name }}.tar.zst SKIP_IF_DOES_NOT_EXIST: ${{ inputs.skip-if-does-not-exist }} - PREFIX: artifacts/${{ inputs.prefix || format('{0}/{1}', github.run_id, github.run_attempt) }} + PREFIX: artifacts/${{ inputs.prefix || format('{0}/{1}/{2}', github.event.pull_request.head.sha || github.sha, github.run_id, github.run_attempt) }} run: | BUCKET=neon-github-public-dev FILENAME=$(basename $ARCHIVE) diff --git a/.github/actions/upload/action.yml b/.github/actions/upload/action.yml index 63973dfbe7..edcece7d2b 100644 --- a/.github/actions/upload/action.yml +++ b/.github/actions/upload/action.yml @@ -8,7 +8,7 @@ inputs: description: "A directory or file to upload" required: true prefix: - description: "S3 prefix. Default is '${GITHUB_RUN_ID}/${GITHUB_RUN_ATTEMPT}'" + description: "S3 prefix. Default is '${GITHUB_SHA}/${GITHUB_RUN_ID}/${GITHUB_RUN_ATTEMPT}'" required: false runs: @@ -45,7 +45,7 @@ runs: env: SOURCE: ${{ inputs.path }} ARCHIVE: /tmp/uploads/${{ inputs.name }}.tar.zst - PREFIX: artifacts/${{ inputs.prefix || format('{0}/{1}', github.run_id, github.run_attempt) }} + PREFIX: artifacts/${{ inputs.prefix || format('{0}/{1}/{2}', github.event.pull_request.head.sha || github.sha, github.run_id , github.run_attempt) }} run: | BUCKET=neon-github-public-dev FILENAME=$(basename $ARCHIVE) diff --git a/.github/workflows/build-build-tools-image.yml b/.github/workflows/build-build-tools-image.yml index 6e90a80ab7..5a94dd8e6f 100644 --- a/.github/workflows/build-build-tools-image.yml +++ b/.github/workflows/build-build-tools-image.yml @@ -78,7 +78,7 @@ jobs: pull: true file: Dockerfile.build-tools cache-from: type=registry,ref=neondatabase/build-tools:cache-${{ matrix.arch }} - cache-to: type=registry,ref=neondatabase/build-tools:cache-${{ matrix.arch }},mode=max + cache-to: ${{ github.ref_name == 'main' && format('type=registry,ref=neondatabase/build-tools:cache-{0},mode=max', matrix.arch) || '' }} tags: neondatabase/build-tools:${{ inputs.image-tag }}-${{ matrix.arch }} - name: Remove custom docker config directory diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index e9adf28b99..87f04996fd 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -763,7 +763,7 @@ jobs: pull: true file: Dockerfile cache-from: type=registry,ref=neondatabase/neon:cache-${{ matrix.arch }} - cache-to: type=registry,ref=neondatabase/neon:cache-${{ matrix.arch }},mode=max + cache-to: ${{ github.ref_name == 'main' && format('type=registry,ref=neondatabase/neon:cache-{0},mode=max', matrix.arch) || '' }} tags: | neondatabase/neon:${{ needs.tag.outputs.build-tag }}-${{ matrix.arch }} @@ -855,7 +855,7 @@ jobs: pull: true file: Dockerfile.compute-node cache-from: type=registry,ref=neondatabase/compute-node-${{ matrix.version }}:cache-${{ matrix.arch }} - cache-to: type=registry,ref=neondatabase/compute-node-${{ matrix.version }}:cache-${{ matrix.arch }},mode=max + cache-to: ${{ github.ref_name == 'main' && format('type=registry,ref=neondatabase/compute-node-{0}:cache-{1},mode=max', matrix.version, matrix.arch) || '' }} tags: | neondatabase/compute-node-${{ matrix.version }}:${{ needs.tag.outputs.build-tag }}-${{ matrix.arch }} @@ -875,7 +875,7 @@ jobs: file: Dockerfile.compute-node target: neon-pg-ext-test cache-from: type=registry,ref=neondatabase/neon-test-extensions-${{ matrix.version }}:cache-${{ matrix.arch }} - cache-to: type=registry,ref=neondatabase/neon-test-extensions-${{ matrix.version }}:cache-${{ matrix.arch }},mode=max + cache-to: ${{ github.ref_name == 'main' && format('type=registry,ref=neondatabase/neon-test-extensions-{0}:cache-{1},mode=max', matrix.version, matrix.arch) || '' }} tags: | neondatabase/neon-test-extensions-${{ matrix.version }}:${{needs.tag.outputs.build-tag}}-${{ matrix.arch }} @@ -1245,6 +1245,7 @@ jobs: run: | if [[ "$GITHUB_REF_NAME" == "main" ]]; then gh workflow --repo neondatabase/aws run deploy-dev.yml --ref main -f branch=main -f dockerTag=${{needs.tag.outputs.build-tag}} -f deployPreprodRegion=false + gh workflow --repo neondatabase/azure run deploy.yml -f dockerTag=${{needs.tag.outputs.build-tag}} elif [[ "$GITHUB_REF_NAME" == "release" ]]; then gh workflow --repo neondatabase/aws run deploy-dev.yml --ref main \ -f deployPgSniRouter=false \ diff --git a/.neon_clippy_args b/.neon_clippy_args index 25e09c61a6..4db32cf35c 100644 --- a/.neon_clippy_args +++ b/.neon_clippy_args @@ -1,4 +1,5 @@ # * `-A unknown_lints` – do not warn about unknown lint suppressions # that people with newer toolchains might use # * `-D warnings` - fail on any warnings (`cargo` returns non-zero exit status) -export CLIPPY_COMMON_ARGS="--locked --workspace --all-targets -- -A unknown_lints -D warnings" +# * `-D clippy::todo` - don't let `todo!()` slip into `main` +export CLIPPY_COMMON_ARGS="--locked --workspace --all-targets -- -A unknown_lints -D warnings -D clippy::todo" diff --git a/Cargo.lock b/Cargo.lock index cf8a0b3286..6b9a169ebf 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1014,6 +1014,9 @@ name = "camino" version = "1.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c59e92b5a388f549b863a7bea62612c09f24c8393560709a54558a9abdfb3b9c" +dependencies = [ + "serde", +] [[package]] name = "camino-tempfile" @@ -1243,7 +1246,7 @@ dependencies = [ "tokio-postgres", "tokio-stream", "tokio-util", - "toml_edit", + "toml_edit 0.19.10", "tracing", "tracing-opentelemetry", "tracing-subscriber", @@ -1359,8 +1362,8 @@ dependencies = [ "tokio", "tokio-postgres", "tokio-util", - "toml", - "toml_edit", + "toml 0.7.4", + "toml_edit 0.19.10", "tracing", "url", "utils", @@ -1666,9 +1669,9 @@ dependencies = [ [[package]] name = "diesel" -version = "2.1.4" +version = "2.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62c6fcf842f17f8c78ecf7c81d75c5ce84436b41ee07e03f490fbb5f5a8731d8" +checksum = "62d6dcd069e7b5fe49a302411f759d4cf1cf2c27fe798ef46fb8baefc053dd2b" dependencies = [ "bitflags 2.4.1", "byteorder", @@ -1681,11 +1684,12 @@ dependencies = [ [[package]] name = "diesel_derives" -version = "2.1.2" +version = "2.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef8337737574f55a468005a83499da720f20c65586241ffea339db9ecdfd2b44" +checksum = "59de76a222c2b8059f789cbe07afbfd8deb8c31dd0bc2a21f85e256c1def8259" dependencies = [ "diesel_table_macro_syntax", + "dsl_auto_type", "proc-macro2", "quote", "syn 2.0.52", @@ -1693,9 +1697,9 @@ dependencies = [ [[package]] name = "diesel_migrations" -version = "2.1.0" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6036b3f0120c5961381b570ee20a02432d7e2d27ea60de9578799cf9156914ac" +checksum = "8a73ce704bad4231f001bff3314d91dce4aba0770cee8b233991859abc15c1f6" dependencies = [ "diesel", "migrations_internals", @@ -1704,9 +1708,9 @@ dependencies = [ [[package]] name = "diesel_table_macro_syntax" -version = "0.1.0" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc5557efc453706fed5e4fa85006fe9817c224c3f480a34c7e5959fd700921c5" +checksum = "209c735641a413bc68c4923a9d6ad4bcb3ca306b794edaa7eb0b3228a99ffb25" dependencies = [ "syn 2.0.52", ] @@ -1742,6 +1746,20 @@ dependencies = [ "const-random", ] +[[package]] +name = "dsl_auto_type" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0892a17df262a24294c382f0d5997571006e7a4348b4327557c4ff1cd4a8bccc" +dependencies = [ + "darling", + "either", + "heck 0.5.0", + "proc-macro2", + "quote", + "syn 2.0.52", +] + [[package]] name = "dyn-clone" version = "1.0.14" @@ -3081,19 +3099,19 @@ dependencies = [ [[package]] name = "migrations_internals" -version = "2.1.0" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f23f71580015254b020e856feac3df5878c2c7a8812297edd6c0a485ac9dada" +checksum = "fd01039851e82f8799046eabbb354056283fb265c8ec0996af940f4e85a380ff" dependencies = [ "serde", - "toml", + "toml 0.8.14", ] [[package]] name = "migrations_macros" -version = "2.1.0" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cce3325ac70e67bbab5bd837a31cae01f1a6db64e0e744a33cb03a543469ef08" +checksum = "ffb161cc72176cb37aa47f1fc520d3ef02263d67d661f44f05d05a079e1237fd" dependencies = [ "migrations_internals", "proc-macro2", @@ -3573,7 +3591,7 @@ dependencies = [ "thiserror", "tokio", "tokio-util", - "toml_edit", + "toml_edit 0.19.10", "utils", "workspace_hack", ] @@ -3656,7 +3674,7 @@ dependencies = [ "tokio-stream", "tokio-tar", "tokio-util", - "toml_edit", + "toml_edit 0.19.10", "tracing", "twox-hash", "url", @@ -4647,6 +4665,7 @@ dependencies = [ "futures-util", "http-types", "humantime", + "humantime-serde", "hyper 0.14.26", "itertools", "metrics", @@ -4661,7 +4680,7 @@ dependencies = [ "tokio", "tokio-stream", "tokio-util", - "toml_edit", + "toml_edit 0.19.10", "tracing", "utils", "workspace_hack", @@ -5160,7 +5179,7 @@ dependencies = [ "tokio-stream", "tokio-tar", "tokio-util", - "toml_edit", + "toml_edit 0.19.10", "tracing", "tracing-subscriber", "url", @@ -5439,9 +5458,9 @@ dependencies = [ [[package]] name = "serde_spanned" -version = "0.6.2" +version = "0.6.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93107647184f6027e3b7dcb2e11034cf95ffa1e3a682c67951963ac69c1c007d" +checksum = "79e674e01f999af37c49f70a6ede167a8a60b2503e56c5599532a65baa5969a0" dependencies = [ "serde", ] @@ -6326,14 +6345,26 @@ dependencies = [ "serde", "serde_spanned", "toml_datetime", - "toml_edit", + "toml_edit 0.19.10", +] + +[[package]] +name = "toml" +version = "0.8.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f49eb2ab21d2f26bd6db7bf383edc527a7ebaee412d17af4d40fdccd442f335" +dependencies = [ + "serde", + "serde_spanned", + "toml_datetime", + "toml_edit 0.22.14", ] [[package]] name = "toml_datetime" -version = "0.6.2" +version = "0.6.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a76a9312f5ba4c2dec6b9161fdf25d87ad8a09256ccea5a556fef03c706a10f" +checksum = "4badfd56924ae69bcc9039335b2e017639ce3f9b001c393c1b2d1ef846ce2cbf" dependencies = [ "serde", ] @@ -6348,7 +6379,20 @@ dependencies = [ "serde", "serde_spanned", "toml_datetime", - "winnow", + "winnow 0.4.6", +] + +[[package]] +name = "toml_edit" +version = "0.22.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f21c7aaf97f1bd9ca9d4f9e73b0a6c74bd5afef56f2bc931943a6e1c37e04e38" +dependencies = [ + "indexmap 2.0.1", + "serde", + "serde_spanned", + "toml_datetime", + "winnow 0.6.13", ] [[package]] @@ -7331,6 +7375,15 @@ dependencies = [ "memchr", ] +[[package]] +name = "winnow" +version = "0.6.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59b5e5f6c299a3c7890b876a2a587f3115162487e704907d9b6cd29473052ba1" +dependencies = [ + "memchr", +] + [[package]] name = "winreg" version = "0.50.0" @@ -7367,6 +7420,7 @@ dependencies = [ "base64 0.21.1", "base64ct", "bytes", + "camino", "cc", "chrono", "clap", @@ -7419,7 +7473,7 @@ dependencies = [ "tokio-rustls 0.24.0", "tokio-util", "toml_datetime", - "toml_edit", + "toml_edit 0.19.10", "tonic", "tower", "tracing", diff --git a/Dockerfile.build-tools b/Dockerfile.build-tools index 5dd2c13c0e..f85706ef6a 100644 --- a/Dockerfile.build-tools +++ b/Dockerfile.build-tools @@ -73,13 +73,6 @@ RUN curl -fsSL 'https://apt.llvm.org/llvm-snapshot.gpg.key' | apt-key add - \ && bash -c 'for f in /usr/bin/clang*-${LLVM_VERSION} /usr/bin/llvm*-${LLVM_VERSION}; do ln -s "${f}" "${f%-${LLVM_VERSION}}"; done' \ && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* -# PostgreSQL 14 -RUN curl -fsSL 'https://www.postgresql.org/media/keys/ACCC4CF8.asc' | apt-key add - \ - && echo 'deb http://apt.postgresql.org/pub/repos/apt bullseye-pgdg main' > /etc/apt/sources.list.d/pgdg.list \ - && apt update \ - && apt install -y postgresql-client-14 \ - && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* - # AWS CLI RUN curl "https://awscli.amazonaws.com/awscli-exe-linux-$(uname -m).zip" -o "awscliv2.zip" \ && unzip -q awscliv2.zip \ @@ -113,10 +106,10 @@ RUN for package in Capture::Tiny DateTime Devel::Cover Digest::MD5 File::Spec JS && rm -rf ../lcov.tar.gz # Compile and install the static OpenSSL library -ENV OPENSSL_VERSION=3.2.2 +ENV OPENSSL_VERSION=1.1.1w ENV OPENSSL_PREFIX=/usr/local/openssl RUN wget -O /tmp/openssl-${OPENSSL_VERSION}.tar.gz https://www.openssl.org/source/openssl-${OPENSSL_VERSION}.tar.gz && \ - echo "197149c18d9e9f292c43f0400acaba12e5f52cacfe050f3d199277ea738ec2e7 /tmp/openssl-${OPENSSL_VERSION}.tar.gz" | sha256sum --check && \ + echo "cf3098950cb4d853ad95c0841f1f9c6d3dc102dccfcacd521d93925208b76ac8 /tmp/openssl-${OPENSSL_VERSION}.tar.gz" | sha256sum --check && \ cd /tmp && \ tar xzvf /tmp/openssl-${OPENSSL_VERSION}.tar.gz && \ rm /tmp/openssl-${OPENSSL_VERSION}.tar.gz && \ diff --git a/Dockerfile.compute-node b/Dockerfile.compute-node index 3a73ac71b0..7ab685625a 100644 --- a/Dockerfile.compute-node +++ b/Dockerfile.compute-node @@ -467,31 +467,6 @@ RUN case "${PG_VERSION}" in \ make install -j $(getconf _NPROCESSORS_ONLN) && \ echo "trusted = true" >> /usr/local/pgsql/share/extension/pg_hint_plan.control -######################################################################################### -# -# Layer "kq-imcx-pg-build" -# compile kq_imcx extension -# -######################################################################################### -FROM build-deps AS kq-imcx-pg-build -COPY --from=pg-build /usr/local/pgsql/ /usr/local/pgsql/ - -ENV PATH "/usr/local/pgsql/bin/:$PATH" -RUN apt-get update && \ - apt-get install -y git libgtk2.0-dev libpq-dev libpam-dev libxslt-dev libkrb5-dev cmake && \ - wget https://github.com/ketteq-neon/postgres-exts/archive/e0bd1a9d9313d7120c1b9c7bb15c48c0dede4c4e.tar.gz -O kq_imcx.tar.gz && \ - echo "dc93a97ff32d152d32737ba7e196d9687041cda15e58ab31344c2f2de8855336 kq_imcx.tar.gz" | sha256sum --check && \ - mkdir kq_imcx-src && cd kq_imcx-src && tar xzf ../kq_imcx.tar.gz --strip-components=1 -C . && \ - find /usr/local/pgsql -type f | sed 's|^/usr/local/pgsql/||' > /before.txt &&\ - mkdir build && cd build && \ - cmake -DCMAKE_BUILD_TYPE=Release .. && \ - make -j $(getconf _NPROCESSORS_ONLN) && \ - make -j $(getconf _NPROCESSORS_ONLN) install && \ - echo 'trusted = true' >> /usr/local/pgsql/share/extension/kq_imcx.control && \ - find /usr/local/pgsql -type f | sed 's|^/usr/local/pgsql/||' > /after.txt &&\ - mkdir -p /extensions/kq_imcx && cp /usr/local/pgsql/share/extension/kq_imcx.control /extensions/kq_imcx && \ - sort -o /before.txt /before.txt && sort -o /after.txt /after.txt && \ - comm -13 /before.txt /after.txt | tar --directory=/usr/local/pgsql --zstd -cf /extensions/kq_imcx.tar.zst -T - ######################################################################################### # @@ -840,7 +815,6 @@ COPY --from=hll-pg-build /usr/local/pgsql/ /usr/local/pgsql/ COPY --from=plpgsql-check-pg-build /usr/local/pgsql/ /usr/local/pgsql/ COPY --from=timescaledb-pg-build /usr/local/pgsql/ /usr/local/pgsql/ COPY --from=pg-hint-plan-pg-build /usr/local/pgsql/ /usr/local/pgsql/ -COPY --from=kq-imcx-pg-build /usr/local/pgsql/ /usr/local/pgsql/ COPY --from=pg-cron-pg-build /usr/local/pgsql/ /usr/local/pgsql/ COPY --from=pg-pgx-ulid-build /usr/local/pgsql/ /usr/local/pgsql/ COPY --from=rdkit-pg-build /usr/local/pgsql/ /usr/local/pgsql/ @@ -961,7 +935,6 @@ COPY --from=plpgsql-check-pg-build /plpgsql_check.tar.gz /ext-src #COPY --from=timescaledb-pg-build /timescaledb.tar.gz /ext-src COPY --from=pg-hint-plan-pg-build /pg_hint_plan.tar.gz /ext-src COPY patches/pg_hintplan.patch /ext-src -#COPY --from=kq-imcx-pg-build /kq_imcx.tar.gz /ext-src COPY --from=pg-cron-pg-build /pg_cron.tar.gz /ext-src COPY patches/pg_cron.patch /ext-src #COPY --from=pg-pgx-ulid-build /home/nonroot/pgx_ulid.tar.gz /ext-src diff --git a/compute_tools/src/config.rs b/compute_tools/src/config.rs index 2c4aec4116..479100eb89 100644 --- a/compute_tools/src/config.rs +++ b/compute_tools/src/config.rs @@ -83,12 +83,6 @@ pub fn write_postgres_conf( ComputeMode::Replica => { // hot_standby is 'on' by default, but let's be explicit writeln!(file, "hot_standby=on")?; - - // Inform the replica about the primary state - // Default is 'false' - if let Some(primary_is_running) = spec.primary_is_running { - writeln!(file, "neon.primary_is_running={}", primary_is_running)?; - } } } diff --git a/control_plane/src/background_process.rs b/control_plane/src/background_process.rs index 3f4ddbdb2b..a272c306e7 100644 --- a/control_plane/src/background_process.rs +++ b/control_plane/src/background_process.rs @@ -36,11 +36,11 @@ use utils::pid_file::{self, PidFileRead}; // it's waiting. If the process hasn't started/stopped after 5 seconds, // it prints a notice that it's taking long, but keeps waiting. // -const RETRY_UNTIL_SECS: u64 = 10; -const RETRIES: u64 = (RETRY_UNTIL_SECS * 1000) / RETRY_INTERVAL_MILLIS; -const RETRY_INTERVAL_MILLIS: u64 = 100; -const DOT_EVERY_RETRIES: u64 = 10; -const NOTICE_AFTER_RETRIES: u64 = 50; +const STOP_RETRY_TIMEOUT: Duration = Duration::from_secs(10); +const STOP_RETRIES: u128 = STOP_RETRY_TIMEOUT.as_millis() / RETRY_INTERVAL.as_millis(); +const RETRY_INTERVAL: Duration = Duration::from_millis(100); +const DOT_EVERY_RETRIES: u128 = 10; +const NOTICE_AFTER_RETRIES: u128 = 50; /// Argument to `start_process`, to indicate whether it should create pidfile or if the process creates /// it itself. @@ -52,6 +52,7 @@ pub enum InitialPidFile { } /// Start a background child process using the parameters given. +#[allow(clippy::too_many_arguments)] pub async fn start_process( process_name: &str, datadir: &Path, @@ -59,6 +60,7 @@ pub async fn start_process( args: AI, envs: EI, initial_pid_file: InitialPidFile, + retry_timeout: &Duration, process_status_check: F, ) -> anyhow::Result<()> where @@ -69,6 +71,7 @@ where // Not generic AsRef, otherwise empty `envs` prevents type inference EI: IntoIterator, { + let retries: u128 = retry_timeout.as_millis() / RETRY_INTERVAL.as_millis(); if !datadir.metadata().context("stat datadir")?.is_dir() { anyhow::bail!("`datadir` must be a directory when calling this function: {datadir:?}"); } @@ -130,7 +133,7 @@ where .unwrap(); }); - for retries in 0..RETRIES { + for retries in 0..retries { match process_started(pid, pid_file_to_check, &process_status_check).await { Ok(true) => { println!("\n{process_name} started and passed status check, pid: {pid}"); @@ -148,7 +151,7 @@ where print!("."); io::stdout().flush().unwrap(); } - thread::sleep(Duration::from_millis(RETRY_INTERVAL_MILLIS)); + thread::sleep(RETRY_INTERVAL); } Err(e) => { println!("error starting process {process_name:?}: {e:#}"); @@ -157,9 +160,10 @@ where } } println!(); - anyhow::bail!( - "{process_name} did not start+pass status checks within {RETRY_UNTIL_SECS} seconds" - ); + anyhow::bail!(format!( + "{} did not start+pass status checks within {:?} seconds", + process_name, retry_timeout + )); } /// Stops the process, using the pid file given. Returns Ok also if the process is already not running. @@ -215,7 +219,7 @@ pub fn stop_process( } pub fn wait_until_stopped(process_name: &str, pid: Pid) -> anyhow::Result<()> { - for retries in 0..RETRIES { + for retries in 0..STOP_RETRIES { match process_has_stopped(pid) { Ok(true) => { println!("\n{process_name} stopped"); @@ -231,7 +235,7 @@ pub fn wait_until_stopped(process_name: &str, pid: Pid) -> anyhow::Result<()> { print!("."); io::stdout().flush().unwrap(); } - thread::sleep(Duration::from_millis(RETRY_INTERVAL_MILLIS)); + thread::sleep(RETRY_INTERVAL); } Err(e) => { println!("{process_name} with pid {pid} failed to stop: {e:#}"); @@ -240,7 +244,10 @@ pub fn wait_until_stopped(process_name: &str, pid: Pid) -> anyhow::Result<()> { } } println!(); - anyhow::bail!("{process_name} with pid {pid} did not stop in {RETRY_UNTIL_SECS} seconds"); + anyhow::bail!(format!( + "{} with pid {} did not stop in {:?} seconds", + process_name, pid, STOP_RETRY_TIMEOUT + )); } fn fill_rust_env_vars(cmd: &mut Command) -> &mut Command { diff --git a/control_plane/src/bin/neon_local.rs b/control_plane/src/bin/neon_local.rs index 8fe959792b..f381337346 100644 --- a/control_plane/src/bin/neon_local.rs +++ b/control_plane/src/bin/neon_local.rs @@ -36,6 +36,7 @@ use std::collections::{BTreeSet, HashMap}; use std::path::PathBuf; use std::process::exit; use std::str::FromStr; +use std::time::Duration; use storage_broker::DEFAULT_LISTEN_ADDR as DEFAULT_BROKER_ADDR; use url::Host; use utils::{ @@ -99,7 +100,7 @@ fn main() -> Result<()> { let subcommand_result = match sub_name { "tenant" => rt.block_on(handle_tenant(sub_args, &mut env)), "timeline" => rt.block_on(handle_timeline(sub_args, &mut env)), - "start" => rt.block_on(handle_start_all(&env)), + "start" => rt.block_on(handle_start_all(&env, get_start_timeout(sub_args))), "stop" => rt.block_on(handle_stop_all(sub_args, &env)), "pageserver" => rt.block_on(handle_pageserver(sub_args, &env)), "storage_controller" => rt.block_on(handle_storage_controller(sub_args, &env)), @@ -599,13 +600,9 @@ async fn handle_timeline(timeline_match: &ArgMatches, env: &mut local_env::Local Some(("import", import_match)) => { let tenant_id = get_tenant_id(import_match, env)?; let timeline_id = parse_timeline_id(import_match)?.expect("No timeline id provided"); - let name = import_match - .get_one::("node-name") - .ok_or_else(|| anyhow!("No node name provided"))?; - let update_catalog = import_match - .get_one::("update-catalog") - .cloned() - .unwrap_or_default(); + let branch_name = import_match + .get_one::("branch-name") + .ok_or_else(|| anyhow!("No branch name provided"))?; // Parse base inputs let base_tarfile = import_match @@ -632,24 +629,11 @@ async fn handle_timeline(timeline_match: &ArgMatches, env: &mut local_env::Local .copied() .context("Failed to parse postgres version from the argument string")?; - let mut cplane = ComputeControlPlane::load(env.clone())?; println!("Importing timeline into pageserver ..."); pageserver .timeline_import(tenant_id, timeline_id, base, pg_wal, pg_version) .await?; - env.register_branch_mapping(name.to_string(), tenant_id, timeline_id)?; - - println!("Creating endpoint for imported timeline ..."); - cplane.new_endpoint( - name, - tenant_id, - timeline_id, - None, - None, - pg_version, - ComputeMode::Primary, - !update_catalog, - )?; + env.register_branch_mapping(branch_name.to_string(), tenant_id, timeline_id)?; println!("Done"); } Some(("branch", branch_match)) => { @@ -1048,10 +1032,20 @@ fn get_pageserver(env: &local_env::LocalEnv, args: &ArgMatches) -> Result &Duration { + let humantime_duration = args + .get_one::("start-timeout") + .expect("invalid value for start-timeout"); + humantime_duration.as_ref() +} + async fn handle_pageserver(sub_match: &ArgMatches, env: &local_env::LocalEnv) -> Result<()> { match sub_match.subcommand() { Some(("start", subcommand_args)) => { - if let Err(e) = get_pageserver(env, subcommand_args)?.start().await { + if let Err(e) = get_pageserver(env, subcommand_args)? + .start(get_start_timeout(subcommand_args)) + .await + { eprintln!("pageserver start failed: {e}"); exit(1); } @@ -1077,7 +1071,7 @@ async fn handle_pageserver(sub_match: &ArgMatches, env: &local_env::LocalEnv) -> exit(1); } - if let Err(e) = pageserver.start().await { + if let Err(e) = pageserver.start(get_start_timeout(sub_match)).await { eprintln!("pageserver start failed: {e}"); exit(1); } @@ -1105,8 +1099,8 @@ async fn handle_storage_controller( ) -> Result<()> { let svc = StorageController::from_env(env); match sub_match.subcommand() { - Some(("start", _start_match)) => { - if let Err(e) = svc.start().await { + Some(("start", start_match)) => { + if let Err(e) = svc.start(get_start_timeout(start_match)).await { eprintln!("start failed: {e}"); exit(1); } @@ -1165,7 +1159,10 @@ async fn handle_safekeeper(sub_match: &ArgMatches, env: &local_env::LocalEnv) -> "start" => { let extra_opts = safekeeper_extra_opts(sub_args); - if let Err(e) = safekeeper.start(extra_opts).await { + if let Err(e) = safekeeper + .start(extra_opts, get_start_timeout(sub_args)) + .await + { eprintln!("safekeeper start failed: {}", e); exit(1); } @@ -1191,7 +1188,10 @@ async fn handle_safekeeper(sub_match: &ArgMatches, env: &local_env::LocalEnv) -> } let extra_opts = safekeeper_extra_opts(sub_args); - if let Err(e) = safekeeper.start(extra_opts).await { + if let Err(e) = safekeeper + .start(extra_opts, get_start_timeout(sub_args)) + .await + { eprintln!("safekeeper start failed: {}", e); exit(1); } @@ -1204,15 +1204,18 @@ async fn handle_safekeeper(sub_match: &ArgMatches, env: &local_env::LocalEnv) -> Ok(()) } -async fn handle_start_all(env: &local_env::LocalEnv) -> anyhow::Result<()> { +async fn handle_start_all( + env: &local_env::LocalEnv, + retry_timeout: &Duration, +) -> anyhow::Result<()> { // Endpoints are not started automatically - broker::start_broker_process(env).await?; + broker::start_broker_process(env, retry_timeout).await?; // Only start the storage controller if the pageserver is configured to need it if env.control_plane_api.is_some() { let storage_controller = StorageController::from_env(env); - if let Err(e) = storage_controller.start().await { + if let Err(e) = storage_controller.start(retry_timeout).await { eprintln!("storage_controller start failed: {:#}", e); try_stop_all(env, true).await; exit(1); @@ -1221,7 +1224,7 @@ async fn handle_start_all(env: &local_env::LocalEnv) -> anyhow::Result<()> { for ps_conf in &env.pageservers { let pageserver = PageServerNode::from_env(env, ps_conf); - if let Err(e) = pageserver.start().await { + if let Err(e) = pageserver.start(retry_timeout).await { eprintln!("pageserver {} start failed: {:#}", ps_conf.id, e); try_stop_all(env, true).await; exit(1); @@ -1230,7 +1233,7 @@ async fn handle_start_all(env: &local_env::LocalEnv) -> anyhow::Result<()> { for node in env.safekeepers.iter() { let safekeeper = SafekeeperNode::from_env(env, node); - if let Err(e) = safekeeper.start(vec![]).await { + if let Err(e) = safekeeper.start(vec![], retry_timeout).await { eprintln!("safekeeper {} start failed: {:#}", safekeeper.id, e); try_stop_all(env, false).await; exit(1); @@ -1290,6 +1293,15 @@ async fn try_stop_all(env: &local_env::LocalEnv, immediate: bool) { } fn cli() -> Command { + let timeout_arg = Arg::new("start-timeout") + .long("start-timeout") + .short('t') + .global(true) + .help("timeout until we fail the command, e.g. 30s") + .value_parser(value_parser!(humantime::Duration)) + .default_value("10s") + .required(false); + let branch_name_arg = Arg::new("branch-name") .long("branch-name") .help("Name of the branch to be created or used as an alias for other services") @@ -1458,8 +1470,7 @@ fn cli() -> Command { .about("Import timeline from basebackup directory") .arg(tenant_id_arg.clone()) .arg(timeline_id_arg.clone()) - .arg(Arg::new("node-name").long("node-name") - .help("Name to assign to the imported timeline")) + .arg(branch_name_arg.clone()) .arg(Arg::new("base-tarfile") .long("base-tarfile") .value_parser(value_parser!(PathBuf)) @@ -1475,7 +1486,6 @@ fn cli() -> Command { .arg(Arg::new("end-lsn").long("end-lsn") .help("Lsn the basebackup ends at")) .arg(pg_version_arg.clone()) - .arg(update_catalog.clone()) ) ).subcommand( Command::new("tenant") @@ -1509,6 +1519,7 @@ fn cli() -> Command { .subcommand(Command::new("status")) .subcommand(Command::new("start") .about("Start local pageserver") + .arg(timeout_arg.clone()) ) .subcommand(Command::new("stop") .about("Stop local pageserver") @@ -1516,13 +1527,15 @@ fn cli() -> Command { ) .subcommand(Command::new("restart") .about("Restart local pageserver") + .arg(timeout_arg.clone()) ) ) .subcommand( Command::new("storage_controller") .arg_required_else_help(true) .about("Manage storage_controller") - .subcommand(Command::new("start").about("Start storage controller")) + .subcommand(Command::new("start").about("Start storage controller") + .arg(timeout_arg.clone())) .subcommand(Command::new("stop").about("Stop storage controller") .arg(stop_mode_arg.clone())) ) @@ -1534,6 +1547,7 @@ fn cli() -> Command { .about("Start local safekeeper") .arg(safekeeper_id_arg.clone()) .arg(safekeeper_extra_opt_arg.clone()) + .arg(timeout_arg.clone()) ) .subcommand(Command::new("stop") .about("Stop local safekeeper") @@ -1545,6 +1559,7 @@ fn cli() -> Command { .arg(safekeeper_id_arg) .arg(stop_mode_arg.clone()) .arg(safekeeper_extra_opt_arg) + .arg(timeout_arg.clone()) ) ) .subcommand( @@ -1579,6 +1594,7 @@ fn cli() -> Command { .arg(remote_ext_config_args) .arg(create_test_user) .arg(allow_multiple.clone()) + .arg(timeout_arg.clone()) ) .subcommand(Command::new("reconfigure") .about("Reconfigure the endpoint") @@ -1630,6 +1646,7 @@ fn cli() -> Command { .subcommand( Command::new("start") .about("Start page server and safekeepers") + .arg(timeout_arg.clone()) ) .subcommand( Command::new("stop") diff --git a/control_plane/src/broker.rs b/control_plane/src/broker.rs index f40705863b..c3cfc140da 100644 --- a/control_plane/src/broker.rs +++ b/control_plane/src/broker.rs @@ -5,13 +5,18 @@ //! ```text //! .neon/safekeepers/ //! ``` +use std::time::Duration; + use anyhow::Context; use camino::Utf8PathBuf; use crate::{background_process, local_env}; -pub async fn start_broker_process(env: &local_env::LocalEnv) -> anyhow::Result<()> { +pub async fn start_broker_process( + env: &local_env::LocalEnv, + retry_timeout: &Duration, +) -> anyhow::Result<()> { let broker = &env.broker; let listen_addr = &broker.listen_addr; @@ -27,6 +32,7 @@ pub async fn start_broker_process(env: &local_env::LocalEnv) -> anyhow::Result<( args, [], background_process::InitialPidFile::Create(storage_broker_pid_file_path(env)), + retry_timeout, || async { let url = broker.client_url(); let status_url = url.join("status").with_context(|| { diff --git a/control_plane/src/endpoint.rs b/control_plane/src/endpoint.rs index 20371e1cb8..b928bbfc30 100644 --- a/control_plane/src/endpoint.rs +++ b/control_plane/src/endpoint.rs @@ -592,7 +592,6 @@ impl Endpoint { remote_extensions, pgbouncer_settings: None, shard_stripe_size: Some(shard_stripe_size), - primary_is_running: None, }; let spec_path = self.endpoint_path().join("spec.json"); std::fs::write(spec_path, serde_json::to_string_pretty(&spec)?)?; diff --git a/control_plane/src/pageserver.rs b/control_plane/src/pageserver.rs index 13e684da24..da4b987849 100644 --- a/control_plane/src/pageserver.rs +++ b/control_plane/src/pageserver.rs @@ -158,8 +158,8 @@ impl PageServerNode { .expect("non-Unicode path") } - pub async fn start(&self) -> anyhow::Result<()> { - self.start_node().await + pub async fn start(&self, retry_timeout: &Duration) -> anyhow::Result<()> { + self.start_node(retry_timeout).await } fn pageserver_init(&self, conf: NeonLocalInitPageserverConf) -> anyhow::Result<()> { @@ -214,14 +214,15 @@ impl PageServerNode { Ok(()) } - async fn start_node(&self) -> anyhow::Result<()> { + async fn start_node(&self, retry_timeout: &Duration) -> anyhow::Result<()> { // TODO: using a thread here because start_process() is not async but we need to call check_status() let datadir = self.repo_path(); print!( - "Starting pageserver node {} at '{}' in {:?}", + "Starting pageserver node {} at '{}' in {:?}, retrying for {:?}", self.conf.id, self.pg_connection_config.raw_address(), - datadir + datadir, + retry_timeout ); io::stdout().flush().context("flush stdout")?; @@ -239,6 +240,7 @@ impl PageServerNode { args, self.pageserver_env_variables()?, background_process::InitialPidFile::Expect(self.pid_file()), + retry_timeout, || async { let st = self.check_status().await; match st { diff --git a/control_plane/src/safekeeper.rs b/control_plane/src/safekeeper.rs index 4a320ce53d..a0a73f5609 100644 --- a/control_plane/src/safekeeper.rs +++ b/control_plane/src/safekeeper.rs @@ -7,6 +7,7 @@ //! ``` use std::io::Write; use std::path::PathBuf; +use std::time::Duration; use std::{io, result}; use anyhow::Context; @@ -111,11 +112,16 @@ impl SafekeeperNode { .expect("non-Unicode path") } - pub async fn start(&self, extra_opts: Vec) -> anyhow::Result<()> { + pub async fn start( + &self, + extra_opts: Vec, + retry_timeout: &Duration, + ) -> anyhow::Result<()> { print!( - "Starting safekeeper at '{}' in '{}'", + "Starting safekeeper at '{}' in '{}', retrying for {:?}", self.pg_connection_config.raw_address(), - self.datadir_path().display() + self.datadir_path().display(), + retry_timeout, ); io::stdout().flush().unwrap(); @@ -200,6 +206,7 @@ impl SafekeeperNode { &args, self.safekeeper_env_variables()?, background_process::InitialPidFile::Expect(self.pid_file()), + retry_timeout, || async { match self.check_status().await { Ok(()) => Ok(true), diff --git a/control_plane/src/storage_controller.rs b/control_plane/src/storage_controller.rs index 4f9f0ba794..1c56d5f80f 100644 --- a/control_plane/src/storage_controller.rs +++ b/control_plane/src/storage_controller.rs @@ -18,7 +18,7 @@ use pageserver_client::mgmt_api::ResponseErrorMessageExt; use postgres_backend::AuthType; use reqwest::Method; use serde::{de::DeserializeOwned, Deserialize, Serialize}; -use std::{fs, str::FromStr}; +use std::{fs, str::FromStr, time::Duration}; use tokio::process::Command; use tracing::instrument; use url::Url; @@ -224,7 +224,7 @@ impl StorageController { Ok(database_url) } - pub async fn start(&self) -> anyhow::Result<()> { + pub async fn start(&self, retry_timeout: &Duration) -> anyhow::Result<()> { // Start a vanilla Postgres process used by the storage controller for persistence. let pg_data_path = Utf8PathBuf::from_path_buf(self.env.base_data_dir.clone()) .unwrap() @@ -272,6 +272,7 @@ impl StorageController { db_start_args, [], background_process::InitialPidFile::Create(self.postgres_pid_file()), + retry_timeout, || self.pg_isready(&pg_bin_dir), ) .await?; @@ -326,6 +327,7 @@ impl StorageController { args, [], background_process::InitialPidFile::Create(self.pid_file()), + retry_timeout, || async { match self.ready().await { Ok(_) => Ok(true), diff --git a/control_plane/storcon_cli/src/main.rs b/control_plane/storcon_cli/src/main.rs index 7b48b75c21..775aedb600 100644 --- a/control_plane/storcon_cli/src/main.rs +++ b/control_plane/storcon_cli/src/main.rs @@ -1,5 +1,5 @@ use futures::StreamExt; -use std::{collections::HashMap, str::FromStr, time::Duration}; +use std::{str::FromStr, time::Duration}; use clap::{Parser, Subcommand}; use pageserver_api::{ @@ -21,7 +21,7 @@ use utils::id::{NodeId, TenantId}; use pageserver_api::controller_api::{ NodeConfigureRequest, NodeRegisterRequest, NodeSchedulingPolicy, PlacementPolicy, - TenantLocateResponse, TenantShardMigrateRequest, TenantShardMigrateResponse, + TenantShardMigrateRequest, TenantShardMigrateResponse, }; #[derive(Subcommand, Debug)] @@ -110,12 +110,6 @@ enum Command { #[arg(long)] config: String, }, - /// Attempt to balance the locations for a tenant across pageservers. This is a client-side - /// alternative to the storage controller's scheduling optimization behavior. - TenantScatter { - #[arg(long)] - tenant_id: TenantId, - }, /// Print details about a particular tenant, including all its shards' states. TenantDescribe { #[arg(long)] @@ -498,88 +492,6 @@ async fn main() -> anyhow::Result<()> { }) .await?; } - Command::TenantScatter { tenant_id } => { - // Find the shards - let locate_response = storcon_client - .dispatch::<(), TenantLocateResponse>( - Method::GET, - format!("control/v1/tenant/{tenant_id}/locate"), - None, - ) - .await?; - let shards = locate_response.shards; - - let mut node_to_shards: HashMap> = HashMap::new(); - let shard_count = shards.len(); - for s in shards { - let entry = node_to_shards.entry(s.node_id).or_default(); - entry.push(s.shard_id); - } - - // Load list of available nodes - let nodes_resp = storcon_client - .dispatch::<(), Vec>( - Method::GET, - "control/v1/node".to_string(), - None, - ) - .await?; - - for node in nodes_resp { - if matches!(node.availability, NodeAvailabilityWrapper::Active) { - node_to_shards.entry(node.id).or_default(); - } - } - - let max_shard_per_node = shard_count / node_to_shards.len(); - - loop { - let mut migrate_shard = None; - for shards in node_to_shards.values_mut() { - if shards.len() > max_shard_per_node { - // Pick the emptiest - migrate_shard = Some(shards.pop().unwrap()); - } - } - let Some(migrate_shard) = migrate_shard else { - break; - }; - - // Pick the emptiest node to migrate to - let mut destinations = node_to_shards - .iter() - .map(|(k, v)| (k, v.len())) - .collect::>(); - destinations.sort_by_key(|i| i.1); - let (destination_node, destination_count) = *destinations.first().unwrap(); - if destination_count + 1 > max_shard_per_node { - // Even the emptiest destination doesn't have space: we're done - break; - } - let destination_node = *destination_node; - - node_to_shards - .get_mut(&destination_node) - .unwrap() - .push(migrate_shard); - - println!("Migrate {} -> {} ...", migrate_shard, destination_node); - - storcon_client - .dispatch::( - Method::PUT, - format!("control/v1/tenant/{migrate_shard}/migrate"), - Some(TenantShardMigrateRequest { - tenant_shard_id: migrate_shard, - node_id: destination_node, - }), - ) - .await?; - println!("Migrate {} -> {} OK", migrate_shard, destination_node); - } - - // Spread the shards across the nodes - } Command::TenantDescribe { tenant_id } => { let describe_response = storcon_client .dispatch::<(), TenantDescribeResponse>( diff --git a/docker-compose/README.md b/docker-compose/README.md new file mode 100644 index 0000000000..bd47805a67 --- /dev/null +++ b/docker-compose/README.md @@ -0,0 +1,10 @@ + +# Example docker compose configuration + +The configuration in this directory is used for testing Neon docker images: it is +not intended for deploying a usable system. To run a development environment where +you can experiment with a minature Neon system, use `cargo neon` rather than container images. + +This configuration does not start the storage controller, because the controller +needs a way to reconfigure running computes, and no such thing exists in this setup. + diff --git a/docker-compose/compute_wrapper/shell/compute.sh b/docker-compose/compute_wrapper/shell/compute.sh index 22660a63ce..f646e36f59 100755 --- a/docker-compose/compute_wrapper/shell/compute.sh +++ b/docker-compose/compute_wrapper/shell/compute.sh @@ -23,11 +23,10 @@ echo "Page server is ready." echo "Create a tenant and timeline" generate_id tenant_id PARAMS=( - -sb - -X POST + -X PUT -H "Content-Type: application/json" - -d "{\"new_tenant_id\": \"${tenant_id}\"}" - http://pageserver:9898/v1/tenant/ + -d "{\"mode\": \"AttachedSingle\", \"generation\": 1, \"tenant_conf\": {}}" + "http://pageserver:9898/v1/tenant/${tenant_id}/location_config" ) result=$(curl "${PARAMS[@]}") echo $result | jq . diff --git a/docs/pageserver-pagecache.md b/docs/pageserver-pagecache.md index d9b120bbb9..d022742dff 100644 --- a/docs/pageserver-pagecache.md +++ b/docs/pageserver-pagecache.md @@ -5,4 +5,3 @@ TODO: - shared across tenants - store pages from layer files - store pages from "in-memory layer" -- store materialized pages diff --git a/docs/settings.md b/docs/settings.md index 817f97d8ba..12a6a4c171 100644 --- a/docs/settings.md +++ b/docs/settings.md @@ -134,7 +134,7 @@ depends on that, so if you change it, bad things will happen. #### page_cache_size -Size of the page cache, to hold materialized page versions. Unit is +Size of the page cache. Unit is number of 8 kB blocks. The default is 8192, which means 64 MB. #### max_file_descriptors diff --git a/libs/compute_api/src/spec.rs b/libs/compute_api/src/spec.rs index 1c4ee2089f..883c624f71 100644 --- a/libs/compute_api/src/spec.rs +++ b/libs/compute_api/src/spec.rs @@ -96,12 +96,6 @@ pub struct ComputeSpec { // Stripe size for pageserver sharding, in pages #[serde(default)] pub shard_stripe_size: Option, - - // When we are starting a new replica in hot standby mode, - // we need to know if the primary is running. - // This is used to determine if replica should wait for - // RUNNING_XACTS from primary or not. - pub primary_is_running: Option, } /// Feature flag to signal `compute_ctl` to enable certain experimental functionality. diff --git a/libs/pageserver_api/src/key.rs b/libs/pageserver_api/src/key.rs index 997c1cc43a..cd430bfab7 100644 --- a/libs/pageserver_api/src/key.rs +++ b/libs/pageserver_api/src/key.rs @@ -160,8 +160,9 @@ impl Key { key } - /// Convert a 18B slice to a key. This function should not be used for metadata keys because field2 is handled differently. - /// Use [`Key::from_i128`] instead if you want to handle 16B keys (i.e., metadata keys). + /// Convert a 18B slice to a key. This function should not be used for 16B metadata keys because `field2` is handled differently. + /// Use [`Key::from_i128`] instead if you want to handle 16B keys (i.e., metadata keys). There are some restrictions on `field2`, + /// and therefore not all 18B slices are valid page server keys. pub fn from_slice(b: &[u8]) -> Self { Key { field1: b[0], @@ -173,7 +174,7 @@ impl Key { } } - /// Convert a key to a 18B slice. This function should not be used for metadata keys because field2 is handled differently. + /// Convert a key to a 18B slice. This function should not be used for getting a 16B metadata key because `field2` is handled differently. /// Use [`Key::to_i128`] instead if you want to get a 16B key (i.e., metadata keys). pub fn write_to_byte_slice(&self, buf: &mut [u8]) { buf[0] = self.field1; diff --git a/libs/pageserver_api/src/models.rs b/libs/pageserver_api/src/models.rs index 70db0b7344..4875f49495 100644 --- a/libs/pageserver_api/src/models.rs +++ b/libs/pageserver_api/src/models.rs @@ -293,22 +293,6 @@ pub struct TenantCreateRequest { pub config: TenantConfig, // as we have a flattened field, we should reject all unknown fields in it } -#[derive(Deserialize, Debug)] -#[serde(deny_unknown_fields)] -pub struct TenantLoadRequest { - #[serde(default)] - #[serde(skip_serializing_if = "Option::is_none")] - pub generation: Option, -} - -impl std::ops::Deref for TenantCreateRequest { - type Target = TenantConfig; - - fn deref(&self) -> &Self::Target { - &self.config - } -} - /// An alternative representation of `pageserver::tenant::TenantConf` with /// simpler types. #[derive(Serialize, Deserialize, Debug, Default, Clone, Eq, PartialEq)] @@ -623,31 +607,6 @@ impl TenantConfigRequest { } } -#[derive(Debug, Deserialize)] -pub struct TenantAttachRequest { - #[serde(default)] - pub config: TenantAttachConfig, - #[serde(default)] - pub generation: Option, -} - -/// Newtype to enforce deny_unknown_fields on TenantConfig for -/// its usage inside `TenantAttachRequest`. -#[derive(Debug, Serialize, Deserialize, Default)] -#[serde(deny_unknown_fields)] -pub struct TenantAttachConfig { - #[serde(flatten)] - allowing_unknown_fields: TenantConfig, -} - -impl std::ops::Deref for TenantAttachConfig { - type Target = TenantConfig; - - fn deref(&self) -> &Self::Target { - &self.allowing_unknown_fields - } -} - /// See [`TenantState::attachment_status`] and the OpenAPI docs for context. #[derive(Serialize, Deserialize, Clone)] #[serde(tag = "slug", content = "data", rename_all = "snake_case")] @@ -666,8 +625,7 @@ pub struct TenantInfo { /// If a layer is present in both local FS and S3, it counts only once. pub current_physical_size: Option, // physical size is only included in `tenant_status` endpoint pub attachment_status: TenantAttachmentStatus, - #[serde(skip_serializing_if = "Option::is_none")] - pub generation: Option, + pub generation: u32, } #[derive(Serialize, Deserialize, Clone)] @@ -1494,7 +1452,7 @@ mod tests { state: TenantState::Active, current_physical_size: Some(42), attachment_status: TenantAttachmentStatus::Attached, - generation: None, + generation: 1, }; let expected_active = json!({ "id": original_active.id.to_string(), @@ -1504,7 +1462,8 @@ mod tests { "current_physical_size": 42, "attachment_status": { "slug":"attached", - } + }, + "generation" : 1 }); let original_broken = TenantInfo { @@ -1515,7 +1474,7 @@ mod tests { }, current_physical_size: Some(42), attachment_status: TenantAttachmentStatus::Attached, - generation: None, + generation: 1, }; let expected_broken = json!({ "id": original_broken.id.to_string(), @@ -1529,7 +1488,8 @@ mod tests { "current_physical_size": 42, "attachment_status": { "slug":"attached", - } + }, + "generation" : 1 }); assert_eq!( @@ -1570,18 +1530,6 @@ mod tests { "expect unknown field `unknown_field` error, got: {}", err ); - - let attach_request = json!({ - "config": { - "unknown_field": "unknown_value".to_string(), - }, - }); - let err = serde_json::from_value::(attach_request).unwrap_err(); - assert!( - err.to_string().contains("unknown field `unknown_field`"), - "expect unknown field `unknown_field` error, got: {}", - err - ); } #[test] diff --git a/libs/remote_storage/Cargo.toml b/libs/remote_storage/Cargo.toml index 78da01c9a0..23d82b90bd 100644 --- a/libs/remote_storage/Cargo.toml +++ b/libs/remote_storage/Cargo.toml @@ -14,8 +14,9 @@ aws-config.workspace = true aws-sdk-s3.workspace = true aws-credential-types.workspace = true bytes.workspace = true -camino.workspace = true +camino = { workspace = true, features = ["serde1"] } humantime.workspace = true +humantime-serde.workspace = true hyper = { workspace = true, features = ["stream"] } futures.workspace = true rand.workspace = true diff --git a/libs/remote_storage/src/azure_blob.rs b/libs/remote_storage/src/azure_blob.rs index dbd64fb5a6..8e590b17c4 100644 --- a/libs/remote_storage/src/azure_blob.rs +++ b/libs/remote_storage/src/azure_blob.rs @@ -34,7 +34,7 @@ use utils::backoff; use crate::metrics::{start_measuring_requests, AttemptOutcome, RequestKind}; use crate::{ - error::Cancelled, AzureConfig, ConcurrencyLimiter, Download, DownloadError, Listing, + config::AzureConfig, error::Cancelled, ConcurrencyLimiter, Download, DownloadError, Listing, ListingMode, RemotePath, RemoteStorage, StorageMetadata, TimeTravelError, TimeoutOrCancel, }; diff --git a/libs/remote_storage/src/config.rs b/libs/remote_storage/src/config.rs new file mode 100644 index 0000000000..8a8f6212e9 --- /dev/null +++ b/libs/remote_storage/src/config.rs @@ -0,0 +1,277 @@ +use std::{fmt::Debug, num::NonZeroUsize, str::FromStr, time::Duration}; + +use anyhow::bail; +use aws_sdk_s3::types::StorageClass; +use camino::Utf8PathBuf; + +use serde::{Deserialize, Serialize}; + +use crate::{ + DEFAULT_MAX_KEYS_PER_LIST_RESPONSE, DEFAULT_REMOTE_STORAGE_AZURE_CONCURRENCY_LIMIT, + DEFAULT_REMOTE_STORAGE_S3_CONCURRENCY_LIMIT, +}; + +/// External backup storage configuration, enough for creating a client for that storage. +#[derive(Debug, Clone, PartialEq, Eq, Deserialize, Serialize)] +pub struct RemoteStorageConfig { + /// The storage connection configuration. + #[serde(flatten)] + pub storage: RemoteStorageKind, + /// A common timeout enforced for all requests after concurrency limiter permit has been + /// acquired. + #[serde( + with = "humantime_serde", + default = "default_timeout", + skip_serializing_if = "is_default_timeout" + )] + pub timeout: Duration, +} + +fn default_timeout() -> Duration { + RemoteStorageConfig::DEFAULT_TIMEOUT +} + +fn is_default_timeout(d: &Duration) -> bool { + *d == RemoteStorageConfig::DEFAULT_TIMEOUT +} + +/// A kind of a remote storage to connect to, with its connection configuration. +#[derive(Debug, Clone, PartialEq, Eq, Deserialize, Serialize)] +#[serde(untagged)] +pub enum RemoteStorageKind { + /// Storage based on local file system. + /// Specify a root folder to place all stored files into. + LocalFs { local_path: Utf8PathBuf }, + /// AWS S3 based storage, storing all files in the S3 bucket + /// specified by the config + AwsS3(S3Config), + /// Azure Blob based storage, storing all files in the container + /// specified by the config + AzureContainer(AzureConfig), +} + +/// AWS S3 bucket coordinates and access credentials to manage the bucket contents (read and write). +#[derive(Clone, PartialEq, Eq, Deserialize, Serialize)] +pub struct S3Config { + /// Name of the bucket to connect to. + pub bucket_name: String, + /// The region where the bucket is located at. + pub bucket_region: String, + /// A "subfolder" in the bucket, to use the same bucket separately by multiple remote storage users at once. + pub prefix_in_bucket: Option, + /// A base URL to send S3 requests to. + /// By default, the endpoint is derived from a region name, assuming it's + /// an AWS S3 region name, erroring on wrong region name. + /// Endpoint provides a way to support other S3 flavors and their regions. + /// + /// Example: `http://127.0.0.1:5000` + pub endpoint: Option, + /// AWS S3 has various limits on its API calls, we need not to exceed those. + /// See [`DEFAULT_REMOTE_STORAGE_S3_CONCURRENCY_LIMIT`] for more details. + #[serde(default = "default_remote_storage_s3_concurrency_limit")] + pub concurrency_limit: NonZeroUsize, + #[serde(default = "default_max_keys_per_list_response")] + pub max_keys_per_list_response: Option, + #[serde( + deserialize_with = "deserialize_storage_class", + serialize_with = "serialize_storage_class", + default + )] + pub upload_storage_class: Option, +} + +fn default_remote_storage_s3_concurrency_limit() -> NonZeroUsize { + DEFAULT_REMOTE_STORAGE_S3_CONCURRENCY_LIMIT + .try_into() + .unwrap() +} + +fn default_max_keys_per_list_response() -> Option { + DEFAULT_MAX_KEYS_PER_LIST_RESPONSE +} + +impl Debug for S3Config { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("S3Config") + .field("bucket_name", &self.bucket_name) + .field("bucket_region", &self.bucket_region) + .field("prefix_in_bucket", &self.prefix_in_bucket) + .field("concurrency_limit", &self.concurrency_limit) + .field( + "max_keys_per_list_response", + &self.max_keys_per_list_response, + ) + .finish() + } +} + +/// Azure bucket coordinates and access credentials to manage the bucket contents (read and write). +#[derive(Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct AzureConfig { + /// Name of the container to connect to. + pub container_name: String, + /// Name of the storage account the container is inside of + pub storage_account: Option, + /// The region where the bucket is located at. + pub container_region: String, + /// A "subfolder" in the container, to use the same container separately by multiple remote storage users at once. + pub prefix_in_container: Option, + /// Azure has various limits on its API calls, we need not to exceed those. + /// See [`DEFAULT_REMOTE_STORAGE_AZURE_CONCURRENCY_LIMIT`] for more details. + #[serde(default = "default_remote_storage_azure_concurrency_limit")] + pub concurrency_limit: NonZeroUsize, + #[serde(default = "default_max_keys_per_list_response")] + pub max_keys_per_list_response: Option, +} + +fn default_remote_storage_azure_concurrency_limit() -> NonZeroUsize { + NonZeroUsize::new(DEFAULT_REMOTE_STORAGE_AZURE_CONCURRENCY_LIMIT).unwrap() +} + +impl Debug for AzureConfig { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("AzureConfig") + .field("bucket_name", &self.container_name) + .field("storage_account", &self.storage_account) + .field("bucket_region", &self.container_region) + .field("prefix_in_container", &self.prefix_in_container) + .field("concurrency_limit", &self.concurrency_limit) + .field( + "max_keys_per_list_response", + &self.max_keys_per_list_response, + ) + .finish() + } +} + +fn deserialize_storage_class<'de, D: serde::Deserializer<'de>>( + deserializer: D, +) -> Result, D::Error> { + Option::::deserialize(deserializer).and_then(|s| { + if let Some(s) = s { + use serde::de::Error; + let storage_class = StorageClass::from_str(&s).expect("infallible"); + #[allow(deprecated)] + if matches!(storage_class, StorageClass::Unknown(_)) { + return Err(D::Error::custom(format!( + "Specified storage class unknown to SDK: '{s}'. Allowed values: {:?}", + StorageClass::values() + ))); + } + Ok(Some(storage_class)) + } else { + Ok(None) + } + }) +} + +fn serialize_storage_class( + val: &Option, + serializer: S, +) -> Result { + let val = val.as_ref().map(StorageClass::as_str); + Option::<&str>::serialize(&val, serializer) +} + +impl RemoteStorageConfig { + pub const DEFAULT_TIMEOUT: Duration = std::time::Duration::from_secs(120); + + pub fn from_toml(toml: &toml_edit::Item) -> anyhow::Result> { + let document: toml_edit::Document = match toml { + toml_edit::Item::Table(toml) => toml.clone().into(), + toml_edit::Item::Value(toml_edit::Value::InlineTable(toml)) => { + toml.clone().into_table().into() + } + _ => bail!("toml not a table or inline table"), + }; + + if document.is_empty() { + return Ok(None); + } + + Ok(Some(toml_edit::de::from_document(document)?)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn parse(input: &str) -> anyhow::Result> { + let toml = input.parse::().unwrap(); + RemoteStorageConfig::from_toml(toml.as_item()) + } + + #[test] + fn parse_localfs_config_with_timeout() { + let input = "local_path = '.' +timeout = '5s'"; + + let config = parse(input).unwrap().expect("it exists"); + + assert_eq!( + config, + RemoteStorageConfig { + storage: RemoteStorageKind::LocalFs { + local_path: Utf8PathBuf::from(".") + }, + timeout: Duration::from_secs(5) + } + ); + } + + #[test] + fn test_s3_parsing() { + let toml = "\ + bucket_name = 'foo-bar' + bucket_region = 'eu-central-1' + upload_storage_class = 'INTELLIGENT_TIERING' + timeout = '7s' + "; + + let config = parse(toml).unwrap().expect("it exists"); + + assert_eq!( + config, + RemoteStorageConfig { + storage: RemoteStorageKind::AwsS3(S3Config { + bucket_name: "foo-bar".into(), + bucket_region: "eu-central-1".into(), + prefix_in_bucket: None, + endpoint: None, + concurrency_limit: default_remote_storage_s3_concurrency_limit(), + max_keys_per_list_response: DEFAULT_MAX_KEYS_PER_LIST_RESPONSE, + upload_storage_class: Some(StorageClass::IntelligentTiering), + }), + timeout: Duration::from_secs(7) + } + ); + } + + #[test] + fn test_azure_parsing() { + let toml = "\ + container_name = 'foo-bar' + container_region = 'westeurope' + upload_storage_class = 'INTELLIGENT_TIERING' + timeout = '7s' + "; + + let config = parse(toml).unwrap().expect("it exists"); + + assert_eq!( + config, + RemoteStorageConfig { + storage: RemoteStorageKind::AzureContainer(AzureConfig { + container_name: "foo-bar".into(), + storage_account: None, + container_region: "westeurope".into(), + prefix_in_container: None, + concurrency_limit: default_remote_storage_azure_concurrency_limit(), + max_keys_per_list_response: DEFAULT_MAX_KEYS_PER_LIST_RESPONSE, + }), + timeout: Duration::from_secs(7) + } + ); + } +} diff --git a/libs/remote_storage/src/lib.rs b/libs/remote_storage/src/lib.rs index 72748e156c..d440c03a0e 100644 --- a/libs/remote_storage/src/lib.rs +++ b/libs/remote_storage/src/lib.rs @@ -10,6 +10,7 @@ #![deny(clippy::undocumented_unsafe_blocks)] mod azure_blob; +mod config; mod error; mod local_fs; mod metrics; @@ -18,17 +19,10 @@ mod simulate_failures; mod support; use std::{ - collections::HashMap, - fmt::Debug, - num::{NonZeroU32, NonZeroUsize}, - pin::Pin, - str::FromStr, - sync::Arc, - time::{Duration, SystemTime}, + collections::HashMap, fmt::Debug, num::NonZeroU32, pin::Pin, sync::Arc, time::SystemTime, }; -use anyhow::{bail, Context}; -use aws_sdk_s3::types::StorageClass; +use anyhow::Context; use camino::{Utf8Path, Utf8PathBuf}; use bytes::Bytes; @@ -36,7 +30,6 @@ use futures::stream::Stream; use serde::{Deserialize, Serialize}; use tokio::sync::Semaphore; use tokio_util::sync::CancellationToken; -use toml_edit::Item; use tracing::info; pub use self::{ @@ -45,6 +38,8 @@ pub use self::{ }; use s3_bucket::RequestKind; +pub use crate::config::{AzureConfig, RemoteStorageConfig, RemoteStorageKind, S3Config}; + /// Azure SDK's ETag type is a simple String wrapper: we use this internally instead of repeating it here. pub use azure_core::Etag; @@ -451,7 +446,7 @@ impl GenericRemoteStorage { pub fn from_config(storage_config: &RemoteStorageConfig) -> anyhow::Result { let timeout = storage_config.timeout; Ok(match &storage_config.storage { - RemoteStorageKind::LocalFs(path) => { + RemoteStorageKind::LocalFs { local_path: path } => { info!("Using fs root '{path}' as a remote storage"); Self::LocalFs(LocalFs::new(path.clone(), timeout)?) } @@ -526,262 +521,6 @@ impl From<[(&str, &str); N]> for StorageMetadata { } } -/// External backup storage configuration, enough for creating a client for that storage. -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct RemoteStorageConfig { - /// The storage connection configuration. - pub storage: RemoteStorageKind, - /// A common timeout enforced for all requests after concurrency limiter permit has been - /// acquired. - pub timeout: Duration, -} - -/// A kind of a remote storage to connect to, with its connection configuration. -#[derive(Debug, Clone, PartialEq, Eq)] -pub enum RemoteStorageKind { - /// Storage based on local file system. - /// Specify a root folder to place all stored files into. - LocalFs(Utf8PathBuf), - /// AWS S3 based storage, storing all files in the S3 bucket - /// specified by the config - AwsS3(S3Config), - /// Azure Blob based storage, storing all files in the container - /// specified by the config - AzureContainer(AzureConfig), -} - -/// AWS S3 bucket coordinates and access credentials to manage the bucket contents (read and write). -#[derive(Clone, PartialEq, Eq)] -pub struct S3Config { - /// Name of the bucket to connect to. - pub bucket_name: String, - /// The region where the bucket is located at. - pub bucket_region: String, - /// A "subfolder" in the bucket, to use the same bucket separately by multiple remote storage users at once. - pub prefix_in_bucket: Option, - /// A base URL to send S3 requests to. - /// By default, the endpoint is derived from a region name, assuming it's - /// an AWS S3 region name, erroring on wrong region name. - /// Endpoint provides a way to support other S3 flavors and their regions. - /// - /// Example: `http://127.0.0.1:5000` - pub endpoint: Option, - /// AWS S3 has various limits on its API calls, we need not to exceed those. - /// See [`DEFAULT_REMOTE_STORAGE_S3_CONCURRENCY_LIMIT`] for more details. - pub concurrency_limit: NonZeroUsize, - pub max_keys_per_list_response: Option, - pub upload_storage_class: Option, -} - -impl Debug for S3Config { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("S3Config") - .field("bucket_name", &self.bucket_name) - .field("bucket_region", &self.bucket_region) - .field("prefix_in_bucket", &self.prefix_in_bucket) - .field("concurrency_limit", &self.concurrency_limit) - .field( - "max_keys_per_list_response", - &self.max_keys_per_list_response, - ) - .finish() - } -} - -/// Azure bucket coordinates and access credentials to manage the bucket contents (read and write). -#[derive(Clone, PartialEq, Eq)] -pub struct AzureConfig { - /// Name of the container to connect to. - pub container_name: String, - /// Name of the storage account the container is inside of - pub storage_account: Option, - /// The region where the bucket is located at. - pub container_region: String, - /// A "subfolder" in the container, to use the same container separately by multiple remote storage users at once. - pub prefix_in_container: Option, - /// Azure has various limits on its API calls, we need not to exceed those. - /// See [`DEFAULT_REMOTE_STORAGE_AZURE_CONCURRENCY_LIMIT`] for more details. - pub concurrency_limit: NonZeroUsize, - pub max_keys_per_list_response: Option, -} - -impl Debug for AzureConfig { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("AzureConfig") - .field("bucket_name", &self.container_name) - .field("storage_account", &self.storage_account) - .field("bucket_region", &self.container_region) - .field("prefix_in_container", &self.prefix_in_container) - .field("concurrency_limit", &self.concurrency_limit) - .field( - "max_keys_per_list_response", - &self.max_keys_per_list_response, - ) - .finish() - } -} - -impl RemoteStorageConfig { - pub const DEFAULT_TIMEOUT: Duration = std::time::Duration::from_secs(120); - - pub fn from_toml(toml: &toml_edit::Item) -> anyhow::Result> { - let local_path = toml.get("local_path"); - let bucket_name = toml.get("bucket_name"); - let bucket_region = toml.get("bucket_region"); - let container_name = toml.get("container_name"); - let container_region = toml.get("container_region"); - - let use_azure = container_name.is_some() && container_region.is_some(); - - let default_concurrency_limit = if use_azure { - DEFAULT_REMOTE_STORAGE_AZURE_CONCURRENCY_LIMIT - } else { - DEFAULT_REMOTE_STORAGE_S3_CONCURRENCY_LIMIT - }; - let concurrency_limit = NonZeroUsize::new( - parse_optional_integer("concurrency_limit", toml)?.unwrap_or(default_concurrency_limit), - ) - .context("Failed to parse 'concurrency_limit' as a positive integer")?; - - let max_keys_per_list_response = - parse_optional_integer::("max_keys_per_list_response", toml) - .context("Failed to parse 'max_keys_per_list_response' as a positive integer")? - .or(DEFAULT_MAX_KEYS_PER_LIST_RESPONSE); - - let endpoint = toml - .get("endpoint") - .map(|endpoint| parse_toml_string("endpoint", endpoint)) - .transpose()?; - - let timeout = toml - .get("timeout") - .map(|timeout| { - timeout - .as_str() - .ok_or_else(|| anyhow::Error::msg("timeout was not a string")) - }) - .transpose() - .and_then(|timeout| { - timeout - .map(humantime::parse_duration) - .transpose() - .map_err(anyhow::Error::new) - }) - .context("parse timeout")? - .unwrap_or(Self::DEFAULT_TIMEOUT); - - if timeout < Duration::from_secs(1) { - bail!("timeout was specified as {timeout:?} which is too low"); - } - - let storage = match ( - local_path, - bucket_name, - bucket_region, - container_name, - container_region, - ) { - // no 'local_path' nor 'bucket_name' options are provided, consider this remote storage disabled - (None, None, None, None, None) => return Ok(None), - (_, Some(_), None, ..) => { - bail!("'bucket_region' option is mandatory if 'bucket_name' is given ") - } - (_, None, Some(_), ..) => { - bail!("'bucket_name' option is mandatory if 'bucket_region' is given ") - } - (None, Some(bucket_name), Some(bucket_region), ..) => { - RemoteStorageKind::AwsS3(S3Config { - bucket_name: parse_toml_string("bucket_name", bucket_name)?, - bucket_region: parse_toml_string("bucket_region", bucket_region)?, - prefix_in_bucket: toml - .get("prefix_in_bucket") - .map(|prefix_in_bucket| { - parse_toml_string("prefix_in_bucket", prefix_in_bucket) - }) - .transpose()?, - endpoint, - concurrency_limit, - max_keys_per_list_response, - upload_storage_class: toml - .get("upload_storage_class") - .map(|prefix_in_bucket| -> anyhow::Result<_> { - let s = parse_toml_string("upload_storage_class", prefix_in_bucket)?; - let storage_class = StorageClass::from_str(&s).expect("infallible"); - #[allow(deprecated)] - if matches!(storage_class, StorageClass::Unknown(_)) { - bail!("Specified storage class unknown to SDK: '{s}'. Allowed values: {:?}", StorageClass::values()); - } - Ok(storage_class) - }) - .transpose()?, - }) - } - (_, _, _, Some(_), None) => { - bail!("'container_name' option is mandatory if 'container_region' is given ") - } - (_, _, _, None, Some(_)) => { - bail!("'container_name' option is mandatory if 'container_region' is given ") - } - (None, None, None, Some(container_name), Some(container_region)) => { - RemoteStorageKind::AzureContainer(AzureConfig { - container_name: parse_toml_string("container_name", container_name)?, - storage_account: toml - .get("storage_account") - .map(|storage_account| { - parse_toml_string("storage_account", storage_account) - }) - .transpose()?, - container_region: parse_toml_string("container_region", container_region)?, - prefix_in_container: toml - .get("prefix_in_container") - .map(|prefix_in_container| { - parse_toml_string("prefix_in_container", prefix_in_container) - }) - .transpose()?, - concurrency_limit, - max_keys_per_list_response, - }) - } - (Some(local_path), None, None, None, None) => RemoteStorageKind::LocalFs( - Utf8PathBuf::from(parse_toml_string("local_path", local_path)?), - ), - (Some(_), Some(_), ..) => { - bail!("'local_path' and 'bucket_name' are mutually exclusive") - } - (Some(_), _, _, Some(_), Some(_)) => { - bail!("local_path and 'container_name' are mutually exclusive") - } - }; - - Ok(Some(RemoteStorageConfig { storage, timeout })) - } -} - -// Helper functions to parse a toml Item -fn parse_optional_integer(name: &str, item: &toml_edit::Item) -> anyhow::Result> -where - I: TryFrom, - E: std::error::Error + Send + Sync + 'static, -{ - let toml_integer = match item.get(name) { - Some(item) => item - .as_integer() - .with_context(|| format!("configure option {name} is not an integer"))?, - None => return Ok(None), - }; - - I::try_from(toml_integer) - .map(Some) - .with_context(|| format!("configure option {name} is too large")) -} - -fn parse_toml_string(name: &str, item: &Item) -> anyhow::Result { - let s = item - .as_str() - .with_context(|| format!("configure option {name} is not a string"))?; - Ok(s.to_string()) -} - struct ConcurrencyLimiter { // Every request to S3 can be throttled or cancelled, if a certain number of requests per second is exceeded. // Same goes to IAM, which is queried before every S3 request, if enabled. IAM has even lower RPS threshold. @@ -849,24 +588,4 @@ mod tests { let err = RemotePath::new(Utf8Path::new("/")).expect_err("Should fail on absolute paths"); assert_eq!(err.to_string(), "Path \"/\" is not relative"); } - - #[test] - fn parse_localfs_config_with_timeout() { - let input = "local_path = '.' -timeout = '5s'"; - - let toml = input.parse::().unwrap(); - - let config = RemoteStorageConfig::from_toml(toml.as_item()) - .unwrap() - .expect("it exists"); - - assert_eq!( - config, - RemoteStorageConfig { - storage: RemoteStorageKind::LocalFs(Utf8PathBuf::from(".")), - timeout: Duration::from_secs(5) - } - ); - } } diff --git a/libs/remote_storage/src/s3_bucket.rs b/libs/remote_storage/src/s3_bucket.rs index 76cf3eac80..ef1bd2c047 100644 --- a/libs/remote_storage/src/s3_bucket.rs +++ b/libs/remote_storage/src/s3_bucket.rs @@ -46,12 +46,12 @@ use utils::backoff; use super::StorageMetadata; use crate::{ + config::S3Config, error::Cancelled, metrics::{start_counting_cancelled_wait, start_measuring_requests}, support::PermitCarrying, ConcurrencyLimiter, Download, DownloadError, Listing, ListingMode, RemotePath, RemoteStorage, - S3Config, TimeTravelError, TimeoutOrCancel, MAX_KEYS_PER_DELETE, - REMOTE_STORAGE_PREFIX_SEPARATOR, + TimeTravelError, TimeoutOrCancel, MAX_KEYS_PER_DELETE, REMOTE_STORAGE_PREFIX_SEPARATOR, }; use crate::metrics::AttemptOutcome; diff --git a/libs/utils/src/generation.rs b/libs/utils/src/generation.rs index b703e883de..5970836033 100644 --- a/libs/utils/src/generation.rs +++ b/libs/utils/src/generation.rs @@ -9,20 +9,11 @@ use serde::{Deserialize, Serialize}; /// numbers are used. #[derive(Copy, Clone, Eq, PartialEq, PartialOrd, Ord, Hash)] pub enum Generation { - // Generations with this magic value will not add a suffix to S3 keys, and will not - // be included in persisted index_part.json. This value is only to be used - // during migration from pre-generation metadata to generation-aware metadata, - // and should eventually go away. - // - // A special Generation is used rather than always wrapping Generation in an Option, - // so that code handling generations doesn't have to be aware of the legacy - // case everywhere it touches a generation. + // The None Generation is used in the metadata of layers written before generations were + // introduced. A running Tenant always has a valid generation, but the layer metadata may + // include None generations. None, - // Generations with this magic value may never be used to construct S3 keys: - // we will panic if someone tries to. This is for Tenants in the "Broken" state, - // so that we can satisfy their constructor with a Generation without risking - // a code bug using it in an S3 write (broken tenants should never write) - Broken, + Valid(u32), } @@ -42,11 +33,6 @@ impl Generation { Self::None } - // Create a new generation that will panic if you try to use get_suffix - pub fn broken() -> Self { - Self::Broken - } - pub const fn new(v: u32) -> Self { Self::Valid(v) } @@ -60,9 +46,6 @@ impl Generation { match self { Self::Valid(v) => GenerationFileSuffix(Some(*v)), Self::None => GenerationFileSuffix(None), - Self::Broken => { - panic!("Tried to use a broken generation"); - } } } @@ -86,7 +69,6 @@ impl Generation { } } Self::None => Self::None, - Self::Broken => panic!("Attempted to use a broken generation"), } } @@ -95,7 +77,6 @@ impl Generation { match self { Self::Valid(n) => Self::Valid(*n + 1), Self::None => Self::Valid(1), - Self::Broken => panic!("Attempted to use a broken generation"), } } @@ -128,7 +109,7 @@ impl Serialize for Generation { if let Self::Valid(v) = self { v.serialize(serializer) } else { - // We should never be asked to serialize a None or Broken. Structures + // We should never be asked to serialize a None. Structures // that include an optional generation should convert None to an // Option::None Err(serde::ser::Error::custom( @@ -159,9 +140,6 @@ impl Debug for Generation { Self::None => { write!(f, "") } - Self::Broken => { - write!(f, "") - } } } } diff --git a/libs/utils/src/http/json.rs b/libs/utils/src/http/json.rs index 7ca62561fe..6c25440b42 100644 --- a/libs/utils/src/http/json.rs +++ b/libs/utils/src/http/json.rs @@ -8,22 +8,15 @@ use super::error::ApiError; pub async fn json_request Deserialize<'de>>( request: &mut Request, ) -> Result { - json_request_or_empty_body(request) - .await? - .context("missing request body") - .map_err(ApiError::BadRequest) -} - -/// Will be removed as part of -pub async fn json_request_or_empty_body Deserialize<'de>>( - request: &mut Request, -) -> Result, ApiError> { let body = hyper::body::aggregate(request.body_mut()) .await .context("Failed to read request body") .map_err(ApiError::BadRequest)?; + if body.remaining() == 0 { - return Ok(None); + return Err(ApiError::BadRequest(anyhow::anyhow!( + "missing request body" + ))); } let mut deser = serde_json::de::Deserializer::from_reader(body.reader()); @@ -31,7 +24,6 @@ pub async fn json_request_or_empty_body Deserialize<'de>>( serde_path_to_error::deserialize(&mut deser) // intentionally stringify because the debug version is not helpful in python logs .map_err(|e| anyhow::anyhow!("Failed to parse json request: {e}")) - .map(Some) .map_err(ApiError::BadRequest) } diff --git a/libs/walproposer/src/walproposer.rs b/libs/walproposer/src/walproposer.rs index f7b72b205f..37b1e0fa87 100644 --- a/libs/walproposer/src/walproposer.rs +++ b/libs/walproposer/src/walproposer.rs @@ -1,3 +1,5 @@ +#![allow(clippy::todo)] + use std::ffi::CString; use crate::{ diff --git a/pageserver/ctl/src/draw_timeline_dir.rs b/pageserver/ctl/src/draw_timeline_dir.rs index 389519c65a..bc939f9688 100644 --- a/pageserver/ctl/src/draw_timeline_dir.rs +++ b/pageserver/ctl/src/draw_timeline_dir.rs @@ -83,10 +83,18 @@ fn parse_filename(name: &str) -> (Range, Range) { let keys: Vec<&str> = split[0].split('-').collect(); let mut lsns: Vec<&str> = split[1].split('-').collect(); + // The current format of the layer file name: 000000067F0000000400000B150100000000-000000067F0000000400000D350100000000__00000000014B7AC8-v1-00000001 + + // Handle generation number `-00000001` part if lsns.last().expect("should").len() == 8 { lsns.pop(); } + // Handle version number `-v1` part + if lsns.last().expect("should").starts_with('v') { + lsns.pop(); + } + if lsns.len() == 1 { lsns.push(lsns[0]); } diff --git a/pageserver/src/config.rs b/pageserver/src/config.rs index b4a0d1ac02..f36e63f035 100644 --- a/pageserver/src/config.rs +++ b/pageserver/src/config.rs @@ -33,15 +33,10 @@ use utils::{ use crate::tenant::timeline::GetVectoredImpl; use crate::tenant::vectored_blob_io::MaxVectoredReadBytes; use crate::tenant::{config::TenantConfOpt, timeline::GetImpl}; -use crate::tenant::{ - TENANTS_SEGMENT_NAME, TENANT_DELETED_MARKER_FILE_NAME, TIMELINES_SEGMENT_NAME, -}; +use crate::tenant::{TENANTS_SEGMENT_NAME, TIMELINES_SEGMENT_NAME}; use crate::{disk_usage_eviction_task::DiskUsageEvictionTaskConfig, virtual_file::io_engine}; use crate::{tenant::config::TenantConf, virtual_file}; -use crate::{ - IGNORED_TENANT_FILE_NAME, TENANT_CONFIG_NAME, TENANT_HEATMAP_BASENAME, - TENANT_LOCATION_CONFIG_NAME, TIMELINE_DELETE_MARK_SUFFIX, -}; +use crate::{TENANT_HEATMAP_BASENAME, TENANT_LOCATION_CONFIG_NAME, TIMELINE_DELETE_MARK_SUFFIX}; use self::defaults::DEFAULT_CONCURRENT_TENANT_WARMUP; @@ -811,21 +806,12 @@ impl PageServerConf { self.tenants_path().join(tenant_shard_id.to_string()) } - pub fn tenant_ignore_mark_file_path(&self, tenant_shard_id: &TenantShardId) -> Utf8PathBuf { - self.tenant_path(tenant_shard_id) - .join(IGNORED_TENANT_FILE_NAME) - } - /// Points to a place in pageserver's local directory, - /// where certain tenant's tenantconf file should be located. - /// - /// Legacy: superseded by tenant_location_config_path. Eventually - /// remove this function. - pub fn tenant_config_path(&self, tenant_shard_id: &TenantShardId) -> Utf8PathBuf { - self.tenant_path(tenant_shard_id).join(TENANT_CONFIG_NAME) - } - - pub fn tenant_location_config_path(&self, tenant_shard_id: &TenantShardId) -> Utf8PathBuf { + /// where certain tenant's LocationConf be stored. + pub(crate) fn tenant_location_config_path( + &self, + tenant_shard_id: &TenantShardId, + ) -> Utf8PathBuf { self.tenant_path(tenant_shard_id) .join(TENANT_LOCATION_CONFIG_NAME) } @@ -860,14 +846,6 @@ impl PageServerConf { ) } - pub(crate) fn tenant_deleted_mark_file_path( - &self, - tenant_shard_id: &TenantShardId, - ) -> Utf8PathBuf { - self.tenant_path(tenant_shard_id) - .join(TENANT_DELETED_MARKER_FILE_NAME) - } - pub fn traces_path(&self) -> Utf8PathBuf { self.workdir.join("traces") } @@ -1468,7 +1446,7 @@ broker_endpoint = '{broker_endpoint}' assert_eq!( parsed_remote_storage_config, RemoteStorageConfig { - storage: RemoteStorageKind::LocalFs(local_storage_path.clone()), + storage: RemoteStorageKind::LocalFs { local_path: local_storage_path.clone() }, timeout: RemoteStorageConfig::DEFAULT_TIMEOUT, }, "Remote storage config should correctly parse the local FS config and fill other storage defaults" diff --git a/pageserver/src/deletion_queue.rs b/pageserver/src/deletion_queue.rs index 3960fc1b99..3e48552ace 100644 --- a/pageserver/src/deletion_queue.rs +++ b/pageserver/src/deletion_queue.rs @@ -382,17 +382,6 @@ pub enum DeletionQueueError { } impl DeletionQueueClient { - pub(crate) fn broken() -> Self { - // Channels whose receivers are immediately dropped. - let (tx, _rx) = tokio::sync::mpsc::unbounded_channel(); - let (executor_tx, _executor_rx) = tokio::sync::mpsc::channel(1); - Self { - tx, - executor_tx, - lsn_table: Arc::default(), - } - } - /// This is cancel-safe. If you drop the future before it completes, the message /// is not pushed, although in the context of the deletion queue it doesn't matter: once /// we decide to do a deletion the decision is always final. @@ -850,7 +839,9 @@ mod test { std::fs::create_dir_all(remote_fs_dir)?; let remote_fs_dir = harness.conf.workdir.join("remote_fs").canonicalize_utf8()?; let storage_config = RemoteStorageConfig { - storage: RemoteStorageKind::LocalFs(remote_fs_dir.clone()), + storage: RemoteStorageKind::LocalFs { + local_path: remote_fs_dir.clone(), + }, timeout: RemoteStorageConfig::DEFAULT_TIMEOUT, }; let storage = GenericRemoteStorage::from_config(&storage_config).unwrap(); diff --git a/pageserver/src/http/openapi_spec.yml b/pageserver/src/http/openapi_spec.yml index 71b486a4d3..58ff6e3f83 100644 --- a/pageserver/src/http/openapi_spec.yml +++ b/pageserver/src/http/openapi_spec.yml @@ -78,29 +78,14 @@ paths: delete: description: | - Attempts to delete specified tenant. 500, 503 and 409 errors should be retried until 404 is retrieved. - 404 means that deletion successfully finished" + Attempts to delete specified tenant. 500, 503 and 409 errors should be retried. Deleting + a non-existent tenant is considered successful (returns 200). responses: "200": description: Tenant was successfully deleted, or was already not found. - "404": - description: Tenant not found. This is a success result, equivalent to 200. - content: - application/json: - schema: - $ref: "#/components/schemas/NotFoundError" - "409": - description: Deletion is already in progress, continue polling - content: - application/json: - schema: - $ref: "#/components/schemas/ConflictError" - "412": - description: Deletion may not proceed, tenant is not in Active state - content: - application/json: - schema: - $ref: "#/components/schemas/PreconditionFailedError" + "503": + description: Service is unavailable, or tenant is already being modified (perhaps concurrently deleted) + /v1/tenant/{tenant_id}/time_travel_remote_storage: parameters: @@ -251,6 +236,13 @@ paths: type: string format: date-time description: A timestamp to get the LSN + - name: with_lease + in: query + required: false + schema: + type: boolean + description: Whether to grant a lease to the corresponding LSN. Default to false. + responses: "200": description: OK @@ -375,62 +367,11 @@ paths: $ref: "#/components/schemas/TenantLocationConfigResponse" "409": description: | - The tenant is already known to Pageserver in some way, - and hence this `/attach` call has been rejected. - - Some examples of how this can happen: - - tenant was created on this pageserver - - tenant attachment was started by an earlier call to `/attach`. - - Callers should poll the tenant status's `attachment_status` field, - like for status 202. See the longer description for `POST /attach` - for details. + The tenant is already being modified, perhaps by a concurrent call to this API content: application/json: schema: $ref: "#/components/schemas/ConflictError" - /v1/tenant/{tenant_id}/ignore: - parameters: - - name: tenant_id - in: path - required: true - schema: - type: string - post: - description: | - Remove tenant data (including all corresponding timelines) from pageserver's memory. - Files on local disk and remote storage are not affected. - - Future pageserver restarts won't load the data back until `load` is called on such tenant. - responses: - "200": - description: Tenant ignored - - - /v1/tenant/{tenant_id}/load: - parameters: - - name: tenant_id - in: path - required: true - schema: - type: string - post: - description: | - Schedules an operation that attempts to load a tenant from the local disk and - synchronise it with the remote storage (if enabled), repeating pageserver's restart logic for tenant load. - If the tenant was ignored before, removes the ignore mark and continues with load scheduling. - - Errors if the tenant is absent on disk, already present in memory or fails to schedule its load. - Scheduling a load does not mean that the tenant would load successfully, check tenant status to ensure load correctness. - requestBody: - required: false - content: - application/json: - schema: - $ref: "#/components/schemas/TenantLoadRequest" - responses: - "202": - description: Tenant scheduled to load successfully /v1/tenant/{tenant_id}/{timeline_id}/preserve_initdb_archive: parameters: @@ -812,8 +753,6 @@ components: For example this can be caused by s3 being unreachable. The retry may be implemented with call to detach, though it would be better to not automate it and inspec failed state manually before proceeding with a retry. - - See the tenant `/attach` endpoint for more information. type: object required: - slug @@ -1086,6 +1025,10 @@ components: kind: type: string enum: [past, present, future, nodata] + valid_until: + type: string + format: date-time + description: The expiration time of the granted lease. LsnLease: type: object diff --git a/pageserver/src/http/routes.rs b/pageserver/src/http/routes.rs index 482879630a..5ebd34a406 100644 --- a/pageserver/src/http/routes.rs +++ b/pageserver/src/http/routes.rs @@ -21,6 +21,7 @@ use pageserver_api::models::IngestAuxFilesRequest; use pageserver_api::models::ListAuxFilesRequest; use pageserver_api::models::LocationConfig; use pageserver_api::models::LocationConfigListResponse; +use pageserver_api::models::LsnLease; use pageserver_api::models::ShardParameters; use pageserver_api::models::TenantDetails; use pageserver_api::models::TenantLocationConfigResponse; @@ -30,13 +31,11 @@ use pageserver_api::models::TenantShardLocation; use pageserver_api::models::TenantShardSplitRequest; use pageserver_api::models::TenantShardSplitResponse; use pageserver_api::models::TenantSorting; -use pageserver_api::models::TenantState; use pageserver_api::models::TopTenantShardItem; use pageserver_api::models::TopTenantShardsRequest; use pageserver_api::models::TopTenantShardsResponse; use pageserver_api::models::{ - DownloadRemoteLayersTaskSpawnRequest, LocationConfigMode, TenantAttachRequest, - TenantLoadRequest, TenantLocationConfigRequest, + DownloadRemoteLayersTaskSpawnRequest, LocationConfigMode, TenantLocationConfigRequest, }; use pageserver_api::shard::ShardCount; use pageserver_api::shard::TenantShardId; @@ -50,7 +49,6 @@ use utils::auth::JwtAuth; use utils::failpoint_support::failpoints_handler; use utils::http::endpoint::prometheus_metrics_handler; use utils::http::endpoint::request_span; -use utils::http::json::json_request_or_empty_body; use utils::http::request::{get_request_param, must_get_query_param, parse_query_param}; use crate::context::{DownloadBehavior, RequestContext}; @@ -205,7 +203,6 @@ impl From for ApiError { NotFound(tenant_id) => { ApiError::NotFound(anyhow::anyhow!("NotFound: tenant {tenant_id}").into()) } - e @ AlreadyExists(_, _) => ApiError::Conflict(format!("{e}")), InProgress => { ApiError::ResourceUnavailable("Tenant is being modified concurrently".into()) } @@ -330,18 +327,12 @@ impl From for ApiError { } } -impl From for ApiError { - fn from(value: crate::tenant::delete::DeleteTenantError) -> Self { - use crate::tenant::delete::DeleteTenantError::*; +impl From for ApiError { + fn from(value: crate::tenant::mgr::DeleteTenantError) -> Self { + use crate::tenant::mgr::DeleteTenantError::*; match value { - Get(g) => ApiError::from(g), - e @ AlreadyInProgress => ApiError::Conflict(e.to_string()), - Timeline(t) => ApiError::from(t), - NotAttached => ApiError::NotFound(anyhow::anyhow!("Tenant is not attached").into()), SlotError(e) => e.into(), - SlotUpsertError(e) => e.into(), Other(o) => ApiError::InternalServerError(o), - e @ InvalidState(_) => ApiError::PreconditionFailed(e.to_string().into_boxed_str()), Cancelled => ApiError::ShuttingDown, } } @@ -735,6 +726,8 @@ async fn get_lsn_by_timestamp_handler( .map_err(ApiError::BadRequest)?; let timestamp_pg = postgres_ffi::to_pg_timestamp(timestamp); + let with_lease = parse_query_param(&request, "with_lease")?.unwrap_or(false); + let ctx = RequestContext::new(TaskKind::MgmtRequest, DownloadBehavior::Download); let timeline = @@ -743,10 +736,15 @@ async fn get_lsn_by_timestamp_handler( let result = timeline .find_lsn_for_timestamp(timestamp_pg, &cancel, &ctx) .await?; + #[derive(serde::Serialize, Debug)] struct Result { lsn: Lsn, kind: &'static str, + #[serde(default)] + #[serde(skip_serializing_if = "Option::is_none")] + #[serde(flatten)] + lease: Option, } let (lsn, kind) = match result { LsnForTimestamp::Present(lsn) => (lsn, "present"), @@ -754,11 +752,28 @@ async fn get_lsn_by_timestamp_handler( LsnForTimestamp::Past(lsn) => (lsn, "past"), LsnForTimestamp::NoData(lsn) => (lsn, "nodata"), }; - let result = Result { lsn, kind }; + + let lease = if with_lease { + timeline + .make_lsn_lease(lsn, timeline.get_lsn_lease_length_for_ts(), &ctx) + .inspect_err(|_| { + warn!("fail to grant a lease to {}", lsn); + }) + .ok() + } else { + None + }; + + let result = Result { lsn, kind, lease }; + let valid_until = result + .lease + .as_ref() + .map(|l| humantime::format_rfc3339_millis(l.valid_until).to_string()); tracing::info!( lsn=?result.lsn, kind=%result.kind, timestamp=%timestamp_raw, + valid_until=?valid_until, "lsn_by_timestamp finished" ); json_response(StatusCode::OK, result) @@ -803,58 +818,6 @@ async fn get_timestamp_of_lsn_handler( } } -async fn tenant_attach_handler( - mut request: Request, - _cancel: CancellationToken, -) -> Result, ApiError> { - let tenant_id: TenantId = parse_request_param(&request, "tenant_id")?; - check_permission(&request, Some(tenant_id))?; - - let maybe_body: Option = json_request_or_empty_body(&mut request).await?; - let tenant_conf = match &maybe_body { - Some(request) => TenantConfOpt::try_from(&*request.config).map_err(ApiError::BadRequest)?, - None => TenantConfOpt::default(), - }; - - let ctx = RequestContext::new(TaskKind::MgmtRequest, DownloadBehavior::Warn); - - info!("Handling tenant attach {tenant_id}"); - - let state = get_state(&request); - - let generation = get_request_generation(state, maybe_body.as_ref().and_then(|r| r.generation))?; - - let tenant_shard_id = TenantShardId::unsharded(tenant_id); - let shard_params = ShardParameters::default(); - let location_conf = LocationConf::attached_single(tenant_conf, generation, &shard_params); - - let tenant = state - .tenant_manager - .upsert_location(tenant_shard_id, location_conf, None, SpawnMode::Eager, &ctx) - .await?; - - let Some(tenant) = tenant else { - // This should never happen: indicates a bug in upsert_location - return Err(ApiError::InternalServerError(anyhow::anyhow!( - "Upsert succeeded but didn't return tenant!" - ))); - }; - - // We might have successfully constructed a Tenant, but it could still - // end up in a broken state: - if let TenantState::Broken { - reason, - backtrace: _, - } = tenant.current_state() - { - return Err(ApiError::InternalServerError(anyhow::anyhow!( - "Tenant state is Broken: {reason}" - ))); - } - - json_response(StatusCode::ACCEPTED, ()) -} - async fn timeline_delete_handler( request: Request, _cancel: CancellationToken, @@ -885,33 +848,6 @@ async fn timeline_delete_handler( json_response(StatusCode::ACCEPTED, ()) } -async fn tenant_detach_handler( - request: Request, - _cancel: CancellationToken, -) -> Result, ApiError> { - let tenant_id: TenantId = parse_request_param(&request, "tenant_id")?; - check_permission(&request, Some(tenant_id))?; - let detach_ignored: Option = parse_query_param(&request, "detach_ignored")?; - - // This is a legacy API (`/location_conf` is the replacement). It only supports unsharded tenants - let tenant_shard_id = TenantShardId::unsharded(tenant_id); - - let state = get_state(&request); - let conf = state.conf; - state - .tenant_manager - .detach_tenant( - conf, - tenant_shard_id, - detach_ignored.unwrap_or(false), - &state.deletion_queue_client, - ) - .instrument(info_span!("tenant_detach", %tenant_id, shard_id=%tenant_shard_id.shard_slug())) - .await?; - - json_response(StatusCode::OK, ()) -} - async fn tenant_reset_handler( request: Request, _cancel: CancellationToken, @@ -932,54 +868,6 @@ async fn tenant_reset_handler( json_response(StatusCode::OK, ()) } -async fn tenant_load_handler( - mut request: Request, - _cancel: CancellationToken, -) -> Result, ApiError> { - let tenant_id: TenantId = parse_request_param(&request, "tenant_id")?; - check_permission(&request, Some(tenant_id))?; - - let ctx = RequestContext::new(TaskKind::MgmtRequest, DownloadBehavior::Warn); - - let maybe_body: Option = json_request_or_empty_body(&mut request).await?; - - let state = get_state(&request); - - // The /load request is only usable when control_plane_api is not set. Once it is set, callers - // should always use /attach instead. - let generation = get_request_generation(state, maybe_body.as_ref().and_then(|r| r.generation))?; - - mgr::load_tenant( - state.conf, - tenant_id, - generation, - state.broker_client.clone(), - state.remote_storage.clone(), - state.deletion_queue_client.clone(), - &ctx, - ) - .instrument(info_span!("load", %tenant_id)) - .await?; - - json_response(StatusCode::ACCEPTED, ()) -} - -async fn tenant_ignore_handler( - request: Request, - _cancel: CancellationToken, -) -> Result, ApiError> { - let tenant_id: TenantId = parse_request_param(&request, "tenant_id")?; - check_permission(&request, Some(tenant_id))?; - - let state = get_state(&request); - let conf = state.conf; - mgr::ignore_tenant(conf, tenant_id) - .instrument(info_span!("ignore_tenant", %tenant_id)) - .await?; - - json_response(StatusCode::OK, ()) -} - async fn tenant_list_handler( request: Request, _cancel: CancellationToken, @@ -999,7 +887,9 @@ async fn tenant_list_handler( state: state.clone(), current_physical_size: None, attachment_status: state.attachment_status(), - generation: (*gen).into(), + generation: (*gen) + .into() + .expect("Tenants are always attached with a generation"), }) .collect::>(); @@ -1047,7 +937,10 @@ async fn tenant_status( state: state.clone(), current_physical_size: Some(current_physical_size), attachment_status: state.attachment_status(), - generation: tenant.generation().into(), + generation: tenant + .generation() + .into() + .expect("Tenants are always attached with a generation"), }, walredo: tenant.wal_redo_manager_status(), timelines: tenant.list_timeline_ids(), @@ -1071,23 +964,16 @@ async fn tenant_delete_handler( let state = get_state(&request); - let status = state + state .tenant_manager - .delete_tenant(tenant_shard_id, ACTIVE_TENANT_TIMEOUT) + .delete_tenant(tenant_shard_id) .instrument(info_span!("tenant_delete_handler", tenant_id = %tenant_shard_id.tenant_id, shard_id = %tenant_shard_id.shard_slug() )) .await?; - // Callers use 404 as success for deletions, for historical reasons. - if status == StatusCode::NOT_FOUND { - return Err(ApiError::NotFound( - anyhow::anyhow!("Deletion complete").into(), - )); - } - - json_response(status, ()) + json_response(StatusCode::OK, ()) } /// HTTP endpoint to query the current tenant_size of a tenant. @@ -1507,7 +1393,7 @@ async fn put_tenant_location_config_handler( if let LocationConfigMode::Detached = request_data.config.mode { if let Err(e) = state .tenant_manager - .detach_tenant(conf, tenant_shard_id, true, &state.deletion_queue_client) + .detach_tenant(conf, tenant_shard_id, &state.deletion_queue_client) .instrument(info_span!("tenant_detach", tenant_id = %tenant_shard_id.tenant_id, shard_id = %tenant_shard_id.shard_slug() @@ -1771,6 +1657,14 @@ async fn timeline_compact_handler( if Some(true) == parse_query_param::<_, bool>(&request, "force_image_layer_creation")? { flags |= CompactFlags::ForceImageLayerCreation; } + if Some(true) == parse_query_param::<_, bool>(&request, "enhanced_gc_bottom_most_compaction")? { + if !cfg!(feature = "testing") { + return Err(ApiError::InternalServerError(anyhow!( + "enhanced_gc_bottom_most_compaction is only available in testing mode" + ))); + } + flags |= CompactFlags::EnhancedGcBottomMostCompaction; + } let wait_until_uploaded = parse_query_param::<_, bool>(&request, "wait_until_uploaded")?.unwrap_or(false); @@ -2755,21 +2649,9 @@ pub fn make_router( .post("/v1/tenant/:tenant_shard_id/timeline", |r| { api_handler(r, timeline_create_handler) }) - .post("/v1/tenant/:tenant_id/attach", |r| { - api_handler(r, tenant_attach_handler) - }) - .post("/v1/tenant/:tenant_id/detach", |r| { - api_handler(r, tenant_detach_handler) - }) .post("/v1/tenant/:tenant_shard_id/reset", |r| { api_handler(r, tenant_reset_handler) }) - .post("/v1/tenant/:tenant_id/load", |r| { - api_handler(r, tenant_load_handler) - }) - .post("/v1/tenant/:tenant_id/ignore", |r| { - api_handler(r, tenant_ignore_handler) - }) .post( "/v1/tenant/:tenant_shard_id/timeline/:timeline_id/preserve_initdb_archive", |r| api_handler(r, timeline_preserve_initdb_handler), diff --git a/pageserver/src/lib.rs b/pageserver/src/lib.rs index c69fb8c83b..353f97264c 100644 --- a/pageserver/src/lib.rs +++ b/pageserver/src/lib.rs @@ -113,11 +113,7 @@ pub async fn shutdown_pageserver( } /// Per-tenant configuration file. -/// Full path: `tenants//config`. -pub(crate) const TENANT_CONFIG_NAME: &str = "config"; - -/// Per-tenant configuration file. -/// Full path: `tenants//config`. +/// Full path: `tenants//config-v1`. pub(crate) const TENANT_LOCATION_CONFIG_NAME: &str = "config-v1"; /// Per-tenant copy of their remote heatmap, downloaded into the local @@ -136,13 +132,6 @@ pub(crate) const TIMELINE_UNINIT_MARK_SUFFIX: &str = "___uninit"; pub(crate) const TIMELINE_DELETE_MARK_SUFFIX: &str = "___delete"; -/// A marker file to prevent pageserver from loading a certain tenant on restart. -/// Different from [`TIMELINE_UNINIT_MARK_SUFFIX`] due to semantics of the corresponding -/// `ignore` management API command, that expects the ignored tenant to be properly loaded -/// into pageserver's memory before being ignored. -/// Full path: `tenants//___ignored_tenant`. -pub const IGNORED_TENANT_FILE_NAME: &str = "___ignored_tenant"; - pub fn is_temporary(path: &Utf8Path) -> bool { match path.file_name() { Some(name) => name.ends_with(TEMP_FILE_SUFFIX), diff --git a/pageserver/src/metrics.rs b/pageserver/src/metrics.rs index e8a1e063c5..ca697afcf6 100644 --- a/pageserver/src/metrics.rs +++ b/pageserver/src/metrics.rs @@ -145,14 +145,6 @@ impl ReconstructTimeMetrics { } } -pub(crate) static MATERIALIZED_PAGE_CACHE_HIT_DIRECT: Lazy = Lazy::new(|| { - register_int_counter!( - "pageserver_materialized_cache_hits_direct_total", - "Number of cache hits from materialized page cache without redo", - ) - .expect("failed to define a metric") -}); - pub(crate) struct ReconstructDataTimeMetrics { singular: Histogram, vectored: Histogram, @@ -182,14 +174,6 @@ pub(crate) static GET_RECONSTRUCT_DATA_TIME: Lazy = } }); -pub(crate) static MATERIALIZED_PAGE_CACHE_HIT: Lazy = Lazy::new(|| { - register_int_counter!( - "pageserver_materialized_cache_hits_total", - "Number of cache hits from materialized page cache", - ) - .expect("failed to define a metric") -}); - pub(crate) struct GetVectoredLatency { map: EnumMap>, } @@ -298,12 +282,8 @@ pub(crate) static SCAN_LATENCY: Lazy = Lazy::new(|| { }); pub(crate) struct PageCacheMetricsForTaskKind { - pub read_accesses_materialized_page: IntCounter, pub read_accesses_immutable: IntCounter, - pub read_hits_immutable: IntCounter, - pub read_hits_materialized_page_exact: IntCounter, - pub read_hits_materialized_page_older_lsn: IntCounter, } pub(crate) struct PageCacheMetrics { @@ -336,16 +316,6 @@ pub(crate) static PAGE_CACHE: Lazy = Lazy::new(|| PageCacheMet let content_kind = ::from_usize(content_kind); let content_kind: &'static str = content_kind.into(); PageCacheMetricsForTaskKind { - read_accesses_materialized_page: { - PAGE_CACHE_READ_ACCESSES - .get_metric_with_label_values(&[ - task_kind, - "materialized_page", - content_kind, - ]) - .unwrap() - }, - read_accesses_immutable: { PAGE_CACHE_READ_ACCESSES .get_metric_with_label_values(&[task_kind, "immutable", content_kind]) @@ -357,28 +327,6 @@ pub(crate) static PAGE_CACHE: Lazy = Lazy::new(|| PageCacheMet .get_metric_with_label_values(&[task_kind, "immutable", content_kind, "-"]) .unwrap() }, - - read_hits_materialized_page_exact: { - PAGE_CACHE_READ_HITS - .get_metric_with_label_values(&[ - task_kind, - "materialized_page", - content_kind, - "exact", - ]) - .unwrap() - }, - - read_hits_materialized_page_older_lsn: { - PAGE_CACHE_READ_HITS - .get_metric_with_label_values(&[ - task_kind, - "materialized_page", - content_kind, - "older_lsn", - ]) - .unwrap() - }, } })) })), @@ -394,7 +342,6 @@ pub(crate) struct PageCacheSizeMetrics { pub max_bytes: UIntGauge, pub current_bytes_immutable: UIntGauge, - pub current_bytes_materialized_page: UIntGauge, } static PAGE_CACHE_SIZE_CURRENT_BYTES: Lazy = Lazy::new(|| { @@ -420,11 +367,6 @@ pub(crate) static PAGE_CACHE_SIZE: Lazy = .get_metric_with_label_values(&["immutable"]) .unwrap() }, - current_bytes_materialized_page: { - PAGE_CACHE_SIZE_CURRENT_BYTES - .get_metric_with_label_values(&["materialized_page"]) - .unwrap() - }, }); pub(crate) mod page_cache_eviction_metrics { @@ -603,6 +545,15 @@ static AUX_FILE_SIZE: Lazy = Lazy::new(|| { .expect("failed to define a metric") }); +static VALID_LSN_LEASE_COUNT: Lazy = Lazy::new(|| { + register_uint_gauge_vec!( + "pageserver_valid_lsn_lease_count", + "The number of valid leases after refreshing gc info.", + &["tenant_id", "shard_id", "timeline_id"], + ) + .expect("failed to define a metric") +}); + pub(crate) mod initial_logical_size { use metrics::{register_int_counter, register_int_counter_vec, IntCounter, IntCounterVec}; use once_cell::sync::Lazy; @@ -1405,17 +1356,23 @@ static COMPUTE_STARTUP_BUCKETS: Lazy<[f64; 28]> = Lazy::new(|| { .map(|ms| (ms as f64) / 1000.0) }); -pub(crate) struct BasebackupQueryTime(HistogramVec); +pub(crate) struct BasebackupQueryTime { + ok: Histogram, + error: Histogram, +} + pub(crate) static BASEBACKUP_QUERY_TIME: Lazy = Lazy::new(|| { - BasebackupQueryTime({ - register_histogram_vec!( - "pageserver_basebackup_query_seconds", - "Histogram of basebackup queries durations, by result type", - &["result"], - COMPUTE_STARTUP_BUCKETS.to_vec(), - ) - .expect("failed to define a metric") - }) + let vec = register_histogram_vec!( + "pageserver_basebackup_query_seconds", + "Histogram of basebackup queries durations, by result type", + &["result"], + COMPUTE_STARTUP_BUCKETS.to_vec(), + ) + .expect("failed to define a metric"); + BasebackupQueryTime { + ok: vec.get_metric_with_label_values(&["ok"]).unwrap(), + error: vec.get_metric_with_label_values(&["error"]).unwrap(), + } }); pub(crate) struct BasebackupQueryTimeOngoingRecording<'a, 'c> { @@ -1470,12 +1427,11 @@ impl<'a, 'c> BasebackupQueryTimeOngoingRecording<'a, 'c> { elapsed } }; - let label_value = if res.is_ok() { "ok" } else { "error" }; - let metric = self - .parent - .0 - .get_metric_with_label_values(&[label_value]) - .unwrap(); + let metric = if res.is_ok() { + &self.parent.ok + } else { + &self.parent.error + }; metric.observe(ex_throttled.as_secs_f64()); } } @@ -1489,6 +1445,46 @@ pub(crate) static LIVE_CONNECTIONS_COUNT: Lazy = Lazy::new(|| { .expect("failed to define a metric") }); +#[derive(Clone, Copy, enum_map::Enum, IntoStaticStr)] +pub(crate) enum ComputeCommandKind { + PageStreamV2, + PageStream, + Basebackup, + GetLastRecordRlsn, + Fullbackup, + ImportBasebackup, + ImportWal, + LeaseLsn, + Show, +} + +pub(crate) struct ComputeCommandCounters { + map: EnumMap, +} + +pub(crate) static COMPUTE_COMMANDS_COUNTERS: Lazy = Lazy::new(|| { + let inner = register_int_counter_vec!( + "pageserver_compute_commands", + "Number of compute -> pageserver commands processed", + &["command"] + ) + .expect("failed to define a metric"); + + ComputeCommandCounters { + map: EnumMap::from_array(std::array::from_fn(|i| { + let command = ::from_usize(i); + let command_str: &'static str = command.into(); + inner.with_label_values(&[command_str]) + })), + } +}); + +impl ComputeCommandCounters { + pub(crate) fn for_command(&self, command: ComputeCommandKind) -> &IntCounter { + &self.map[command] + } +} + // remote storage metrics static REMOTE_TIMELINE_CLIENT_CALLS: Lazy = Lazy::new(|| { @@ -2108,6 +2104,8 @@ pub(crate) struct TimelineMetrics { pub directory_entries_count_gauge: Lazy UIntGauge>>, pub evictions: IntCounter, pub evictions_with_low_residence_duration: std::sync::RwLock, + /// Number of valid LSN leases. + pub valid_lsn_lease_count_gauge: UIntGauge, shutdown: std::sync::atomic::AtomicBool, } @@ -2206,6 +2204,10 @@ impl TimelineMetrics { let evictions_with_low_residence_duration = evictions_with_low_residence_duration_builder .build(&tenant_id, &shard_id, &timeline_id); + let valid_lsn_lease_count_gauge = VALID_LSN_LEASE_COUNT + .get_metric_with_label_values(&[&tenant_id, &shard_id, &timeline_id]) + .unwrap(); + TimelineMetrics { tenant_id, shard_id, @@ -2228,6 +2230,7 @@ impl TimelineMetrics { evictions_with_low_residence_duration: std::sync::RwLock::new( evictions_with_low_residence_duration, ), + valid_lsn_lease_count_gauge, shutdown: std::sync::atomic::AtomicBool::default(), } } @@ -2277,6 +2280,7 @@ impl TimelineMetrics { } let _ = EVICTIONS.remove_label_values(&[tenant_id, shard_id, timeline_id]); let _ = AUX_FILE_SIZE.remove_label_values(&[tenant_id, shard_id, timeline_id]); + let _ = VALID_LSN_LEASE_COUNT.remove_label_values(&[tenant_id, shard_id, timeline_id]); self.evictions_with_low_residence_duration .write() @@ -2918,13 +2922,11 @@ pub fn preinitialize_metrics() { // FIXME(4813): make it so that we have no top level metrics as this fn will easily fall out of // order: // - global metrics reside in a Lazy - // - access via crate::metrics::PS_METRICS.materialized_page_cache_hit.inc() + // - access via crate::metrics::PS_METRICS.some_metric.inc() // - could move the statics into TimelineMetrics::new()? // counters [ - &MATERIALIZED_PAGE_CACHE_HIT, - &MATERIALIZED_PAGE_CACHE_HIT_DIRECT, &UNEXPECTED_ONDEMAND_DOWNLOADS, &WALRECEIVER_STARTED_CONNECTIONS, &WALRECEIVER_BROKER_UPDATES, @@ -2986,4 +2988,6 @@ pub fn preinitialize_metrics() { // Custom Lazy::force(&RECONSTRUCT_TIME); Lazy::force(&tenant_throttling::TIMELINE_GET); + Lazy::force(&BASEBACKUP_QUERY_TIME); + Lazy::force(&COMPUTE_COMMANDS_COUNTERS); } diff --git a/pageserver/src/page_cache.rs b/pageserver/src/page_cache.rs index 529fb9bb07..f386c825b8 100644 --- a/pageserver/src/page_cache.rs +++ b/pageserver/src/page_cache.rs @@ -17,7 +17,6 @@ //! //! Two types of pages are supported: //! -//! * **Materialized pages**, filled & used by page reconstruction //! * **Immutable File pages**, filled & used by [`crate::tenant::block_io`] and [`crate::tenant::ephemeral_file`]. //! //! Note that [`crate::tenant::ephemeral_file::EphemeralFile`] is generally mutable, but, it's append-only. @@ -28,9 +27,6 @@ //! Page cache maps from a cache key to a buffer slot. //! The cache key uniquely identifies the piece of data that is being cached. //! -//! The cache key for **materialized pages** is [`TenantShardId`], [`TimelineId`], [`Key`], and [`Lsn`]. -//! Use [`PageCache::memorize_materialized_page`] and [`PageCache::lookup_materialized_page`] for fill & access. -//! //! The cache key for **immutable file** pages is [`FileId`] and a block number. //! Users of page cache that wish to page-cache an arbitrary (immutable!) on-disk file do the following: //! * Have a mechanism to deterministically associate the on-disk file with a [`FileId`]. @@ -82,13 +78,10 @@ use std::{ use anyhow::Context; use once_cell::sync::OnceCell; -use pageserver_api::shard::TenantShardId; -use utils::{id::TimelineId, lsn::Lsn}; use crate::{ context::RequestContext, metrics::{page_cache_eviction_metrics, PageCacheSizeMetrics}, - repository::Key, }; static PAGE_CACHE: OnceCell = OnceCell::new(); @@ -139,33 +132,7 @@ pub fn next_file_id() -> FileId { #[derive(Debug, PartialEq, Eq, Clone)] #[allow(clippy::enum_variant_names)] enum CacheKey { - MaterializedPage { - hash_key: MaterializedPageHashKey, - lsn: Lsn, - }, - ImmutableFilePage { - file_id: FileId, - blkno: u32, - }, -} - -#[derive(Debug, PartialEq, Eq, Hash, Clone)] -struct MaterializedPageHashKey { - /// Why is this TenantShardId rather than TenantId? - /// - /// Usually, the materialized value of a page@lsn is identical on any shard in the same tenant. However, this - /// this not the case for certain internally-generated pages (e.g. relation sizes). In future, we may make this - /// key smaller by omitting the shard, if we ensure that reads to such pages always skip the cache, or are - /// special-cased in some other way. - tenant_shard_id: TenantShardId, - timeline_id: TimelineId, - key: Key, -} - -#[derive(Clone)] -struct Version { - lsn: Lsn, - slot_idx: usize, + ImmutableFilePage { file_id: FileId, blkno: u32 }, } struct Slot { @@ -236,17 +203,6 @@ impl SlotInner { } pub struct PageCache { - /// This contains the mapping from the cache key to buffer slot that currently - /// contains the page, if any. - /// - /// TODO: This is protected by a single lock. If that becomes a bottleneck, - /// this HashMap can be replaced with a more concurrent version, there are - /// plenty of such crates around. - /// - /// If you add support for caching different kinds of objects, each object kind - /// can have a separate mapping map, next to this field. - materialized_page_map: std::sync::RwLock>>, - immutable_page_map: std::sync::RwLock>, /// The actual buffers with their metadata. @@ -371,175 +327,14 @@ pub enum ReadBufResult<'a> { } impl PageCache { - // - // Section 1.1: Public interface functions for looking up and memorizing materialized page - // versions in the page cache - // - - /// Look up a materialized page version. - /// - /// The 'lsn' is an upper bound, this will return the latest version of - /// the given block, but not newer than 'lsn'. Returns the actual LSN of the - /// returned page. - pub async fn lookup_materialized_page( - &self, - tenant_shard_id: TenantShardId, - timeline_id: TimelineId, - key: &Key, - lsn: Lsn, - ctx: &RequestContext, - ) -> Option<(Lsn, PageReadGuard)> { - let Ok(permit) = self.try_get_pinned_slot_permit().await else { - return None; - }; - - crate::metrics::PAGE_CACHE - .for_ctx(ctx) - .read_accesses_materialized_page - .inc(); - - let mut cache_key = CacheKey::MaterializedPage { - hash_key: MaterializedPageHashKey { - tenant_shard_id, - timeline_id, - key: *key, - }, - lsn, - }; - - if let Some(guard) = self - .try_lock_for_read(&mut cache_key, &mut Some(permit)) - .await - { - if let CacheKey::MaterializedPage { - hash_key: _, - lsn: available_lsn, - } = cache_key - { - if available_lsn == lsn { - crate::metrics::PAGE_CACHE - .for_ctx(ctx) - .read_hits_materialized_page_exact - .inc(); - } else { - crate::metrics::PAGE_CACHE - .for_ctx(ctx) - .read_hits_materialized_page_older_lsn - .inc(); - } - Some((available_lsn, guard)) - } else { - panic!("unexpected key type in slot"); - } - } else { - None - } - } - - /// - /// Store an image of the given page in the cache. - /// - pub async fn memorize_materialized_page( - &self, - tenant_shard_id: TenantShardId, - timeline_id: TimelineId, - key: Key, - lsn: Lsn, - img: &[u8], - ) -> anyhow::Result<()> { - let cache_key = CacheKey::MaterializedPage { - hash_key: MaterializedPageHashKey { - tenant_shard_id, - timeline_id, - key, - }, - lsn, - }; - - let mut permit = Some(self.try_get_pinned_slot_permit().await?); - loop { - // First check if the key already exists in the cache. - if let Some(slot_idx) = self.search_mapping_exact(&cache_key) { - // The page was found in the mapping. Lock the slot, and re-check - // that it's still what we expected (because we don't released the mapping - // lock already, another thread could have evicted the page) - let slot = &self.slots[slot_idx]; - let inner = slot.inner.write().await; - if inner.key.as_ref() == Some(&cache_key) { - slot.inc_usage_count(); - debug_assert!( - { - let guard = inner.permit.lock().unwrap(); - guard.upgrade().is_none() - }, - "we hold a write lock, so, no one else should have a permit" - ); - debug_assert_eq!(inner.buf.len(), img.len()); - // We already had it in cache. Another thread must've put it there - // concurrently. Check that it had the same contents that we - // replayed. - assert!(inner.buf == img); - return Ok(()); - } - } - debug_assert!(permit.is_some()); - - // Not found. Find a victim buffer - let (slot_idx, mut inner) = self - .find_victim(permit.as_ref().unwrap()) - .await - .context("Failed to find evict victim")?; - - // Insert mapping for this. At this point, we may find that another - // thread did the same thing concurrently. In that case, we evicted - // our victim buffer unnecessarily. Put it into the free list and - // continue with the slot that the other thread chose. - if let Some(_existing_slot_idx) = self.try_insert_mapping(&cache_key, slot_idx) { - // TODO: put to free list - - // We now just loop back to start from beginning. This is not - // optimal, we'll perform the lookup in the mapping again, which - // is not really necessary because we already got - // 'existing_slot_idx'. But this shouldn't happen often enough - // to matter much. - continue; - } - - // Make the slot ready - let slot = &self.slots[slot_idx]; - inner.key = Some(cache_key.clone()); - slot.set_usage_count(1); - // Create a write guard for the slot so we go through the expected motions. - debug_assert!( - { - let guard = inner.permit.lock().unwrap(); - guard.upgrade().is_none() - }, - "we hold a write lock, so, no one else should have a permit" - ); - let mut write_guard = PageWriteGuard { - state: PageWriteGuardState::Invalid { - _permit: permit.take().unwrap(), - inner, - }, - }; - write_guard.copy_from_slice(img); - let _ = write_guard.mark_valid(); - return Ok(()); - } - } - - // Section 1.2: Public interface functions for working with immutable file pages. - pub async fn read_immutable_buf( &self, file_id: FileId, blkno: u32, ctx: &RequestContext, ) -> anyhow::Result { - let mut cache_key = CacheKey::ImmutableFilePage { file_id, blkno }; - - self.lock_for_read(&mut cache_key, ctx).await + self.lock_for_read(&(CacheKey::ImmutableFilePage { file_id, blkno }), ctx) + .await } // @@ -573,19 +368,11 @@ impl PageCache { /// Look up a page in the cache. /// - /// If the search criteria is not exact, *cache_key is updated with the key - /// for exact key of the returned page. (For materialized pages, that means - /// that the LSN in 'cache_key' is updated with the LSN of the returned page - /// version.) - /// - /// If no page is found, returns None and *cache_key is left unmodified. - /// async fn try_lock_for_read( &self, - cache_key: &mut CacheKey, + cache_key: &CacheKey, permit: &mut Option, ) -> Option { - let cache_key_orig = cache_key.clone(); if let Some(slot_idx) = self.search_mapping(cache_key) { // The page was found in the mapping. Lock the slot, and re-check // that it's still what we expected (because we released the mapping @@ -598,9 +385,6 @@ impl PageCache { _permit: inner.coalesce_readers_permit(permit.take().unwrap()), slot_guard: inner, }); - } else { - // search_mapping might have modified the search key; restore it. - *cache_key = cache_key_orig; } } None @@ -637,15 +421,12 @@ impl PageCache { /// async fn lock_for_read( &self, - cache_key: &mut CacheKey, + cache_key: &CacheKey, ctx: &RequestContext, ) -> anyhow::Result { let mut permit = Some(self.try_get_pinned_slot_permit().await?); let (read_access, hit) = match cache_key { - CacheKey::MaterializedPage { .. } => { - unreachable!("Materialized pages use lookup_materialized_page") - } CacheKey::ImmutableFilePage { .. } => ( &crate::metrics::PAGE_CACHE .for_ctx(ctx) @@ -717,52 +498,15 @@ impl PageCache { /// Search for a page in the cache using the given search key. /// - /// Returns the slot index, if any. If the search criteria is not exact, - /// *cache_key is updated with the actual key of the found page. + /// Returns the slot index, if any. /// /// NOTE: We don't hold any lock on the mapping on return, so the slot might /// get recycled for an unrelated page immediately after this function /// returns. The caller is responsible for re-checking that the slot still /// contains the page with the same key before using it. /// - fn search_mapping(&self, cache_key: &mut CacheKey) -> Option { + fn search_mapping(&self, cache_key: &CacheKey) -> Option { match cache_key { - CacheKey::MaterializedPage { hash_key, lsn } => { - let map = self.materialized_page_map.read().unwrap(); - let versions = map.get(hash_key)?; - - let version_idx = match versions.binary_search_by_key(lsn, |v| v.lsn) { - Ok(version_idx) => version_idx, - Err(0) => return None, - Err(version_idx) => version_idx - 1, - }; - let version = &versions[version_idx]; - *lsn = version.lsn; - Some(version.slot_idx) - } - CacheKey::ImmutableFilePage { file_id, blkno } => { - let map = self.immutable_page_map.read().unwrap(); - Some(*map.get(&(*file_id, *blkno))?) - } - } - } - - /// Search for a page in the cache using the given search key. - /// - /// Like 'search_mapping, but performs an "exact" search. Used for - /// allocating a new buffer. - fn search_mapping_exact(&self, key: &CacheKey) -> Option { - match key { - CacheKey::MaterializedPage { hash_key, lsn } => { - let map = self.materialized_page_map.read().unwrap(); - let versions = map.get(hash_key)?; - - if let Ok(version_idx) = versions.binary_search_by_key(lsn, |v| v.lsn) { - Some(versions[version_idx].slot_idx) - } else { - None - } - } CacheKey::ImmutableFilePage { file_id, blkno } => { let map = self.immutable_page_map.read().unwrap(); Some(*map.get(&(*file_id, *blkno))?) @@ -775,27 +519,6 @@ impl PageCache { /// fn remove_mapping(&self, old_key: &CacheKey) { match old_key { - CacheKey::MaterializedPage { - hash_key: old_hash_key, - lsn: old_lsn, - } => { - let mut map = self.materialized_page_map.write().unwrap(); - if let Entry::Occupied(mut old_entry) = map.entry(old_hash_key.clone()) { - let versions = old_entry.get_mut(); - - if let Ok(version_idx) = versions.binary_search_by_key(old_lsn, |v| v.lsn) { - versions.remove(version_idx); - self.size_metrics - .current_bytes_materialized_page - .sub_page_sz(1); - if versions.is_empty() { - old_entry.remove_entry(); - } - } - } else { - panic!("could not find old key in mapping") - } - } CacheKey::ImmutableFilePage { file_id, blkno } => { let mut map = self.immutable_page_map.write().unwrap(); map.remove(&(*file_id, *blkno)) @@ -812,30 +535,6 @@ impl PageCache { /// of the existing mapping and leaves it untouched. fn try_insert_mapping(&self, new_key: &CacheKey, slot_idx: usize) -> Option { match new_key { - CacheKey::MaterializedPage { - hash_key: new_key, - lsn: new_lsn, - } => { - let mut map = self.materialized_page_map.write().unwrap(); - let versions = map.entry(new_key.clone()).or_default(); - match versions.binary_search_by_key(new_lsn, |v| v.lsn) { - Ok(version_idx) => Some(versions[version_idx].slot_idx), - Err(version_idx) => { - versions.insert( - version_idx, - Version { - lsn: *new_lsn, - slot_idx, - }, - ); - self.size_metrics - .current_bytes_materialized_page - .add_page_sz(1); - None - } - } - } - CacheKey::ImmutableFilePage { file_id, blkno } => { let mut map = self.immutable_page_map.write().unwrap(); match map.entry((*file_id, *blkno)) { @@ -949,7 +648,6 @@ impl PageCache { let size_metrics = &crate::metrics::PAGE_CACHE_SIZE; size_metrics.max_bytes.set_page_sz(num_pages); size_metrics.current_bytes_immutable.set_page_sz(0); - size_metrics.current_bytes_materialized_page.set_page_sz(0); let slots = page_buffer .chunks_exact_mut(PAGE_SZ) @@ -968,7 +666,6 @@ impl PageCache { .collect(); Self { - materialized_page_map: Default::default(), immutable_page_map: Default::default(), slots, next_evict_slot: AtomicUsize::new(0), diff --git a/pageserver/src/page_service.rs b/pageserver/src/page_service.rs index ebc23e8945..6ea5f396d0 100644 --- a/pageserver/src/page_service.rs +++ b/pageserver/src/page_service.rs @@ -55,7 +55,7 @@ use crate::basebackup::BasebackupError; use crate::context::{DownloadBehavior, RequestContext}; use crate::import_datadir::import_wal_from_tar; use crate::metrics; -use crate::metrics::LIVE_CONNECTIONS_COUNT; +use crate::metrics::{ComputeCommandKind, COMPUTE_COMMANDS_COUNTERS, LIVE_CONNECTIONS_COUNT}; use crate::pgdatadir_mapping::Version; use crate::span::debug_assert_current_span_has_tenant_and_timeline_id; use crate::span::debug_assert_current_span_has_tenant_and_timeline_id_no_shard_id; @@ -1554,6 +1554,10 @@ where self.check_permission(Some(tenant_id))?; + COMPUTE_COMMANDS_COUNTERS + .for_command(ComputeCommandKind::PageStreamV2) + .inc(); + self.handle_pagerequests( pgb, tenant_id, @@ -1579,6 +1583,10 @@ where self.check_permission(Some(tenant_id))?; + COMPUTE_COMMANDS_COUNTERS + .for_command(ComputeCommandKind::PageStream) + .inc(); + self.handle_pagerequests( pgb, tenant_id, @@ -1605,6 +1613,10 @@ where self.check_permission(Some(tenant_id))?; + COMPUTE_COMMANDS_COUNTERS + .for_command(ComputeCommandKind::Basebackup) + .inc(); + let lsn = if let Some(lsn_str) = params.get(2) { Some( Lsn::from_str(lsn_str) @@ -1662,6 +1674,11 @@ where .record("timeline_id", field::display(timeline_id)); self.check_permission(Some(tenant_id))?; + + COMPUTE_COMMANDS_COUNTERS + .for_command(ComputeCommandKind::GetLastRecordRlsn) + .inc(); + async { let timeline = self .get_active_tenant_timeline(tenant_id, timeline_id, ShardSelector::Zero) @@ -1723,6 +1740,10 @@ where self.check_permission(Some(tenant_id))?; + COMPUTE_COMMANDS_COUNTERS + .for_command(ComputeCommandKind::Fullbackup) + .inc(); + // Check that the timeline exists self.handle_basebackup_request( pgb, @@ -1771,6 +1792,10 @@ where self.check_permission(Some(tenant_id))?; + COMPUTE_COMMANDS_COUNTERS + .for_command(ComputeCommandKind::ImportBasebackup) + .inc(); + match self .handle_import_basebackup( pgb, @@ -1818,6 +1843,10 @@ where self.check_permission(Some(tenant_id))?; + COMPUTE_COMMANDS_COUNTERS + .for_command(ComputeCommandKind::ImportWal) + .inc(); + match self .handle_import_wal(pgb, tenant_id, timeline_id, start_lsn, end_lsn, ctx) .await @@ -1855,6 +1884,10 @@ where self.check_permission(Some(tenant_shard_id.tenant_id))?; + COMPUTE_COMMANDS_COUNTERS + .for_command(ComputeCommandKind::LeaseLsn) + .inc(); + // The caller is responsible for providing correct lsn. let lsn = Lsn::from_str(params[2]) .with_context(|| format!("Failed to parse Lsn from {}", params[2]))?; @@ -1886,6 +1919,10 @@ where self.check_permission(Some(tenant_id))?; + COMPUTE_COMMANDS_COUNTERS + .for_command(ComputeCommandKind::Show) + .inc(); + let tenant = self .get_active_tenant_with_timeout( tenant_id, diff --git a/pageserver/src/tenant.rs b/pageserver/src/tenant.rs index ca5765c99b..45e542a336 100644 --- a/pageserver/src/tenant.rs +++ b/pageserver/src/tenant.rs @@ -55,11 +55,9 @@ use self::config::AttachedLocationConfig; use self::config::AttachmentMode; use self::config::LocationConf; use self::config::TenantConf; -use self::delete::DeleteTenantFlow; use self::metadata::TimelineMetadata; use self::mgr::GetActiveTenantError; use self::mgr::GetTenantError; -use self::mgr::TenantsMap; use self::remote_timeline_client::upload::upload_index_part; use self::remote_timeline_client::RemoteTimelineClient; use self::timeline::uninit::TimelineCreateGuard; @@ -137,7 +135,6 @@ pub mod remote_timeline_client; pub mod storage_layer; pub mod config; -pub mod delete; pub mod mgr; pub mod secondary; pub mod tasks; @@ -161,8 +158,6 @@ pub const TENANTS_SEGMENT_NAME: &str = "tenants"; /// Parts of the `.neon/tenants//timelines/` directory prefix. pub const TIMELINES_SEGMENT_NAME: &str = "timelines"; -pub const TENANT_DELETED_MARKER_FILE_NAME: &str = "deleted"; - /// References to shared objects that are passed into each tenant, such /// as the shared remote storage client and process initialization state. #[derive(Clone)] @@ -207,7 +202,6 @@ struct TimelinePreload { } pub(crate) struct TenantPreload { - deleting: bool, timelines: HashMap, } @@ -286,8 +280,6 @@ pub struct Tenant { /// background warmup. pub(crate) activate_now_sem: tokio::sync::Semaphore, - pub(crate) delete_progress: Arc>, - // Cancellation token fires when we have entered shutdown(). This is a parent of // Timelines' cancellation token. pub(crate) cancel: CancellationToken, @@ -654,10 +646,9 @@ impl Tenant { attached_conf: AttachedTenantConf, shard_identity: ShardIdentity, init_order: Option, - tenants: &'static std::sync::RwLock, mode: SpawnMode, ctx: &RequestContext, - ) -> anyhow::Result> { + ) -> Arc { let wal_redo_manager = Arc::new(WalRedoManager::from(PostgresRedoManager::new( conf, tenant_shard_id, @@ -828,52 +819,6 @@ impl Tenant { // Remote preload is complete. drop(remote_load_completion); - let pending_deletion = { - match DeleteTenantFlow::should_resume_deletion( - conf, - preload.as_ref().map(|p| p.deleting).unwrap_or(false), - &tenant_clone, - ) - .await - { - Ok(should_resume_deletion) => should_resume_deletion, - Err(err) => { - make_broken(&tenant_clone, anyhow::anyhow!(err), BrokenVerbosity::Error); - return Ok(()); - } - } - }; - - info!("pending_deletion {}", pending_deletion.is_some()); - - if let Some(deletion) = pending_deletion { - // as we are no longer loading, signal completion by dropping - // the completion while we resume deletion - drop(_completion); - let background_jobs_can_start = - init_order.as_ref().map(|x| &x.background_jobs_can_start); - if let Some(background) = background_jobs_can_start { - info!("waiting for backgound jobs barrier"); - background.clone().wait().await; - info!("ready for backgound jobs barrier"); - } - - let deleted = DeleteTenantFlow::resume_from_attach( - deletion, - &tenant_clone, - preload, - tenants, - &ctx, - ) - .await; - - if let Err(e) = deleted { - make_broken(&tenant_clone, anyhow::anyhow!(e), BrokenVerbosity::Error); - } - - return Ok(()); - } - // We will time the duration of the attach phase unless this is a creation (attach will do no work) let attached = { let _attach_timer = match mode { @@ -911,7 +856,7 @@ impl Tenant { } .instrument(tracing::info_span!(parent: None, "attach", tenant_id=%tenant_shard_id.tenant_id, shard_id=%tenant_shard_id.shard_slug(), gen=?generation)), ); - Ok(tenant) + tenant } #[instrument(skip_all)] @@ -931,21 +876,13 @@ impl Tenant { ) .await?; - let deleting = other_keys.contains(TENANT_DELETED_MARKER_FILE_NAME); - info!( - "found {} timelines, deleting={}", - remote_timeline_ids.len(), - deleting - ); + info!("found {} timelines", remote_timeline_ids.len(),); for k in other_keys { - if k != TENANT_DELETED_MARKER_FILE_NAME { - warn!("Unexpected non timeline key {k}"); - } + warn!("Unexpected non timeline key {k}"); } Ok(TenantPreload { - deleting, timelines: Self::load_timeline_metadata( self, remote_timeline_ids, @@ -974,7 +911,6 @@ impl Tenant { let preload = match (preload, mode) { (Some(p), _) => p, (None, SpawnMode::Create) => TenantPreload { - deleting: false, timelines: HashMap::new(), }, (None, _) => { @@ -1211,30 +1147,6 @@ impl Tenant { .await } - /// Create a placeholder Tenant object for a broken tenant - pub fn create_broken_tenant( - conf: &'static PageServerConf, - tenant_shard_id: TenantShardId, - remote_storage: GenericRemoteStorage, - reason: String, - ) -> Arc { - Arc::new(Tenant::new( - TenantState::Broken { - reason, - backtrace: String::new(), - }, - conf, - AttachedTenantConf::try_from(LocationConf::default()).unwrap(), - // Shard identity isn't meaningful for a broken tenant: it's just a placeholder - // to occupy the slot for this TenantShardId. - ShardIdentity::broken(tenant_shard_id.shard_number, tenant_shard_id.shard_count), - None, - tenant_shard_id, - remote_storage, - DeletionQueueClient::broken(), - )) - } - async fn load_timeline_metadata( self: &Arc, timeline_ids: HashSet, @@ -2215,6 +2127,7 @@ impl Tenant { // Upload an index from the parent: this is partly to provide freshness for the // child tenants that will copy it, and partly for general ease-of-debugging: there will // always be a parent shard index in the same generation as we wrote the child shard index. + tracing::info!(timeline_id=%timeline.timeline_id, "Uploading index"); timeline .remote_client .schedule_index_upload_for_file_changes()?; @@ -2222,12 +2135,14 @@ impl Tenant { // Shut down the timeline's remote client: this means that the indices we write // for child shards will not be invalidated by the parent shard deleting layers. + tracing::info!(timeline_id=%timeline.timeline_id, "Shutting down remote storage client"); timeline.remote_client.shutdown().await; // Download methods can still be used after shutdown, as they don't flow through the remote client's // queue. In principal the RemoteTimelineClient could provide this without downloading it, but this // operation is rare, so it's simpler to just download it (and robustly guarantees that the index // we use here really is the remotely persistent one). + tracing::info!(timeline_id=%timeline.timeline_id, "Downloading index_part from parent"); let result = timeline.remote_client .download_index_file(&self.cancel) .instrument(info_span!("download_index_file", tenant_id=%self.tenant_shard_id.tenant_id, shard_id=%self.tenant_shard_id.shard_slug(), timeline_id=%timeline.timeline_id)) @@ -2240,6 +2155,7 @@ impl Tenant { }; for child_shard in child_shards { + tracing::info!(timeline_id=%timeline.timeline_id, "Uploading index_part for child {}", child_shard.to_index()); upload_index_part( &self.remote_storage, child_shard, @@ -2554,6 +2470,10 @@ impl Tenant { remote_storage: GenericRemoteStorage, deletion_queue_client: DeletionQueueClient, ) -> Tenant { + debug_assert!( + !attached_conf.location.generation.is_none() || conf.control_plane_api.is_none() + ); + let (state, mut rx) = watch::channel(state); tokio::spawn(async move { @@ -2628,7 +2548,6 @@ impl Tenant { cached_synthetic_tenant_size: Arc::new(AtomicU64::new(0)), eviction_task_tenant_state: tokio::sync::Mutex::new(EvictionTaskTenantState::default()), activate_now_sem: tokio::sync::Semaphore::new(0), - delete_progress: Arc::new(tokio::sync::Mutex::new(DeleteTenantFlow::default())), cancel: CancellationToken::default(), gate: Gate::default(), timeline_get_throttle: Arc::new(throttle::Throttle::new( @@ -2645,45 +2564,22 @@ impl Tenant { conf: &'static PageServerConf, tenant_shard_id: &TenantShardId, ) -> anyhow::Result { - let legacy_config_path = conf.tenant_config_path(tenant_shard_id); let config_path = conf.tenant_location_config_path(tenant_shard_id); if config_path.exists() { // New-style config takes precedence let deserialized = Self::read_config(&config_path)?; Ok(toml_edit::de::from_document::(deserialized)?) - } else if legacy_config_path.exists() { - // Upgrade path: found an old-style configuration only - let deserialized = Self::read_config(&legacy_config_path)?; - - let mut tenant_conf = TenantConfOpt::default(); - for (key, item) in deserialized.iter() { - match key { - "tenant_config" => { - tenant_conf = TenantConfOpt::try_from(item.to_owned()).context(format!("Failed to parse config from file '{legacy_config_path}' as pageserver config"))?; - } - _ => bail!( - "config file {legacy_config_path} has unrecognized pageserver option '{key}'" - ), - } - } - - // Legacy configs are implicitly in attached state, and do not support sharding - Ok(LocationConf::attached_single( - tenant_conf, - Generation::none(), - &models::ShardParameters::default(), - )) } else { - // FIXME If the config file is not found, assume that we're attaching - // a detached tenant and config is passed via attach command. - // https://github.com/neondatabase/neon/issues/1555 - // OR: we're loading after incomplete deletion that managed to remove config. - info!( - "tenant config not found in {} or {}", - config_path, legacy_config_path - ); - Ok(LocationConf::default()) + // The config should almost always exist for a tenant directory: + // - When attaching a tenant, the config is the first thing we write + // - When detaching a tenant, we atomically move the directory to a tmp location + // before deleting contents. + // + // The very rare edge case that can result in a missing config is if we crash during attach + // between creating directory and writing config. Callers should handle that as if the + // directory didn't exist. + anyhow::bail!("tenant config not found in {}", config_path); } } @@ -2705,47 +2601,17 @@ impl Tenant { tenant_shard_id: &TenantShardId, location_conf: &LocationConf, ) -> anyhow::Result<()> { - let legacy_config_path = conf.tenant_config_path(tenant_shard_id); let config_path = conf.tenant_location_config_path(tenant_shard_id); - Self::persist_tenant_config_at( - tenant_shard_id, - &config_path, - &legacy_config_path, - location_conf, - ) - .await + Self::persist_tenant_config_at(tenant_shard_id, &config_path, location_conf).await } #[tracing::instrument(skip_all, fields(tenant_id=%tenant_shard_id.tenant_id, shard_id=%tenant_shard_id.shard_slug()))] pub(super) async fn persist_tenant_config_at( tenant_shard_id: &TenantShardId, config_path: &Utf8Path, - legacy_config_path: &Utf8Path, location_conf: &LocationConf, ) -> anyhow::Result<()> { - if let LocationMode::Attached(attach_conf) = &location_conf.mode { - // The modern-style LocationConf config file requires a generation to be set. In case someone - // is running a pageserver without the infrastructure to set generations, write out the legacy-style - // config file that only contains TenantConf. - // - // This will eventually be removed in https://github.com/neondatabase/neon/issues/5388 - - if attach_conf.generation.is_none() { - tracing::info!( - "Running without generations, writing legacy-style tenant config file" - ); - Self::persist_tenant_config_legacy( - tenant_shard_id, - legacy_config_path, - &location_conf.tenant_conf, - ) - .await?; - - return Ok(()); - } - } - debug!("persisting tenantconf to {config_path}"); let mut conf_content = r#"# This file contains a specific per-tenant's config. @@ -2772,37 +2638,6 @@ impl Tenant { Ok(()) } - #[tracing::instrument(skip_all, fields(tenant_id=%tenant_shard_id.tenant_id, shard_id=%tenant_shard_id.shard_slug()))] - async fn persist_tenant_config_legacy( - tenant_shard_id: &TenantShardId, - target_config_path: &Utf8Path, - tenant_conf: &TenantConfOpt, - ) -> anyhow::Result<()> { - debug!("persisting tenantconf to {target_config_path}"); - - let mut conf_content = r#"# This file contains a specific per-tenant's config. -# It is read in case of pageserver restart. - -[tenant_config] -"# - .to_string(); - - // Convert the config to a toml file. - conf_content += &toml_edit::ser::to_string(&tenant_conf)?; - - let temp_path = path_with_suffix_extension(target_config_path, TEMP_FILE_SUFFIX); - - let tenant_shard_id = *tenant_shard_id; - let target_config_path = target_config_path.to_owned(); - let conf_content = conf_content.into_bytes(); - VirtualFile::crashsafe_overwrite(target_config_path.clone(), temp_path, conf_content) - .await - .with_context(|| { - format!("write tenant {tenant_shard_id} config to {target_config_path}") - })?; - Ok(()) - } - // // How garbage collection works: // @@ -3021,6 +2856,11 @@ impl Tenant { let now = SystemTime::now(); target.leases.retain(|_, lease| !lease.is_expired(&now)); + timeline + .metrics + .valid_lsn_lease_count_gauge + .set(target.leases.len() as u64); + match gc_cutoffs.remove(&timeline.timeline_id) { Some(cutoffs) => { target.retain_lsns = branchpoints; @@ -3906,7 +3746,9 @@ pub(crate) mod harness { let remote_fs_dir = conf.workdir.join("localfs"); std::fs::create_dir_all(&remote_fs_dir).unwrap(); let config = RemoteStorageConfig { - storage: RemoteStorageKind::LocalFs(remote_fs_dir.clone()), + storage: RemoteStorageKind::LocalFs { + local_path: remote_fs_dir.clone(), + }, timeout: RemoteStorageConfig::DEFAULT_TIMEOUT, }; let remote_storage = GenericRemoteStorage::from_config(&config).unwrap(); @@ -4066,6 +3908,7 @@ mod tests { use storage_layer::PersistentLayerKey; use tests::storage_layer::ValuesReconstructState; use tests::timeline::{GetVectoredError, ShutdownMode}; + use timeline::GcInfo; use utils::bin_ser::BeSer; use utils::id::TenantId; @@ -6743,49 +6586,48 @@ mod tests { // img layer at 0x10 let img_layer = (0..10) - .map(|id| (get_key(id), test_img(&format!("value {id}@0x10")))) + .map(|id| (get_key(id), Bytes::from(format!("value {id}@0x10")))) .collect_vec(); let delta1 = vec![ - // TODO: we should test a real delta record here, which requires us to add a variant of NeonWalRecord for testing purpose. ( get_key(1), Lsn(0x20), - Value::Image(test_img("value 1@0x20")), + Value::Image(Bytes::from("value 1@0x20")), ), ( get_key(2), Lsn(0x30), - Value::Image(test_img("value 2@0x30")), + Value::Image(Bytes::from("value 2@0x30")), ), ( get_key(3), Lsn(0x40), - Value::Image(test_img("value 3@0x40")), + Value::Image(Bytes::from("value 3@0x40")), ), ]; let delta2 = vec![ ( get_key(5), Lsn(0x20), - Value::Image(test_img("value 5@0x20")), + Value::Image(Bytes::from("value 5@0x20")), ), ( get_key(6), Lsn(0x20), - Value::Image(test_img("value 6@0x20")), + Value::Image(Bytes::from("value 6@0x20")), ), ]; let delta3 = vec![ ( get_key(8), Lsn(0x40), - Value::Image(test_img("value 8@0x40")), + Value::Image(Bytes::from("value 8@0x40")), ), ( get_key(9), Lsn(0x40), - Value::Image(test_img("value 9@0x40")), + Value::Image(Bytes::from("value 9@0x40")), ), ]; @@ -6807,9 +6649,42 @@ mod tests { guard.cutoffs.horizon = Lsn(0x30); } + let expected_result = [ + Bytes::from_static(b"value 0@0x10"), + Bytes::from_static(b"value 1@0x20"), + Bytes::from_static(b"value 2@0x30"), + Bytes::from_static(b"value 3@0x40"), + Bytes::from_static(b"value 4@0x10"), + Bytes::from_static(b"value 5@0x20"), + Bytes::from_static(b"value 6@0x20"), + Bytes::from_static(b"value 7@0x10"), + Bytes::from_static(b"value 8@0x40"), + Bytes::from_static(b"value 9@0x40"), + ]; + + for (idx, expected) in expected_result.iter().enumerate() { + assert_eq!( + tline + .get(get_key(idx as u32), Lsn(0x50), &ctx) + .await + .unwrap(), + expected + ); + } + let cancel = CancellationToken::new(); tline.compact_with_gc(&cancel, &ctx).await.unwrap(); + for (idx, expected) in expected_result.iter().enumerate() { + assert_eq!( + tline + .get(get_key(idx as u32), Lsn(0x50), &ctx) + .await + .unwrap(), + expected + ); + } + // Check if the image layer at the GC horizon contains exactly what we want let image_at_gc_horizon = tline .inspect_image_layers(Lsn(0x30), &ctx) @@ -6820,14 +6695,22 @@ mod tests { .collect::>(); assert_eq!(image_at_gc_horizon.len(), 10); - let expected_lsn = [0x10, 0x20, 0x30, 0x10, 0x10, 0x20, 0x20, 0x10, 0x10, 0x10]; + let expected_result = [ + Bytes::from_static(b"value 0@0x10"), + Bytes::from_static(b"value 1@0x20"), + Bytes::from_static(b"value 2@0x30"), + Bytes::from_static(b"value 3@0x10"), + Bytes::from_static(b"value 4@0x10"), + Bytes::from_static(b"value 5@0x20"), + Bytes::from_static(b"value 6@0x20"), + Bytes::from_static(b"value 7@0x10"), + Bytes::from_static(b"value 8@0x10"), + Bytes::from_static(b"value 9@0x10"), + ]; for idx in 0..10 { assert_eq!( image_at_gc_horizon[idx], - ( - get_key(idx as u32), - test_img(&format!("value {idx}@{:#x}", expected_lsn[idx])) - ) + (get_key(idx as u32), expected_result[idx].clone()) ); } @@ -6860,7 +6743,7 @@ mod tests { }, // The delta layer that is cut in the middle PersistentLayerKey { - key_range: Key::MIN..get_key(9), + key_range: get_key(3)..get_key(4), lsn_range: Lsn(0x30)..Lsn(0x41), is_delta: true }, @@ -6945,6 +6828,9 @@ mod tests { tline.get(get_key(2), Lsn(0x50), &ctx).await?, Bytes::from_static(b"0x10,0x20,0x30") ); + + // Need to remove the limit of "Neon WAL redo requires base image". + // assert_eq!(tline.get(get_key(3), Lsn(0x50), &ctx).await?, Bytes::new()); // assert_eq!(tline.get(get_key(4), Lsn(0x50), &ctx).await?, Bytes::new()); @@ -7039,4 +6925,174 @@ mod tests { Ok(()) } + + #[tokio::test] + async fn test_simple_bottom_most_compaction_deltas() -> anyhow::Result<()> { + let harness = TenantHarness::create("test_simple_bottom_most_compaction_deltas")?; + let (tenant, ctx) = harness.load().await; + + fn get_key(id: u32) -> Key { + // using aux key here b/c they are guaranteed to be inside `collect_keyspace`. + let mut key = Key::from_hex("620000000033333333444444445500000000").unwrap(); + key.field6 = id; + key + } + + // We create one bottom-most image layer, a delta layer D1 crossing the GC horizon, D2 below the horizon, and D3 above the horizon. + // + // | D1 | | D3 | + // -| |-- gc horizon ----------------- + // | | | D2 | + // --------- img layer ------------------ + // + // What we should expact from this compaction is: + // | Part of D1 | | D3 | + // --------- img layer with D1+D2 at GC horizon------------------ + + // img layer at 0x10 + let img_layer = (0..10) + .map(|id| (get_key(id), Bytes::from(format!("value {id}@0x10")))) + .collect_vec(); + + let delta1 = vec![ + ( + get_key(1), + Lsn(0x20), + Value::WalRecord(NeonWalRecord::wal_append("@0x20")), + ), + ( + get_key(2), + Lsn(0x30), + Value::WalRecord(NeonWalRecord::wal_append("@0x30")), + ), + ( + get_key(3), + Lsn(0x28), + Value::WalRecord(NeonWalRecord::wal_append("@0x28")), + ), + ( + get_key(3), + Lsn(0x30), + Value::WalRecord(NeonWalRecord::wal_append("@0x30")), + ), + ( + get_key(3), + Lsn(0x40), + Value::WalRecord(NeonWalRecord::wal_append("@0x40")), + ), + ]; + let delta2 = vec![ + ( + get_key(5), + Lsn(0x20), + Value::WalRecord(NeonWalRecord::wal_append("@0x20")), + ), + ( + get_key(6), + Lsn(0x20), + Value::WalRecord(NeonWalRecord::wal_append("@0x20")), + ), + ]; + let delta3 = vec![ + ( + get_key(8), + Lsn(0x40), + Value::WalRecord(NeonWalRecord::wal_append("@0x40")), + ), + ( + get_key(9), + Lsn(0x40), + Value::WalRecord(NeonWalRecord::wal_append("@0x40")), + ), + ]; + + let tline = tenant + .create_test_timeline_with_layers( + TIMELINE_ID, + Lsn(0x10), + DEFAULT_PG_VERSION, + &ctx, + vec![delta1, delta2, delta3], // delta layers + vec![(Lsn(0x10), img_layer)], // image layers + Lsn(0x50), + ) + .await?; + { + // Update GC info + let mut guard = tline.gc_info.write().unwrap(); + *guard = GcInfo { + retain_lsns: vec![], + cutoffs: GcCutoffs { + pitr: Lsn(0x30), + horizon: Lsn(0x30), + }, + leases: Default::default(), + }; + } + + let expected_result = [ + Bytes::from_static(b"value 0@0x10"), + Bytes::from_static(b"value 1@0x10@0x20"), + Bytes::from_static(b"value 2@0x10@0x30"), + Bytes::from_static(b"value 3@0x10@0x28@0x30@0x40"), + Bytes::from_static(b"value 4@0x10"), + Bytes::from_static(b"value 5@0x10@0x20"), + Bytes::from_static(b"value 6@0x10@0x20"), + Bytes::from_static(b"value 7@0x10"), + Bytes::from_static(b"value 8@0x10@0x40"), + Bytes::from_static(b"value 9@0x10@0x40"), + ]; + + let expected_result_at_gc_horizon = [ + Bytes::from_static(b"value 0@0x10"), + Bytes::from_static(b"value 1@0x10@0x20"), + Bytes::from_static(b"value 2@0x10@0x30"), + Bytes::from_static(b"value 3@0x10@0x28@0x30"), + Bytes::from_static(b"value 4@0x10"), + Bytes::from_static(b"value 5@0x10@0x20"), + Bytes::from_static(b"value 6@0x10@0x20"), + Bytes::from_static(b"value 7@0x10"), + Bytes::from_static(b"value 8@0x10"), + Bytes::from_static(b"value 9@0x10"), + ]; + + for idx in 0..10 { + assert_eq!( + tline + .get(get_key(idx as u32), Lsn(0x50), &ctx) + .await + .unwrap(), + &expected_result[idx] + ); + assert_eq!( + tline + .get(get_key(idx as u32), Lsn(0x30), &ctx) + .await + .unwrap(), + &expected_result_at_gc_horizon[idx] + ); + } + + let cancel = CancellationToken::new(); + tline.compact_with_gc(&cancel, &ctx).await.unwrap(); + + for idx in 0..10 { + assert_eq!( + tline + .get(get_key(idx as u32), Lsn(0x50), &ctx) + .await + .unwrap(), + &expected_result[idx] + ); + assert_eq!( + tline + .get(get_key(idx as u32), Lsn(0x30), &ctx) + .await + .unwrap(), + &expected_result_at_gc_horizon[idx] + ); + } + + Ok(()) + } } diff --git a/pageserver/src/tenant/block_io.rs b/pageserver/src/tenant/block_io.rs index 92928116c1..b406d50332 100644 --- a/pageserver/src/tenant/block_io.rs +++ b/pageserver/src/tenant/block_io.rs @@ -160,6 +160,7 @@ impl<'a> BlockCursor<'a> { /// /// The file is assumed to be immutable. This doesn't provide any functions /// for modifying the file, nor for invalidating the cache if it is modified. +#[derive(Clone)] pub struct FileBlockReader<'a> { pub file: &'a VirtualFile, diff --git a/pageserver/src/tenant/config.rs b/pageserver/src/tenant/config.rs index 1b9be12642..5b532e4830 100644 --- a/pageserver/src/tenant/config.rs +++ b/pageserver/src/tenant/config.rs @@ -281,22 +281,6 @@ impl LocationConf { } } -impl Default for LocationConf { - // TODO: this should be removed once tenant loading can guarantee that we are never - // loading from a directory without a configuration. - // => tech debt since https://github.com/neondatabase/neon/issues/1555 - fn default() -> Self { - Self { - mode: LocationMode::Attached(AttachedLocationConfig { - generation: Generation::none(), - attach_mode: AttachmentMode::Single, - }), - tenant_conf: TenantConfOpt::default(), - shard: ShardIdentity::unsharded(), - } - } -} - /// A tenant's calcuated configuration, which is the result of merging a /// tenant's TenantConfOpt with the global TenantConf from PageServerConf. /// diff --git a/pageserver/src/tenant/delete.rs b/pageserver/src/tenant/delete.rs deleted file mode 100644 index 8b36aa15e5..0000000000 --- a/pageserver/src/tenant/delete.rs +++ /dev/null @@ -1,662 +0,0 @@ -use std::sync::Arc; - -use anyhow::Context; -use camino::{Utf8Path, Utf8PathBuf}; -use pageserver_api::{models::TenantState, shard::TenantShardId}; -use remote_storage::{GenericRemoteStorage, RemotePath, TimeoutOrCancel}; -use tokio::sync::OwnedMutexGuard; -use tokio_util::sync::CancellationToken; -use tracing::{error, instrument, Instrument}; - -use utils::{backoff, completion, crashsafe, fs_ext, id::TimelineId, pausable_failpoint}; - -use crate::{ - config::PageServerConf, - context::RequestContext, - task_mgr::{self, TaskKind}, - tenant::{ - mgr::{TenantSlot, TenantsMapRemoveResult}, - remote_timeline_client::remote_heatmap_path, - timeline::ShutdownMode, - }, -}; - -use super::{ - mgr::{GetTenantError, TenantSlotError, TenantSlotUpsertError, TenantsMap}, - remote_timeline_client::{FAILED_REMOTE_OP_RETRIES, FAILED_UPLOAD_WARN_THRESHOLD}, - span, - timeline::delete::DeleteTimelineFlow, - tree_sort_timelines, DeleteTimelineError, Tenant, TenantPreload, -}; - -#[derive(Debug, thiserror::Error)] -pub(crate) enum DeleteTenantError { - #[error("GetTenant {0}")] - Get(#[from] GetTenantError), - - #[error("Tenant not attached")] - NotAttached, - - #[error("Invalid state {0}. Expected Active or Broken")] - InvalidState(TenantState), - - #[error("Tenant deletion is already in progress")] - AlreadyInProgress, - - #[error("Tenant map slot error {0}")] - SlotError(#[from] TenantSlotError), - - #[error("Tenant map slot upsert error {0}")] - SlotUpsertError(#[from] TenantSlotUpsertError), - - #[error("Timeline {0}")] - Timeline(#[from] DeleteTimelineError), - - #[error("Cancelled")] - Cancelled, - - #[error(transparent)] - Other(#[from] anyhow::Error), -} - -type DeletionGuard = tokio::sync::OwnedMutexGuard; - -fn remote_tenant_delete_mark_path( - conf: &PageServerConf, - tenant_shard_id: &TenantShardId, -) -> anyhow::Result { - let tenant_remote_path = conf - .tenant_path(tenant_shard_id) - .strip_prefix(&conf.workdir) - .context("Failed to strip workdir prefix") - .and_then(RemotePath::new) - .context("tenant path")?; - Ok(tenant_remote_path.join(Utf8Path::new("timelines/deleted"))) -} - -async fn create_remote_delete_mark( - conf: &PageServerConf, - remote_storage: &GenericRemoteStorage, - tenant_shard_id: &TenantShardId, - cancel: &CancellationToken, -) -> Result<(), DeleteTenantError> { - let remote_mark_path = remote_tenant_delete_mark_path(conf, tenant_shard_id)?; - - let data: &[u8] = &[]; - backoff::retry( - || async { - let data = bytes::Bytes::from_static(data); - let stream = futures::stream::once(futures::future::ready(Ok(data))); - remote_storage - .upload(stream, 0, &remote_mark_path, None, cancel) - .await - }, - TimeoutOrCancel::caused_by_cancel, - FAILED_UPLOAD_WARN_THRESHOLD, - FAILED_REMOTE_OP_RETRIES, - "mark_upload", - cancel, - ) - .await - .ok_or_else(|| anyhow::Error::new(TimeoutOrCancel::Cancel)) - .and_then(|x| x) - .context("mark_upload")?; - - Ok(()) -} - -async fn create_local_delete_mark( - conf: &PageServerConf, - tenant_shard_id: &TenantShardId, -) -> Result<(), DeleteTenantError> { - let marker_path = conf.tenant_deleted_mark_file_path(tenant_shard_id); - - // Note: we're ok to replace existing file. - let _ = std::fs::OpenOptions::new() - .write(true) - .create(true) - .truncate(true) - .open(&marker_path) - .with_context(|| format!("could not create delete marker file {marker_path:?}"))?; - - crashsafe::fsync_file_and_parent(&marker_path).context("sync_mark")?; - - Ok(()) -} - -async fn schedule_ordered_timeline_deletions( - tenant: &Arc, -) -> Result>, TimelineId)>, DeleteTenantError> { - // Tenant is stopping at this point. We know it will be deleted. - // No new timelines should be created. - // Tree sort timelines to delete from leafs to the root. - // NOTE: by calling clone we release the mutex which creates a possibility for a race: pending deletion - // can complete and remove timeline from the map in between our call to clone - // and `DeleteTimelineFlow::run`, so `run` wont find timeline in `timelines` map. - // timelines.lock is currently synchronous so we cant hold it across await point. - // So just ignore NotFound error if we get it from `run`. - // Beware: in case it becomes async and we try to hold it here, `run` also locks it, which can create a deadlock. - let timelines = tenant.timelines.lock().unwrap().clone(); - let sorted = - tree_sort_timelines(timelines, |t| t.get_ancestor_timeline_id()).context("tree sort")?; - - let mut already_running_deletions = vec![]; - - for (timeline_id, _) in sorted.into_iter().rev() { - let span = tracing::info_span!("timeline_delete", %timeline_id); - let res = DeleteTimelineFlow::run(tenant, timeline_id, true) - .instrument(span) - .await; - if let Err(e) = res { - match e { - DeleteTimelineError::NotFound => { - // Timeline deletion finished after call to clone above but before call - // to `DeleteTimelineFlow::run` and removed timeline from the map. - continue; - } - DeleteTimelineError::AlreadyInProgress(guard) => { - already_running_deletions.push((guard, timeline_id)); - continue; - } - e => return Err(DeleteTenantError::Timeline(e)), - } - } - } - - Ok(already_running_deletions) -} - -async fn ensure_timelines_dir_empty(timelines_path: &Utf8Path) -> Result<(), DeleteTenantError> { - // Assert timelines dir is empty. - if !fs_ext::is_directory_empty(timelines_path).await? { - // Display first 10 items in directory - let list = fs_ext::list_dir(timelines_path).await.context("list_dir")?; - let list = &list.into_iter().take(10).collect::>(); - return Err(DeleteTenantError::Other(anyhow::anyhow!( - "Timelines directory is not empty after all timelines deletion: {list:?}" - ))); - } - - Ok(()) -} - -async fn remove_tenant_remote_delete_mark( - conf: &PageServerConf, - remote_storage: &GenericRemoteStorage, - tenant_shard_id: &TenantShardId, - cancel: &CancellationToken, -) -> Result<(), DeleteTenantError> { - let path = remote_tenant_delete_mark_path(conf, tenant_shard_id)?; - backoff::retry( - || async { remote_storage.delete(&path, cancel).await }, - TimeoutOrCancel::caused_by_cancel, - FAILED_UPLOAD_WARN_THRESHOLD, - FAILED_REMOTE_OP_RETRIES, - "remove_tenant_remote_delete_mark", - cancel, - ) - .await - .ok_or_else(|| anyhow::Error::new(TimeoutOrCancel::Cancel)) - .and_then(|x| x) - .context("remove_tenant_remote_delete_mark")?; - Ok(()) -} - -// Cleanup fs traces: tenant config, timelines dir local delete mark, tenant dir -async fn cleanup_remaining_fs_traces( - conf: &PageServerConf, - tenant_shard_id: &TenantShardId, -) -> Result<(), DeleteTenantError> { - let rm = |p: Utf8PathBuf, is_dir: bool| async move { - if is_dir { - tokio::fs::remove_dir(&p).await - } else { - tokio::fs::remove_file(&p).await - } - .or_else(fs_ext::ignore_not_found) - .with_context(|| format!("failed to delete {p}")) - }; - - rm(conf.tenant_config_path(tenant_shard_id), false).await?; - rm(conf.tenant_location_config_path(tenant_shard_id), false).await?; - - fail::fail_point!("tenant-delete-before-remove-timelines-dir", |_| { - Err(anyhow::anyhow!( - "failpoint: tenant-delete-before-remove-timelines-dir" - ))? - }); - - rm(conf.timelines_path(tenant_shard_id), true).await?; - - fail::fail_point!("tenant-delete-before-remove-deleted-mark", |_| { - Err(anyhow::anyhow!( - "failpoint: tenant-delete-before-remove-deleted-mark" - ))? - }); - - // Make sure previous deletions are ordered before mark removal. - // Otherwise there is no guarantee that they reach the disk before mark deletion. - // So its possible for mark to reach disk first and for other deletions - // to be reordered later and thus missed if a crash occurs. - // Note that we dont need to sync after mark file is removed - // because we can tolerate the case when mark file reappears on startup. - let tenant_path = &conf.tenant_path(tenant_shard_id); - if tenant_path.exists() { - crashsafe::fsync_async(&conf.tenant_path(tenant_shard_id)) - .await - .context("fsync_pre_mark_remove")?; - } - - rm(conf.tenant_deleted_mark_file_path(tenant_shard_id), false).await?; - - rm(conf.tenant_heatmap_path(tenant_shard_id), false).await?; - - fail::fail_point!("tenant-delete-before-remove-tenant-dir", |_| { - Err(anyhow::anyhow!( - "failpoint: tenant-delete-before-remove-tenant-dir" - ))? - }); - - rm(conf.tenant_path(tenant_shard_id), true).await?; - - Ok(()) -} - -/// Orchestrates tenant shut down of all tasks, removes its in-memory structures, -/// and deletes its data from both disk and s3. -/// The sequence of steps: -/// 1. Upload remote deletion mark. -/// 2. Create local mark file. -/// 3. Shutdown tasks -/// 4. Run ordered timeline deletions -/// 5. Wait for timeline deletion operations that were scheduled before tenant deletion was requested -/// 6. Remove remote mark -/// 7. Cleanup remaining fs traces, tenant dir, config, timelines dir, local delete mark -/// It is resumable from any step in case a crash/restart occurs. -/// There are two entrypoints to the process: -/// 1. [`DeleteTenantFlow::run`] this is the main one called by a management api handler. -/// 2. [`DeleteTenantFlow::resume_from_attach`] is called when deletion is resumed tenant is found to be deleted during attach process. -/// Note the only other place that messes around timeline delete mark is the `Tenant::spawn_load` function. -#[derive(Default)] -pub enum DeleteTenantFlow { - #[default] - NotStarted, - InProgress, - Finished, -} - -impl DeleteTenantFlow { - // These steps are run in the context of management api request handler. - // Long running steps are continued to run in the background. - // NB: If this fails half-way through, and is retried, the retry will go through - // all the same steps again. Make sure the code here is idempotent, and don't - // error out if some of the shutdown tasks have already been completed! - // NOTE: static needed for background part. - // We assume that calling code sets up the span with tenant_id. - #[instrument(skip_all)] - pub(crate) async fn run( - conf: &'static PageServerConf, - remote_storage: GenericRemoteStorage, - tenants: &'static std::sync::RwLock, - tenant: Arc, - cancel: &CancellationToken, - ) -> Result<(), DeleteTenantError> { - span::debug_assert_current_span_has_tenant_id(); - - pausable_failpoint!("tenant-delete-before-run"); - - let mut guard = Self::prepare(&tenant).await?; - - if let Err(e) = Self::run_inner(&mut guard, conf, &remote_storage, &tenant, cancel).await { - tenant.set_broken(format!("{e:#}")).await; - return Err(e); - } - - Self::schedule_background(guard, conf, remote_storage, tenants, tenant); - - Ok(()) - } - - // Helper function needed to be able to match once on returned error and transition tenant into broken state. - // This is needed because tenant.shutwodn is not idempotent. If tenant state is set to stopping another call to tenant.shutdown - // will result in an error, but here we need to be able to retry shutdown when tenant deletion is retried. - // So the solution is to set tenant state to broken. - async fn run_inner( - guard: &mut OwnedMutexGuard, - conf: &'static PageServerConf, - remote_storage: &GenericRemoteStorage, - tenant: &Tenant, - cancel: &CancellationToken, - ) -> Result<(), DeleteTenantError> { - guard.mark_in_progress()?; - - fail::fail_point!("tenant-delete-before-create-remote-mark", |_| { - Err(anyhow::anyhow!( - "failpoint: tenant-delete-before-create-remote-mark" - ))? - }); - - create_remote_delete_mark(conf, remote_storage, &tenant.tenant_shard_id, cancel) - .await - .context("remote_mark")?; - - fail::fail_point!("tenant-delete-before-create-local-mark", |_| { - Err(anyhow::anyhow!( - "failpoint: tenant-delete-before-create-local-mark" - ))? - }); - - create_local_delete_mark(conf, &tenant.tenant_shard_id) - .await - .context("local delete mark")?; - - fail::fail_point!("tenant-delete-before-background", |_| { - Err(anyhow::anyhow!( - "failpoint: tenant-delete-before-background" - ))? - }); - - Ok(()) - } - - fn mark_in_progress(&mut self) -> anyhow::Result<()> { - match self { - Self::Finished => anyhow::bail!("Bug. Is in finished state"), - Self::InProgress { .. } => { /* We're in a retry */ } - Self::NotStarted => { /* Fresh start */ } - } - - *self = Self::InProgress; - - Ok(()) - } - - pub(crate) async fn should_resume_deletion( - conf: &'static PageServerConf, - remote_mark_exists: bool, - tenant: &Tenant, - ) -> Result, DeleteTenantError> { - let acquire = |t: &Tenant| { - Some( - Arc::clone(&t.delete_progress) - .try_lock_owned() - .expect("we're the only owner during init"), - ) - }; - - if remote_mark_exists { - return Ok(acquire(tenant)); - } - - // Check local mark first, if its there there is no need to go to s3 to check whether remote one exists. - if conf - .tenant_deleted_mark_file_path(&tenant.tenant_shard_id) - .exists() - { - Ok(acquire(tenant)) - } else { - Ok(None) - } - } - - pub(crate) async fn resume_from_attach( - guard: DeletionGuard, - tenant: &Arc, - preload: Option, - tenants: &'static std::sync::RwLock, - ctx: &RequestContext, - ) -> Result<(), DeleteTenantError> { - let (_, progress) = completion::channel(); - - tenant - .set_stopping(progress, false, true) - .await - .expect("cant be stopping or broken"); - - tenant - .attach(preload, super::SpawnMode::Eager, ctx) - .await - .context("attach")?; - - Self::background( - guard, - tenant.conf, - tenant.remote_storage.clone(), - tenants, - tenant, - ) - .await - } - - /// Check whether background deletion of this tenant is currently in progress - pub(crate) fn is_in_progress(tenant: &Tenant) -> bool { - tenant.delete_progress.try_lock().is_err() - } - - async fn prepare( - tenant: &Arc, - ) -> Result, DeleteTenantError> { - // FIXME: unsure about active only. Our init jobs may not be cancellable properly, - // so at least for now allow deletions only for active tenants. TODO recheck - // Broken and Stopping is needed for retries. - if !matches!( - tenant.current_state(), - TenantState::Active | TenantState::Broken { .. } - ) { - return Err(DeleteTenantError::InvalidState(tenant.current_state())); - } - - let guard = Arc::clone(&tenant.delete_progress) - .try_lock_owned() - .map_err(|_| DeleteTenantError::AlreadyInProgress)?; - - fail::fail_point!("tenant-delete-before-shutdown", |_| { - Err(anyhow::anyhow!("failpoint: tenant-delete-before-shutdown"))? - }); - - // make pageserver shutdown not to wait for our completion - let (_, progress) = completion::channel(); - - // It would be good to only set stopping here and continue shutdown in the background, but shutdown is not idempotent. - // i e it is an error to do: - // tenant.set_stopping - // tenant.shutdown - // Its also bad that we're holding tenants.read here. - // TODO relax set_stopping to be idempotent? - if tenant.shutdown(progress, ShutdownMode::Hard).await.is_err() { - return Err(DeleteTenantError::Other(anyhow::anyhow!( - "tenant shutdown is already in progress" - ))); - } - - Ok(guard) - } - - fn schedule_background( - guard: OwnedMutexGuard, - conf: &'static PageServerConf, - remote_storage: GenericRemoteStorage, - tenants: &'static std::sync::RwLock, - tenant: Arc, - ) { - let tenant_shard_id = tenant.tenant_shard_id; - - task_mgr::spawn( - task_mgr::BACKGROUND_RUNTIME.handle(), - TaskKind::TimelineDeletionWorker, - Some(tenant_shard_id), - None, - "tenant_delete", - false, - async move { - if let Err(err) = - Self::background(guard, conf, remote_storage, tenants, &tenant).await - { - error!("Error: {err:#}"); - tenant.set_broken(format!("{err:#}")).await; - }; - Ok(()) - } - .instrument(tracing::info_span!(parent: None, "delete_tenant", tenant_id=%tenant_shard_id.tenant_id, shard_id=%tenant_shard_id.shard_slug())), - ); - } - - async fn background( - mut guard: OwnedMutexGuard, - conf: &PageServerConf, - remote_storage: GenericRemoteStorage, - tenants: &'static std::sync::RwLock, - tenant: &Arc, - ) -> Result<(), DeleteTenantError> { - // Tree sort timelines, schedule delete for them. Mention retries from the console side. - // Note that if deletion fails we dont mark timelines as broken, - // the whole tenant will become broken as by `Self::schedule_background` logic - let already_running_timeline_deletions = schedule_ordered_timeline_deletions(tenant) - .await - .context("schedule_ordered_timeline_deletions")?; - - fail::fail_point!("tenant-delete-before-polling-ongoing-deletions", |_| { - Err(anyhow::anyhow!( - "failpoint: tenant-delete-before-polling-ongoing-deletions" - ))? - }); - - // Wait for deletions that were already running at the moment when tenant deletion was requested. - // When we can lock deletion guard it means that corresponding timeline deletion finished. - for (guard, timeline_id) in already_running_timeline_deletions { - let flow = guard.lock().await; - if !flow.is_finished() { - return Err(DeleteTenantError::Other(anyhow::anyhow!( - "already running timeline deletion failed: {timeline_id}" - ))); - } - } - - // Remove top-level tenant objects that don't belong to a timeline, such as heatmap - let heatmap_path = remote_heatmap_path(&tenant.tenant_shard_id()); - if let Some(Err(e)) = backoff::retry( - || async { - remote_storage - .delete(&heatmap_path, &task_mgr::shutdown_token()) - .await - }, - TimeoutOrCancel::caused_by_cancel, - FAILED_UPLOAD_WARN_THRESHOLD, - FAILED_REMOTE_OP_RETRIES, - "remove_remote_tenant_heatmap", - &task_mgr::shutdown_token(), - ) - .await - { - tracing::warn!("Failed to delete heatmap at {heatmap_path}: {e}"); - } - - let timelines_path = conf.timelines_path(&tenant.tenant_shard_id); - // May not exist if we fail in cleanup_remaining_fs_traces after removing it - if timelines_path.exists() { - // sanity check to guard against layout changes - ensure_timelines_dir_empty(&timelines_path) - .await - .context("timelines dir not empty")?; - } - - remove_tenant_remote_delete_mark( - conf, - &remote_storage, - &tenant.tenant_shard_id, - &task_mgr::shutdown_token(), - ) - .await?; - - pausable_failpoint!("tenant-delete-before-cleanup-remaining-fs-traces-pausable"); - fail::fail_point!("tenant-delete-before-cleanup-remaining-fs-traces", |_| { - Err(anyhow::anyhow!( - "failpoint: tenant-delete-before-cleanup-remaining-fs-traces" - ))? - }); - - cleanup_remaining_fs_traces(conf, &tenant.tenant_shard_id) - .await - .context("cleanup_remaining_fs_traces")?; - - { - pausable_failpoint!("tenant-delete-before-map-remove"); - - // This block is simply removing the TenantSlot for this tenant. It requires a loop because - // we might conflict with a TenantSlot::InProgress marker and need to wait for it. - // - // This complexity will go away when we simplify how deletion works: - // https://github.com/neondatabase/neon/issues/5080 - loop { - // Under the TenantMap lock, try to remove the tenant. We usually succeed, but if - // we encounter an InProgress marker, yield the barrier it contains and wait on it. - let barrier = { - let mut locked = tenants.write().unwrap(); - let removed = locked.remove(tenant.tenant_shard_id); - - // FIXME: we should not be modifying this from outside of mgr.rs. - // This will go away when we simplify deletion (https://github.com/neondatabase/neon/issues/5080) - - // Update stats - match &removed { - TenantsMapRemoveResult::Occupied(slot) => { - crate::metrics::TENANT_MANAGER.slot_removed(slot); - } - TenantsMapRemoveResult::InProgress(barrier) => { - crate::metrics::TENANT_MANAGER - .slot_removed(&TenantSlot::InProgress(barrier.clone())); - } - TenantsMapRemoveResult::Vacant => { - // Nothing changed in map, no metric update - } - } - - match removed { - TenantsMapRemoveResult::Occupied(TenantSlot::Attached(tenant)) => { - match tenant.current_state() { - TenantState::Stopping { .. } | TenantState::Broken { .. } => { - // Expected: we put the tenant into stopping state before we start deleting it - } - state => { - // Unexpected state - tracing::warn!( - "Tenant in unexpected state {state} after deletion" - ); - } - } - break; - } - TenantsMapRemoveResult::Occupied(TenantSlot::Secondary(_)) => { - // This is unexpected: this secondary tenants should not have been created, and we - // are not in a position to shut it down from here. - tracing::warn!("Tenant transitioned to secondary mode while deleting!"); - break; - } - TenantsMapRemoveResult::Occupied(TenantSlot::InProgress(_)) => { - unreachable!("TenantsMap::remove handles InProgress separately, should never return it here"); - } - TenantsMapRemoveResult::Vacant => { - tracing::warn!( - "Tenant removed from TenantsMap before deletion completed" - ); - break; - } - TenantsMapRemoveResult::InProgress(barrier) => { - // An InProgress entry was found, we must wait on its barrier - barrier - } - } - }; - - tracing::info!( - "Waiting for competing operation to complete before deleting state for tenant" - ); - barrier.wait().await; - } - } - - *guard = Self::Finished; - - Ok(()) - } -} diff --git a/pageserver/src/tenant/disk_btree.rs b/pageserver/src/tenant/disk_btree.rs index 119df3e6c4..b76498b608 100644 --- a/pageserver/src/tenant/disk_btree.rs +++ b/pageserver/src/tenant/disk_btree.rs @@ -212,6 +212,7 @@ impl<'a, const L: usize> OnDiskNode<'a, L> { /// /// Public reader object, to search the tree. /// +#[derive(Clone)] pub struct DiskBtreeReader where R: BlockReader, @@ -259,27 +260,38 @@ where Ok(result) } - pub fn iter<'a>( - &'a self, - start_key: &'a [u8; L], - ctx: &'a RequestContext, - ) -> DiskBtreeIterator<'a> { + pub fn iter<'a>(self, start_key: &'a [u8; L], ctx: &'a RequestContext) -> DiskBtreeIterator<'a> + where + R: 'a, + { DiskBtreeIterator { - stream: Box::pin(self.get_stream_from(start_key, ctx)), + stream: Box::pin(self.into_stream(start_key, ctx)), } } /// Return a stream which yields all key, value pairs from the index /// starting from the first key greater or equal to `start_key`. /// - /// Note that this is a copy of [`Self::visit`]. + /// Note 1: that this is a copy of [`Self::visit`]. /// TODO: Once the sequential read path is removed this will become /// the only index traversal method. - pub fn get_stream_from<'a>( - &'a self, + /// + /// Note 2: this function used to take `&self` but it now consumes `self`. This is due to + /// the lifetime constraints of the reader and the stream / iterator it creates. Using `&self` + /// requires the reader to be present when the stream is used, and this creates a lifetime + /// dependency between the reader and the stream. Now if we want to create an iterator that + /// holds the stream, someone will need to keep a reference to the reader, which is inconvenient + /// to use from the image/delta layer APIs. + /// + /// Feel free to add the `&self` variant back if it's necessary. + pub fn into_stream<'a>( + self, start_key: &'a [u8; L], ctx: &'a RequestContext, - ) -> impl Stream, u64), DiskBtreeError>> + 'a { + ) -> impl Stream, u64), DiskBtreeError>> + 'a + where + R: 'a, + { try_stream! { let mut stack = Vec::new(); stack.push((self.root_blk, None)); diff --git a/pageserver/src/tenant/mgr.rs b/pageserver/src/tenant/mgr.rs index 4520bb9295..08c3f19b6f 100644 --- a/pageserver/src/tenant/mgr.rs +++ b/pageserver/src/tenant/mgr.rs @@ -3,7 +3,6 @@ use camino::{Utf8DirEntry, Utf8Path, Utf8PathBuf}; use futures::StreamExt; -use hyper::StatusCode; use itertools::Itertools; use pageserver_api::key::Key; use pageserver_api::models::LocationConfigMode; @@ -27,8 +26,7 @@ use tokio::task::JoinSet; use tokio_util::sync::CancellationToken; use tracing::*; -use remote_storage::GenericRemoteStorage; -use utils::{completion, crashsafe}; +use utils::{backoff, completion, crashsafe}; use crate::config::PageServerConf; use crate::context::{DownloadBehavior, RequestContext}; @@ -42,19 +40,17 @@ use crate::task_mgr::{self, TaskKind}; use crate::tenant::config::{ AttachedLocationConfig, AttachmentMode, LocationConf, LocationMode, SecondaryLocationConfig, }; -use crate::tenant::delete::DeleteTenantFlow; use crate::tenant::span::debug_assert_current_span_has_tenant_id; use crate::tenant::storage_layer::inmemory_layer; use crate::tenant::timeline::ShutdownMode; use crate::tenant::{AttachedTenantConf, GcError, SpawnMode, Tenant, TenantState}; -use crate::{InitializationOrder, IGNORED_TENANT_FILE_NAME, TEMP_FILE_SUFFIX}; +use crate::{InitializationOrder, TEMP_FILE_SUFFIX}; use utils::crashsafe::path_with_suffix_extension; use utils::fs_ext::PathExt; use utils::generation::Generation; use utils::id::{TenantId, TimelineId}; -use super::delete::DeleteTenantError; use super::remote_timeline_client::remote_tenant_path; use super::secondary::SecondaryTenant; use super::timeline::detach_ancestor::PreparedTimelineDetach; @@ -112,12 +108,6 @@ pub(crate) enum TenantsMap { ShuttingDown(BTreeMap), } -pub(crate) enum TenantsMapRemoveResult { - Occupied(TenantSlot), - Vacant, - InProgress(utils::completion::Barrier), -} - /// When resolving a TenantId to a shard, we may be looking for the 0th /// shard, or we might be looking for whichever shard holds a particular page. #[derive(Copy, Clone)] @@ -194,26 +184,6 @@ impl TenantsMap { } } - /// Only for use from DeleteTenantFlow. This method directly removes a TenantSlot from the map. - /// - /// The normal way to remove a tenant is using a SlotGuard, which will gracefully remove the guarded - /// slot if the enclosed tenant is shutdown. - pub(crate) fn remove(&mut self, tenant_shard_id: TenantShardId) -> TenantsMapRemoveResult { - use std::collections::btree_map::Entry; - match self { - TenantsMap::Initializing => TenantsMapRemoveResult::Vacant, - TenantsMap::Open(m) | TenantsMap::ShuttingDown(m) => match m.entry(tenant_shard_id) { - Entry::Occupied(entry) => match entry.get() { - TenantSlot::InProgress(barrier) => { - TenantsMapRemoveResult::InProgress(barrier.clone()) - } - _ => TenantsMapRemoveResult::Occupied(entry.remove()), - }, - Entry::Vacant(_entry) => TenantsMapRemoveResult::Vacant, - }, - } - } - #[cfg(all(debug_assertions, not(test)))] pub(crate) fn len(&self) -> usize { match self { @@ -422,12 +392,6 @@ fn load_tenant_config( } }; - let tenant_ignore_mark_file = tenant_dir_path.join(IGNORED_TENANT_FILE_NAME); - if tenant_ignore_mark_file.exists() { - info!("Found an ignore mark file {tenant_ignore_mark_file:?}, skipping the tenant"); - return Ok(None); - } - Ok(Some(( tenant_shard_id, Tenant::load_tenant_config(conf, &tenant_shard_id), @@ -469,6 +433,18 @@ async fn init_load_tenant_configs( Ok(configs) } +#[derive(Debug, thiserror::Error)] +pub(crate) enum DeleteTenantError { + #[error("Tenant map slot error {0}")] + SlotError(#[from] TenantSlotError), + + #[error("Cancelled")] + Cancelled, + + #[error(transparent)] + Other(#[from] anyhow::Error), +} + /// Initialize repositories with locally available timelines. /// Timelines that are only partially available locally (remote storage has more data than this pageserver) /// are scheduled for download and added to the tenant once download is completed. @@ -519,17 +495,8 @@ pub async fn init_tenant_mgr( let mut location_conf = match location_conf { Ok(l) => l, Err(e) => { - warn!(tenant_id=%tenant_shard_id.tenant_id, shard_id=%tenant_shard_id.shard_slug(), "Marking tenant broken, failed to {e:#}"); - - tenants.insert( - tenant_shard_id, - TenantSlot::Attached(Tenant::create_broken_tenant( - conf, - tenant_shard_id, - resources.remote_storage.clone(), - format!("{}", e), - )), - ); + // This should only happen in the case of a serialization bug or critical local I/O error: we cannot load this tenant + error!(tenant_id=%tenant_shard_id.tenant_id, shard_id=%tenant_shard_id.shard_slug(), "Failed to load tenant config, failed to {e:#}"); continue; } }; @@ -638,7 +605,6 @@ pub async fn init_tenant_mgr( AttachedTenantConf::new(location_conf.tenant_conf, attached_conf), shard_identity, Some(init_order.clone()), - &TENANTS, SpawnMode::Lazy, &ctx, ) { @@ -694,7 +660,6 @@ fn tenant_spawn( location_conf: AttachedTenantConf, shard_identity: ShardIdentity, init_order: Option, - tenants: &'static std::sync::RwLock, mode: SpawnMode, ctx: &RequestContext, ) -> anyhow::Result> { @@ -713,30 +678,16 @@ fn tenant_spawn( "Cannot load tenant from empty directory {tenant_path:?}" ); - let tenant_ignore_mark = conf.tenant_ignore_mark_file_path(&tenant_shard_id); - anyhow::ensure!( - !conf.tenant_ignore_mark_file_path(&tenant_shard_id).exists(), - "Cannot load tenant, ignore mark found at {tenant_ignore_mark:?}" - ); - - let remote_storage = resources.remote_storage.clone(); - let tenant = match Tenant::spawn( + let tenant = Tenant::spawn( conf, tenant_shard_id, resources, location_conf, shard_identity, init_order, - tenants, mode, ctx, - ) { - Ok(tenant) => tenant, - Err(e) => { - error!("Failed to spawn tenant {tenant_shard_id}, reason: {e:#}"); - Tenant::create_broken_tenant(conf, tenant_shard_id, remote_storage, format!("{e:#}")) - } - }; + ); Ok(tenant) } @@ -1067,7 +1018,7 @@ impl TenantManager { // not do significant I/O, and shutdowns should be prompt via cancellation tokens. let mut slot_guard = tenant_map_acquire_slot(&tenant_shard_id, TenantSlotAcquireMode::Any) .map_err(|e| match e { - TenantSlotError::AlreadyExists(_, _) | TenantSlotError::NotFound(_) => { + TenantSlotError::NotFound(_) => { unreachable!("Called with mode Any") } TenantSlotError::InProgress => UpsertLocationError::InProgress, @@ -1176,7 +1127,6 @@ impl TenantManager { attached_conf, shard_identity, None, - self.tenants, spawn_mode, ctx, )?; @@ -1298,7 +1248,6 @@ impl TenantManager { AttachedTenantConf::try_from(config)?, shard_identity, None, - self.tenants, SpawnMode::Eager, ctx, )?; @@ -1367,56 +1316,10 @@ impl TenantManager { } } - pub(crate) async fn delete_tenant( + async fn delete_tenant_remote( &self, tenant_shard_id: TenantShardId, - activation_timeout: Duration, - ) -> Result { - super::span::debug_assert_current_span_has_tenant_id(); - // We acquire a SlotGuard during this function to protect against concurrent - // changes while the ::prepare phase of DeleteTenantFlow executes, but then - // have to return the Tenant to the map while the background deletion runs. - // - // TODO: refactor deletion to happen outside the lifetime of a Tenant. - // Currently, deletion requires a reference to the tenants map in order to - // keep the Tenant in the map until deletion is complete, and then remove - // it at the end. - // - // See https://github.com/neondatabase/neon/issues/5080 - - // Tenant deletion can happen two ways: - // - Legacy: called on an attached location. The attached Tenant object stays alive in Stopping - // state until deletion is complete. - // - New: called on a pageserver without an attached location. We proceed with deletion from - // remote storage. - // - // See https://github.com/neondatabase/neon/issues/5080 for more context on this transition. - - let slot_guard = tenant_map_acquire_slot(&tenant_shard_id, TenantSlotAcquireMode::Any)?; - match &slot_guard.old_value { - Some(TenantSlot::Attached(tenant)) => { - // Legacy deletion flow: the tenant remains attached, goes to Stopping state, and - // deletion will be resumed across restarts. - let tenant = tenant.clone(); - return self - .delete_tenant_attached(slot_guard, tenant, activation_timeout) - .await; - } - Some(TenantSlot::Secondary(secondary_tenant)) => { - secondary_tenant.shutdown().await; - let local_tenant_directory = self.conf.tenant_path(&tenant_shard_id); - let tmp_dir = safe_rename_tenant_dir(&local_tenant_directory) - .await - .with_context(|| { - format!("local tenant directory {local_tenant_directory:?} rename") - })?; - spawn_background_purge(tmp_dir); - } - Some(TenantSlot::InProgress(_)) => unreachable!(), - None => {} - }; - - // Fall through: local state for this tenant is no longer present, proceed with remote delete + ) -> Result<(), DeleteTenantError> { let remote_path = remote_tenant_path(&tenant_shard_id); let keys = match self .resources @@ -1433,7 +1336,7 @@ impl TenantManager { Err(remote_storage::DownloadError::Cancelled) => { return Err(DeleteTenantError::Cancelled) } - Err(remote_storage::DownloadError::NotFound) => return Ok(StatusCode::NOT_FOUND), + Err(remote_storage::DownloadError::NotFound) => return Ok(()), Err(other) => return Err(DeleteTenantError::Other(anyhow::anyhow!(other))), }; @@ -1447,60 +1350,83 @@ impl TenantManager { .await?; } - // Callers use 404 as success for deletions, for historical reasons. - Ok(StatusCode::NOT_FOUND) + Ok(()) } - async fn delete_tenant_attached( + /// If a tenant is attached, detach it. Then remove its data from remote storage. + /// + /// A tenant is considered deleted once it is gone from remote storage. It is the caller's + /// responsibility to avoid trying to attach the tenant again or use it any way once deletion + /// has started: this operation is not atomic, and must be retried until it succeeds. + pub(crate) async fn delete_tenant( &self, - slot_guard: SlotGuard, - tenant: Arc, - activation_timeout: Duration, - ) -> Result { - match tenant.current_state() { - TenantState::Broken { .. } | TenantState::Stopping { .. } => { - // If deletion is already in progress, return success (the semantics of this - // function are to rerturn success afterr deletion is spawned in background). - // Otherwise fall through and let [`DeleteTenantFlow`] handle this state. - if DeleteTenantFlow::is_in_progress(&tenant) { - // The `delete_progress` lock is held: deletion is already happening - // in the bacckground - slot_guard.revert(); - return Ok(StatusCode::ACCEPTED); - } - } - _ => { - tenant - .wait_to_become_active(activation_timeout) - .await - .map_err(|e| match e { - GetActiveTenantError::WillNotBecomeActive(_) - | GetActiveTenantError::Broken(_) => { - DeleteTenantError::InvalidState(tenant.current_state()) - } - GetActiveTenantError::Cancelled => DeleteTenantError::Cancelled, - GetActiveTenantError::NotFound(_) => DeleteTenantError::NotAttached, - GetActiveTenantError::WaitForActiveTimeout { - latest_state: _latest_state, - wait_time: _wait_time, - } => DeleteTenantError::InvalidState(tenant.current_state()), - })?; - } + tenant_shard_id: TenantShardId, + ) -> Result<(), DeleteTenantError> { + super::span::debug_assert_current_span_has_tenant_id(); + + async fn delete_local( + conf: &PageServerConf, + tenant_shard_id: &TenantShardId, + ) -> anyhow::Result<()> { + let local_tenant_directory = conf.tenant_path(tenant_shard_id); + let tmp_dir = safe_rename_tenant_dir(&local_tenant_directory) + .await + .with_context(|| { + format!("local tenant directory {local_tenant_directory:?} rename") + })?; + spawn_background_purge(tmp_dir); + Ok(()) } - let result = DeleteTenantFlow::run( - self.conf, - self.resources.remote_storage.clone(), - &TENANTS, - tenant, + let slot_guard = tenant_map_acquire_slot(&tenant_shard_id, TenantSlotAcquireMode::Any)?; + match &slot_guard.old_value { + Some(TenantSlot::Attached(tenant)) => { + // Legacy deletion flow: the tenant remains attached, goes to Stopping state, and + // deletion will be resumed across restarts. + let tenant = tenant.clone(); + let (_guard, progress) = utils::completion::channel(); + match tenant.shutdown(progress, ShutdownMode::Hard).await { + Ok(()) => {} + Err(barrier) => { + info!("Shutdown already in progress, waiting for it to complete"); + barrier.wait().await; + } + } + delete_local(self.conf, &tenant_shard_id).await?; + } + Some(TenantSlot::Secondary(secondary_tenant)) => { + secondary_tenant.shutdown().await; + + delete_local(self.conf, &tenant_shard_id).await?; + } + Some(TenantSlot::InProgress(_)) => unreachable!(), + None => {} + }; + + // Fall through: local state for this tenant is no longer present, proceed with remote delete. + // - We use a retry wrapper here so that common transient S3 errors (e.g. 503, 429) do not result + // in 500 responses to delete requests. + // - We keep the `SlotGuard` during this I/O, so that if a concurrent delete request comes in, it will + // 503/retry, rather than kicking off a wasteful concurrent deletion. + match backoff::retry( + || async move { self.delete_tenant_remote(tenant_shard_id).await }, + |e| match e { + DeleteTenantError::Cancelled => true, + DeleteTenantError::SlotError(_) => { + unreachable!("Remote deletion doesn't touch slots") + } + _ => false, + }, + 1, + 3, + &format!("delete_tenant[tenant_shard_id={tenant_shard_id}]"), &self.cancel, ) - .await; - - // The Tenant goes back into the map in Stopping state, it will eventually be removed by DeleteTenantFLow - slot_guard.revert(); - let () = result?; - Ok(StatusCode::ACCEPTED) + .await + { + Some(r) => r, + None => Err(DeleteTenantError::Cancelled), + } } #[instrument(skip_all, fields(tenant_id=%tenant.get_tenant_shard_id().tenant_id, shard_id=%tenant.get_tenant_shard_id().shard_slug(), new_shard_count=%new_shard_count.literal()))] @@ -1672,7 +1598,7 @@ impl TenantManager { for child_shard_id in &child_shards { let child_shard_id = *child_shard_id; let child_shard = { - let locked = TENANTS.read().unwrap(); + let locked = self.tenants.read().unwrap(); let peek_slot = tenant_map_peek_slot(&locked, &child_shard_id, TenantSlotPeekMode::Read)?; peek_slot.and_then(|s| s.get_attached()).cloned() @@ -1773,6 +1699,7 @@ impl TenantManager { let timelines = parent_shard.timelines.lock().unwrap().clone(); let parent_timelines = timelines.keys().cloned().collect::>(); for timeline in timelines.values() { + tracing::info!(timeline_id=%timeline.timeline_id, "Loading list of layers to hardlink"); let timeline_layers = timeline .layers .read() @@ -1812,7 +1739,12 @@ impl TenantManager { // Since we will do a large number of small filesystem metadata operations, batch them into // spawn_blocking calls rather than doing each one as a tokio::fs round-trip. + let span = tracing::Span::current(); let jh = tokio::task::spawn_blocking(move || -> anyhow::Result { + // Run this synchronous code in the same log context as the outer function that spawned it. + let _span = span.enter(); + + tracing::info!("Creating {} directories", create_dirs.len()); for dir in &create_dirs { if let Err(e) = std::fs::create_dir_all(dir) { // Ignore AlreadyExists errors, drop out on all other errors @@ -1826,6 +1758,11 @@ impl TenantManager { } for child_prefix in child_prefixes { + tracing::info!( + "Hard-linking {} parent layers into child path {}", + parent_layers.len(), + child_prefix + ); for relative_layer in &parent_layers { let parent_path = parent_path.join(relative_layer); let child_path = child_prefix.join(relative_layer); @@ -1851,6 +1788,7 @@ impl TenantManager { // Durability is not required for correctness, but if we crashed during split and // then came restarted with empty timeline dirs, it would be very inefficient to // re-populate from remote storage. + tracing::info!("fsyncing {} directories", create_dirs.len()); for dir in create_dirs { if let Err(e) = crashsafe::fsync(&dir) { // Something removed a newly created timeline dir out from underneath us? Extremely @@ -1901,17 +1839,10 @@ impl TenantManager { &self, conf: &'static PageServerConf, tenant_shard_id: TenantShardId, - detach_ignored: bool, deletion_queue_client: &DeletionQueueClient, ) -> Result<(), TenantStateError> { let tmp_path = self - .detach_tenant0( - conf, - &TENANTS, - tenant_shard_id, - detach_ignored, - deletion_queue_client, - ) + .detach_tenant0(conf, tenant_shard_id, deletion_queue_client) .await?; spawn_background_purge(tmp_path); @@ -1921,9 +1852,7 @@ impl TenantManager { async fn detach_tenant0( &self, conf: &'static PageServerConf, - tenants: &std::sync::RwLock, tenant_shard_id: TenantShardId, - detach_ignored: bool, deletion_queue_client: &DeletionQueueClient, ) -> Result { let tenant_dir_rename_operation = |tenant_id_to_clean: TenantShardId| async move { @@ -1936,7 +1865,7 @@ impl TenantManager { }; let removal_result = remove_tenant_from_memory( - tenants, + self.tenants, tenant_shard_id, tenant_dir_rename_operation(tenant_shard_id), ) @@ -1946,33 +1875,13 @@ impl TenantManager { // before this tenant is potentially re-attached elsewhere. deletion_queue_client.flush_advisory(); - // Ignored tenants are not present in memory and will bail the removal from memory operation. - // Before returning the error, check for ignored tenant removal case — we only need to clean its local files then. - if detach_ignored - && matches!( - removal_result, - Err(TenantStateError::SlotError(TenantSlotError::NotFound(_))) - ) - { - let tenant_ignore_mark = conf.tenant_ignore_mark_file_path(&tenant_shard_id); - if tenant_ignore_mark.exists() { - info!("Detaching an ignored tenant"); - let tmp_path = tenant_dir_rename_operation(tenant_shard_id) - .await - .with_context(|| { - format!("Ignored tenant {tenant_shard_id} local directory rename") - })?; - return Ok(tmp_path); - } - } - removal_result } pub(crate) fn list_tenants( &self, ) -> Result, TenantMapListError> { - let tenants = TENANTS.read().unwrap(); + let tenants = self.tenants.read().unwrap(); let m = match &*tenants { TenantsMap::Initializing => return Err(TenantMapListError::Initializing), TenantsMap::Open(m) | TenantsMap::ShuttingDown(m) => m, @@ -2073,7 +1982,6 @@ impl TenantManager { AttachedTenantConf::try_from(config)?, shard_identity, None, - self.tenants, SpawnMode::Eager, ctx, )?; @@ -2222,97 +2130,6 @@ pub(crate) enum TenantStateError { Other(#[from] anyhow::Error), } -pub(crate) async fn load_tenant( - conf: &'static PageServerConf, - tenant_id: TenantId, - generation: Generation, - broker_client: storage_broker::BrokerClientChannel, - remote_storage: GenericRemoteStorage, - deletion_queue_client: DeletionQueueClient, - ctx: &RequestContext, -) -> Result<(), TenantMapInsertError> { - // This is a legacy API (replaced by `/location_conf`). It does not support sharding - let tenant_shard_id = TenantShardId::unsharded(tenant_id); - - let slot_guard = - tenant_map_acquire_slot(&tenant_shard_id, TenantSlotAcquireMode::MustNotExist)?; - let tenant_path = conf.tenant_path(&tenant_shard_id); - - let tenant_ignore_mark = conf.tenant_ignore_mark_file_path(&tenant_shard_id); - if tenant_ignore_mark.exists() { - std::fs::remove_file(&tenant_ignore_mark).with_context(|| { - format!( - "Failed to remove tenant ignore mark {tenant_ignore_mark:?} during tenant loading" - ) - })?; - } - - let resources = TenantSharedResources { - broker_client, - remote_storage, - deletion_queue_client, - }; - - let mut location_conf = - Tenant::load_tenant_config(conf, &tenant_shard_id).map_err(TenantMapInsertError::Other)?; - location_conf.attach_in_generation(AttachmentMode::Single, generation); - - Tenant::persist_tenant_config(conf, &tenant_shard_id, &location_conf).await?; - - let shard_identity = location_conf.shard; - let new_tenant = tenant_spawn( - conf, - tenant_shard_id, - &tenant_path, - resources, - AttachedTenantConf::try_from(location_conf)?, - shard_identity, - None, - &TENANTS, - SpawnMode::Eager, - ctx, - ) - .with_context(|| format!("Failed to schedule tenant processing in path {tenant_path:?}"))?; - - slot_guard.upsert(TenantSlot::Attached(new_tenant))?; - Ok(()) -} - -pub(crate) async fn ignore_tenant( - conf: &'static PageServerConf, - tenant_id: TenantId, -) -> Result<(), TenantStateError> { - ignore_tenant0(conf, &TENANTS, tenant_id).await -} - -#[instrument(skip_all, fields(shard_id))] -async fn ignore_tenant0( - conf: &'static PageServerConf, - tenants: &std::sync::RwLock, - tenant_id: TenantId, -) -> Result<(), TenantStateError> { - // This is a legacy API (replaced by `/location_conf`). It does not support sharding - let tenant_shard_id = TenantShardId::unsharded(tenant_id); - tracing::Span::current().record( - "shard_id", - tracing::field::display(tenant_shard_id.shard_slug()), - ); - - remove_tenant_from_memory(tenants, tenant_shard_id, async { - let ignore_mark_file = conf.tenant_ignore_mark_file_path(&tenant_shard_id); - fs::File::create(&ignore_mark_file) - .await - .context("Failed to create ignore mark file") - .and_then(|_| { - crashsafe::fsync_file_and_parent(&ignore_mark_file) - .context("Failed to fsync ignore mark file") - }) - .with_context(|| format!("Failed to crate ignore mark for tenant {tenant_shard_id}"))?; - Ok(()) - }) - .await -} - #[derive(Debug, thiserror::Error)] pub(crate) enum TenantMapListError { #[error("tenant map is still initiailizing")] @@ -2337,10 +2154,6 @@ pub(crate) enum TenantSlotError { #[error("Tenant {0} not found")] NotFound(TenantShardId), - /// When acquiring a slot with the expectation that the tenant does not already exist. - #[error("tenant {0} already exists, state: {1:?}")] - AlreadyExists(TenantShardId, TenantState), - // Tried to read a slot that is currently being mutated by another administrative // operation. #[error("tenant has a state change in progress, try again later")] @@ -2656,8 +2469,6 @@ enum TenantSlotAcquireMode { Any, /// Return an error if trying to acquire a slot and it doesn't already exist MustExist, - /// Return an error if trying to acquire a slot and it already exists - MustNotExist, } fn tenant_map_acquire_slot( @@ -2711,27 +2522,6 @@ fn tenant_map_acquire_slot_impl( tracing::debug!("Occupied, failing for InProgress"); Err(TenantSlotError::InProgress) } - (slot, MustNotExist) => match slot { - TenantSlot::Attached(tenant) => { - tracing::debug!("Attached && MustNotExist, return AlreadyExists"); - Err(TenantSlotError::AlreadyExists( - *tenant_shard_id, - tenant.current_state(), - )) - } - _ => { - // FIXME: the AlreadyExists error assumes that we have a Tenant - // to get the state from - tracing::debug!("Occupied & MustNotExist, return AlreadyExists"); - Err(TenantSlotError::AlreadyExists( - *tenant_shard_id, - TenantState::Broken { - reason: "Present but not attached".to_string(), - backtrace: "".to_string(), - }, - )) - } - }, _ => { // Happy case: the slot was not in any state that violated our mode let (completion, barrier) = utils::completion::channel(); diff --git a/pageserver/src/tenant/secondary/heatmap_uploader.rs b/pageserver/src/tenant/secondary/heatmap_uploader.rs index 9c7a9c4234..0aad5bf392 100644 --- a/pageserver/src/tenant/secondary/heatmap_uploader.rs +++ b/pageserver/src/tenant/secondary/heatmap_uploader.rs @@ -367,10 +367,9 @@ async fn upload_tenant_heatmap( debug_assert_current_span_has_tenant_id(); let generation = tenant.get_generation(); + debug_assert!(!generation.is_none()); if generation.is_none() { - // We do not expect this: generations were implemented before heatmap uploads. However, - // handle it so that we don't have to make the generation in the heatmap an Option<> - // (Generation::none is not serializable) + // We do not expect this: None generations should only appear in historic layer metadata, not in running Tenants tracing::warn!("Skipping heatmap upload for tenant with generation==None"); return Ok(UploadHeatmapOutcome::Skipped); } diff --git a/pageserver/src/tenant/storage_layer/delta_layer.rs b/pageserver/src/tenant/storage_layer/delta_layer.rs index 5e01ecd71d..bf5d9249eb 100644 --- a/pageserver/src/tenant/storage_layer/delta_layer.rs +++ b/pageserver/src/tenant/storage_layer/delta_layer.rs @@ -928,7 +928,6 @@ impl DeltaLayerInner { } /// Load all key-values in the delta layer, should be replaced by an iterator-based interface in the future. - #[cfg(test)] pub(super) async fn load_key_values( &self, ctx: &RequestContext, @@ -941,7 +940,7 @@ impl DeltaLayerInner { ); let mut result = Vec::new(); let mut stream = - Box::pin(self.stream_index_forwards(&index_reader, &[0; DELTA_KEY_SIZE], ctx)); + Box::pin(self.stream_index_forwards(index_reader, &[0; DELTA_KEY_SIZE], ctx)); let block_reader = FileBlockReader::new(&self.file, self.file_id); let cursor = block_reader.block_cursor(); let mut buf = Vec::new(); @@ -976,7 +975,7 @@ impl DeltaLayerInner { ctx: &RequestContext, ) -> anyhow::Result> where - Reader: BlockReader, + Reader: BlockReader + Clone, { let ctx = RequestContextBuilder::extend(ctx) .page_content_kind(PageContentKind::DeltaLayerBtreeNode) @@ -986,7 +985,7 @@ impl DeltaLayerInner { let mut range_end_handled = false; let start_key = DeltaKey::from_key_lsn(&range.start, lsn_range.start); - let index_stream = index_reader.get_stream_from(&start_key.0, &ctx); + let index_stream = index_reader.clone().into_stream(&start_key.0, &ctx); let mut index_stream = std::pin::pin!(index_stream); while let Some(index_entry) = index_stream.next().await { @@ -1241,7 +1240,7 @@ impl DeltaLayerInner { block_reader, ); - let stream = self.stream_index_forwards(&tree_reader, &[0u8; DELTA_KEY_SIZE], ctx); + let stream = self.stream_index_forwards(tree_reader, &[0u8; DELTA_KEY_SIZE], ctx); let stream = stream.map_ok(|(key, lsn, pos)| Item::Actual(key, lsn, pos)); // put in a sentinel value for getting the end offset for last item, and not having to // repeat the whole read part @@ -1300,7 +1299,7 @@ impl DeltaLayerInner { offsets.start.pos(), offsets.end.pos(), meta, - max_read_size, + Some(max_read_size), )) } } else { @@ -1459,17 +1458,17 @@ impl DeltaLayerInner { fn stream_index_forwards<'a, R>( &'a self, - reader: &'a DiskBtreeReader, + reader: DiskBtreeReader, start: &'a [u8; DELTA_KEY_SIZE], ctx: &'a RequestContext, ) -> impl futures::stream::Stream< Item = Result<(Key, Lsn, BlobRef), crate::tenant::disk_btree::DiskBtreeError>, > + 'a where - R: BlockReader, + R: BlockReader + 'a, { use futures::stream::TryStreamExt; - let stream = reader.get_stream_from(start, ctx); + let stream = reader.into_stream(start, ctx); stream.map_ok(|(key, value)| { let key = DeltaKey::from_slice(&key); let (key, lsn) = (key.key(), key.lsn()); @@ -1857,7 +1856,7 @@ mod test { .finish(entries_meta.key_range.end, &timeline, &ctx) .await?; - let inner = resident.as_delta(&ctx).await?; + let inner = resident.get_as_delta(&ctx).await?; let file_size = inner.file.metadata().await?.len(); tracing::info!( @@ -2044,11 +2043,11 @@ mod test { let copied_layer = writer.finish(Key::MAX, &branch, ctx).await.unwrap(); - copied_layer.as_delta(ctx).await.unwrap(); + copied_layer.get_as_delta(ctx).await.unwrap(); assert_keys_and_values_eq( - new_layer.as_delta(ctx).await.unwrap(), - copied_layer.as_delta(ctx).await.unwrap(), + new_layer.get_as_delta(ctx).await.unwrap(), + copied_layer.get_as_delta(ctx).await.unwrap(), truncate_at, ctx, ) @@ -2073,7 +2072,7 @@ mod test { source.index_root_blk, &source_reader, ); - let source_stream = source.stream_index_forwards(&source_tree, &start_key, ctx); + let source_stream = source.stream_index_forwards(source_tree, &start_key, ctx); let source_stream = source_stream.filter(|res| match res { Ok((_, lsn, _)) => ready(lsn < &truncated_at), _ => ready(true), @@ -2086,7 +2085,7 @@ mod test { truncated.index_root_blk, &truncated_reader, ); - let truncated_stream = truncated.stream_index_forwards(&truncated_tree, &start_key, ctx); + let truncated_stream = truncated.stream_index_forwards(truncated_tree, &start_key, ctx); let mut truncated_stream = std::pin::pin!(truncated_stream); let mut scratch_left = Vec::new(); diff --git a/pageserver/src/tenant/storage_layer/image_layer.rs b/pageserver/src/tenant/storage_layer/image_layer.rs index 06e2f09384..50aacbd9ad 100644 --- a/pageserver/src/tenant/storage_layer/image_layer.rs +++ b/pageserver/src/tenant/storage_layer/image_layer.rs @@ -486,7 +486,6 @@ impl ImageLayerInner { } /// Load all key-values in the delta layer, should be replaced by an iterator-based interface in the future. - #[cfg(test)] pub(super) async fn load_key_values( &self, ctx: &RequestContext, @@ -495,7 +494,7 @@ impl ImageLayerInner { let tree_reader = DiskBtreeReader::new(self.index_start_blk, self.index_root_blk, &block_reader); let mut result = Vec::new(); - let mut stream = Box::pin(tree_reader.get_stream_from(&[0; KEY_SIZE], ctx)); + let mut stream = Box::pin(tree_reader.into_stream(&[0; KEY_SIZE], ctx)); let block_reader = FileBlockReader::new(&self.file, self.file_id); let cursor = block_reader.block_cursor(); while let Some(item) = stream.next().await { @@ -544,7 +543,7 @@ impl ImageLayerInner { let mut search_key: [u8; KEY_SIZE] = [0u8; KEY_SIZE]; range.start.write_to_byte_slice(&mut search_key); - let index_stream = tree_reader.get_stream_from(&search_key, &ctx); + let index_stream = tree_reader.clone().into_stream(&search_key, &ctx); let mut index_stream = std::pin::pin!(index_stream); while let Some(index_entry) = index_stream.next().await { @@ -689,6 +688,24 @@ impl ImageLayerInner { }; } } + + #[cfg(test)] + pub(crate) fn iter<'a>(&'a self, ctx: &'a RequestContext) -> ImageLayerIterator<'a> { + let block_reader = FileBlockReader::new(&self.file, self.file_id); + let tree_reader = + DiskBtreeReader::new(self.index_start_blk, self.index_root_blk, block_reader); + ImageLayerIterator { + image_layer: self, + ctx, + index_iter: tree_reader.iter(&[0; KEY_SIZE], ctx), + key_values_batch: std::collections::VecDeque::new(), + is_end: false, + planner: crate::tenant::vectored_blob_io::StreamingVectoredReadPlanner::new( + 1024 * 8192, // The default value. Unit tests might use a different value. 1024 * 8K = 8MB buffer. + 1024, // The default value. Unit tests might use a different value + ), + } + } } /// A builder object for constructing a new image layer. @@ -943,11 +960,77 @@ impl Drop for ImageLayerWriter { } } +#[cfg(test)] +pub struct ImageLayerIterator<'a> { + image_layer: &'a ImageLayerInner, + ctx: &'a RequestContext, + planner: crate::tenant::vectored_blob_io::StreamingVectoredReadPlanner, + index_iter: crate::tenant::disk_btree::DiskBtreeIterator<'a>, + key_values_batch: std::collections::VecDeque<(Key, Lsn, Value)>, + is_end: bool, +} + +#[cfg(test)] +impl<'a> ImageLayerIterator<'a> { + /// Retrieve a batch of key-value pairs into the iterator buffer. + async fn next_batch(&mut self) -> anyhow::Result<()> { + assert!(self.key_values_batch.is_empty()); + assert!(!self.is_end); + + let plan = loop { + if let Some(res) = self.index_iter.next().await { + let (raw_key, offset) = res?; + if let Some(batch_plan) = self.planner.handle( + Key::from_slice(&raw_key[..KEY_SIZE]), + self.image_layer.lsn, + offset, + BlobFlag::None, + ) { + break batch_plan; + } + } else { + self.is_end = true; + let payload_end = self.image_layer.index_start_blk as u64 * PAGE_SZ as u64; + break self.planner.handle_range_end(payload_end); + } + }; + let vectored_blob_reader = VectoredBlobReader::new(&self.image_layer.file); + let mut next_batch = std::collections::VecDeque::new(); + let buf_size = plan.size(); + let buf = BytesMut::with_capacity(buf_size); + let blobs_buf = vectored_blob_reader + .read_blobs(&plan, buf, self.ctx) + .await?; + let frozen_buf: Bytes = blobs_buf.buf.freeze(); + for meta in blobs_buf.blobs.iter() { + let img_buf = frozen_buf.slice(meta.start..meta.end); + next_batch.push_back((meta.meta.key, self.image_layer.lsn, Value::Image(img_buf))); + } + self.key_values_batch = next_batch; + Ok(()) + } + + pub async fn next(&mut self) -> anyhow::Result> { + if self.key_values_batch.is_empty() { + if self.is_end { + return Ok(None); + } + self.next_batch().await?; + } + Ok(Some( + self.key_values_batch + .pop_front() + .expect("should not be empty"), + )) + } +} + #[cfg(test)] mod test { - use std::time::Duration; + use std::{sync::Arc, time::Duration}; use bytes::Bytes; + use itertools::Itertools; use pageserver_api::{ key::Key, shard::{ShardCount, ShardIdentity, ShardNumber, ShardStripeSize}, @@ -959,11 +1042,19 @@ mod test { }; use crate::{ - tenant::{config::TenantConf, harness::TenantHarness}, + context::RequestContext, + repository::Value, + tenant::{ + config::TenantConf, + harness::{TenantHarness, TIMELINE_ID}, + storage_layer::ResidentLayer, + vectored_blob_io::StreamingVectoredReadPlanner, + Tenant, Timeline, + }, DEFAULT_PG_VERSION, }; - use super::ImageLayerWriter; + use super::{ImageLayerIterator, ImageLayerWriter}; #[tokio::test] async fn image_layer_rewrite() { @@ -1134,4 +1225,111 @@ mod test { } } } + + async fn produce_image_layer( + tenant: &Tenant, + tline: &Arc, + mut images: Vec<(Key, Bytes)>, + lsn: Lsn, + ctx: &RequestContext, + ) -> anyhow::Result { + images.sort(); + let (key_start, _) = images.first().unwrap(); + let (key_last, _) = images.last().unwrap(); + let key_end = key_last.next(); + let key_range = *key_start..key_end; + let mut writer = ImageLayerWriter::new( + tenant.conf, + tline.timeline_id, + tenant.tenant_shard_id, + &key_range, + lsn, + ctx, + ) + .await?; + + for (key, img) in images { + writer.put_image(key, img, ctx).await?; + } + let img_layer = writer.finish(tline, ctx).await?; + + Ok::<_, anyhow::Error>(img_layer) + } + + async fn assert_img_iter_equal( + img_iter: &mut ImageLayerIterator<'_>, + expect: &[(Key, Bytes)], + expect_lsn: Lsn, + ) { + let mut expect_iter = expect.iter(); + loop { + let o1 = img_iter.next().await.unwrap(); + let o2 = expect_iter.next(); + match (o1, o2) { + (None, None) => break, + (Some((k1, l1, v1)), Some((k2, i2))) => { + let Value::Image(i1) = v1 else { + panic!("expect Value::Image") + }; + assert_eq!(&k1, k2); + assert_eq!(l1, expect_lsn); + assert_eq!(&i1, i2); + } + (o1, o2) => panic!("iterators length mismatch: {:?}, {:?}", o1, o2), + } + } + } + + #[tokio::test] + async fn image_layer_iterator() { + let harness = TenantHarness::create("image_layer_iterator").unwrap(); + let (tenant, ctx) = harness.load().await; + + let tline = tenant + .create_test_timeline(TIMELINE_ID, Lsn(0x10), DEFAULT_PG_VERSION, &ctx) + .await + .unwrap(); + + fn get_key(id: u32) -> Key { + let mut key = Key::from_hex("000000000033333333444444445500000000").unwrap(); + key.field6 = id; + key + } + const N: usize = 1000; + let test_imgs = (0..N) + .map(|idx| (get_key(idx as u32), Bytes::from(format!("img{idx:05}")))) + .collect_vec(); + let resident_layer = + produce_image_layer(&tenant, &tline, test_imgs.clone(), Lsn(0x10), &ctx) + .await + .unwrap(); + let img_layer = resident_layer.get_as_image(&ctx).await.unwrap(); + for max_read_size in [1, 1024] { + for batch_size in [1, 2, 4, 8, 3, 7, 13] { + println!("running with batch_size={batch_size} max_read_size={max_read_size}"); + // Test if the batch size is correctly determined + let mut iter = img_layer.iter(&ctx); + iter.planner = StreamingVectoredReadPlanner::new(max_read_size, batch_size); + let mut num_items = 0; + for _ in 0..3 { + iter.next_batch().await.unwrap(); + num_items += iter.key_values_batch.len(); + if max_read_size == 1 { + // every key should be a batch b/c the value is larger than max_read_size + assert_eq!(iter.key_values_batch.len(), 1); + } else { + assert_eq!(iter.key_values_batch.len(), batch_size); + } + if num_items >= N { + break; + } + iter.key_values_batch.clear(); + } + // Test if the result is correct + let mut iter = img_layer.iter(&ctx); + iter.planner = StreamingVectoredReadPlanner::new(max_read_size, batch_size); + assert_img_iter_equal(&mut iter, &test_imgs, Lsn(0x10)).await; + } + } + } } diff --git a/pageserver/src/tenant/storage_layer/inmemory_layer.rs b/pageserver/src/tenant/storage_layer/inmemory_layer.rs index 1ecc56ce99..6624fb7e6b 100644 --- a/pageserver/src/tenant/storage_layer/inmemory_layer.rs +++ b/pageserver/src/tenant/storage_layer/inmemory_layer.rs @@ -622,18 +622,16 @@ impl InMemoryLayer { let end_lsn = *self.end_lsn.get().unwrap(); - let keys: Vec<_> = if let Some(key_range) = key_range { + let key_count = if let Some(key_range) = key_range { inner .index .iter() .filter(|(k, _)| key_range.contains(k)) - .map(|(k, m)| (k.to_i128(), m)) - .collect() + .count() } else { - inner.index.iter().map(|(k, m)| (k.to_i128(), m)).collect() + inner.index.len() }; - - if keys.is_empty() { + if key_count == 0 { return Ok(None); } diff --git a/pageserver/src/tenant/storage_layer/layer.rs b/pageserver/src/tenant/storage_layer/layer.rs index 32acb3f0cd..5dd9472535 100644 --- a/pageserver/src/tenant/storage_layer/layer.rs +++ b/pageserver/src/tenant/storage_layer/layer.rs @@ -93,16 +93,12 @@ pub(crate) struct Layer(Arc); impl std::fmt::Display for Layer { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - if matches!(self.0.generation, Generation::Broken) { - write!(f, "{}-broken", self.layer_desc().short_id()) - } else { - write!( - f, - "{}{}", - self.layer_desc().short_id(), - self.0.generation.get_suffix() - ) - } + write!( + f, + "{}{}", + self.layer_desc().short_id(), + self.0.generation.get_suffix() + ) } } @@ -389,7 +385,6 @@ impl Layer { } /// Get all key/values in the layer. Should be replaced with an iterator-based API in the future. - #[cfg(test)] pub(crate) async fn load_key_values( &self, ctx: &RequestContext, @@ -1774,7 +1769,6 @@ impl DownloadedLayer { } } - #[cfg(test)] async fn load_key_values( &self, owner: &Arc, @@ -1905,7 +1899,7 @@ impl ResidentLayer { } #[cfg(test)] - pub(crate) async fn as_delta( + pub(crate) async fn get_as_delta( &self, ctx: &RequestContext, ) -> anyhow::Result<&delta_layer::DeltaLayerInner> { @@ -1915,6 +1909,18 @@ impl ResidentLayer { Image(_) => Err(anyhow::anyhow!("image layer")), } } + + #[cfg(test)] + pub(crate) async fn get_as_image( + &self, + ctx: &RequestContext, + ) -> anyhow::Result<&image_layer::ImageLayerInner> { + use LayerKind::*; + match self.downloaded.get(&self.owner.0, ctx).await? { + Image(ref d) => Ok(d), + Delta(_) => Err(anyhow::anyhow!("delta layer")), + } + } } impl AsLayerDesc for ResidentLayer { diff --git a/pageserver/src/tenant/timeline.rs b/pageserver/src/tenant/timeline.rs index a4f1108635..1175b75017 100644 --- a/pageserver/src/tenant/timeline.rs +++ b/pageserver/src/tenant/timeline.rs @@ -101,9 +101,7 @@ use crate::{ use crate::config::PageServerConf; use crate::keyspace::{KeyPartitioning, KeySpace}; -use crate::metrics::{ - TimelineMetrics, MATERIALIZED_PAGE_CACHE_HIT, MATERIALIZED_PAGE_CACHE_HIT_DIRECT, -}; +use crate::metrics::TimelineMetrics; use crate::pgdatadir_mapping::CalculateLogicalSizeError; use crate::tenant::config::TenantConfOpt; use pageserver_api::reltag::RelTag; @@ -120,7 +118,6 @@ use utils::{ simple_rcu::{Rcu, RcuReadGuard}, }; -use crate::page_cache; use crate::repository::GcResult; use crate::repository::{Key, Value}; use crate::task_mgr; @@ -134,7 +131,7 @@ use self::layer_manager::LayerManager; use self::logical_size::LogicalSize; use self::walreceiver::{WalReceiver, WalReceiverConf}; -use super::{config::TenantConf, storage_layer::VectoredValueReconstructState}; +use super::config::TenantConf; use super::{debug_assert_current_span_has_tenant_and_timeline_id, AttachedTenantConf}; use super::{remote_timeline_client::index::IndexPart, storage_layer::LayerFringe}; use super::{remote_timeline_client::RemoteTimelineClient, storage_layer::ReadableLayer}; @@ -689,6 +686,7 @@ pub enum GetLogicalSizePriority { pub(crate) enum CompactFlags { ForceRepartition, ForceImageLayerCreation, + EnhancedGcBottomMostCompaction, } impl std::fmt::Debug for Timeline { @@ -887,32 +885,11 @@ impl Timeline { self.timeline_get_throttle.throttle(ctx, 1).await; - // Check the page cache. We will get back the most recent page with lsn <= `lsn`. - // The cached image can be returned directly if there is no WAL between the cached image - // and requested LSN. The cached image can also be used to reduce the amount of WAL needed - // for redo. - let cached_page_img = match self.lookup_cached_page(&key, lsn, ctx).await { - Some((cached_lsn, cached_img)) => { - match cached_lsn.cmp(&lsn) { - Ordering::Less => {} // there might be WAL between cached_lsn and lsn, we need to check - Ordering::Equal => { - MATERIALIZED_PAGE_CACHE_HIT_DIRECT.inc(); - return Ok(cached_img); // exact LSN match, return the image - } - Ordering::Greater => { - unreachable!("the returned lsn should never be after the requested lsn") - } - } - Some((cached_lsn, cached_img)) - } - None => None, - }; - match self.conf.get_impl { GetImpl::Legacy => { let reconstruct_state = ValueReconstructState { records: Vec::new(), - img: cached_page_img, + img: None, }; self.get_impl(key, lsn, reconstruct_state, ctx).await @@ -926,13 +903,6 @@ impl Timeline { // entry returned above. let mut reconstruct_state = ValuesReconstructState::new(); - // Only add the cached image to the reconstruct state when it exists. - if cached_page_img.is_some() { - let mut key_state = VectoredValueReconstructState::default(); - key_state.img = cached_page_img; - reconstruct_state.keys.insert(key, Ok(key_state)); - } - let vectored_res = self .get_vectored_impl(keyspace.clone(), lsn, &mut reconstruct_state, ctx) .await; @@ -1127,7 +1097,6 @@ impl Timeline { /// scan iterator interface. We could optimize this interface later to avoid some checks in the vectored /// get path to maintain and split the probing and to-be-probe keyspace. We also need to ensure that /// the scan operation will not cause OOM in the future. - #[allow(dead_code)] pub(crate) async fn scan( &self, keyspace: KeySpace, @@ -3240,7 +3209,6 @@ impl Timeline { ValueReconstructResult::Continue => { // If we reached an earlier cached page image, we're done. if cont_lsn == cached_lsn + 1 { - MATERIALIZED_PAGE_CACHE_HIT.inc_by(1); return Ok(traversal_path); } if let Some(prev) = prev_lsn { @@ -3614,26 +3582,6 @@ impl Timeline { }) } - /// # Cancel-safety - /// - /// This method is cancellation-safe. - async fn lookup_cached_page( - &self, - key: &Key, - lsn: Lsn, - ctx: &RequestContext, - ) -> Option<(Lsn, Bytes)> { - let cache = page_cache::get(); - - // FIXME: It's pointless to check the cache for things that are not 8kB pages. - // We should look at the key to determine if it's a cacheable object - let (lsn, read_guard) = cache - .lookup_materialized_page(self.tenant_shard_id, self.timeline_id, key, lsn, ctx) - .await?; - let img = Bytes::from(read_guard.to_vec()); - Some((lsn, img)) - } - async fn get_ready_ancestor_timeline( &self, ancestor: &Arc, @@ -5280,8 +5228,6 @@ impl Timeline { trace!("found {} WAL records that will init the page for {} at {}, performing WAL redo", data.records.len(), key, request_lsn); }; - let last_rec_lsn = data.records.last().unwrap().0; - let img = match self .walredo_mgr .as_ref() @@ -5295,23 +5241,6 @@ impl Timeline { Err(e) => return Err(PageReconstructError::WalRedo(e)), }; - if img.len() == page_cache::PAGE_SZ { - let cache = page_cache::get(); - if let Err(e) = cache - .memorize_materialized_page( - self.tenant_shard_id, - self.timeline_id, - key, - last_rec_lsn, - &img, - ) - .await - .context("Materialized page memoization failed") - { - return Err(PageReconstructError::from(e)); - } - } - Ok(img) } } diff --git a/pageserver/src/tenant/timeline/compaction.rs b/pageserver/src/tenant/timeline/compaction.rs index 8a95029f33..efaa6144af 100644 --- a/pageserver/src/tenant/timeline/compaction.rs +++ b/pageserver/src/tenant/timeline/compaction.rs @@ -47,10 +47,14 @@ impl Timeline { /// TODO: cancellation pub(crate) async fn compact_legacy( self: &Arc, - _cancel: &CancellationToken, + cancel: &CancellationToken, flags: EnumSet, ctx: &RequestContext, ) -> Result<(), CompactionError> { + if flags.contains(CompactFlags::EnhancedGcBottomMostCompaction) { + return self.compact_with_gc(cancel, ctx).await; + } + // High level strategy for compaction / image creation: // // 1. First, calculate the desired "partitioning" of the @@ -959,13 +963,20 @@ impl Timeline { /// the GC horizon without considering retain_lsns. Then, it does a full compaction over all these delta /// layers and image layers, which generates image layers on the gc horizon, drop deltas below gc horizon, /// and create delta layers with all deltas >= gc horizon. - #[cfg(test)] pub(crate) async fn compact_with_gc( self: &Arc, _cancel: &CancellationToken, ctx: &RequestContext, ) -> Result<(), CompactionError> { use crate::tenant::storage_layer::ValueReconstructState; + use std::collections::BTreeSet; + + info!("running enhanced gc bottom-most compaction"); + + scopeguard::defer! { + info!("done enhanced gc bottom-most compaction"); + }; + // Step 0: pick all delta layers + image layers below/intersect with the GC horizon. // The layer selection has the following properties: // 1. If a layer is in the selection, all layers below it are in the selection. @@ -974,6 +985,11 @@ impl Timeline { let guard = self.layers.read().await; let layers = guard.layer_map(); let gc_info = self.gc_info.read().unwrap(); + if !gc_info.retain_lsns.is_empty() || !gc_info.leases.is_empty() { + return Err(CompactionError::Other(anyhow!( + "enhanced legacy compaction currently does not support retain_lsns (branches)" + ))); + } let gc_cutoff = Lsn::min(gc_info.cutoffs.horizon, gc_info.cutoffs.pitr); let mut selected_layers = Vec::new(); // TODO: consider retain_lsns @@ -985,21 +1001,36 @@ impl Timeline { } (selected_layers, gc_cutoff) }; + info!( + "picked {} layers for compaction with gc_cutoff={}", + layer_selection.len(), + gc_cutoff + ); // Step 1: (In the future) construct a k-merge iterator over all layers. For now, simply collect all keys + LSNs. + // Also, collect the layer information to decide when to split the new delta layers. let mut all_key_values = Vec::new(); + let mut delta_split_points = BTreeSet::new(); for layer in &layer_selection { all_key_values.extend(layer.load_key_values(ctx).await?); + let desc = layer.layer_desc(); + if desc.is_delta() { + // TODO: is it correct to only record split points for deltas intersecting with the GC horizon? (exclude those below/above the horizon) + // so that we can avoid having too many small delta layers. + let key_range = desc.get_key_range(); + delta_split_points.insert(key_range.start); + delta_split_points.insert(key_range.end); + } } // Key small to large, LSN low to high, if the same LSN has both image and delta due to the merge of delta layers and - // image layers, make image appear later than delta. + // image layers, make image appear before than delta. struct ValueWrapper<'a>(&'a crate::repository::Value); impl Ord for ValueWrapper<'_> { fn cmp(&self, other: &Self) -> std::cmp::Ordering { use crate::repository::Value; use std::cmp::Ordering; match (self.0, other.0) { - (Value::Image(_), Value::WalRecord(_)) => Ordering::Greater, - (Value::WalRecord(_), Value::Image(_)) => Ordering::Less, + (Value::Image(_), Value::WalRecord(_)) => Ordering::Less, + (Value::WalRecord(_), Value::Image(_)) => Ordering::Greater, _ => Ordering::Equal, } } @@ -1018,13 +1049,6 @@ impl Timeline { all_key_values.sort_by(|(k1, l1, v1), (k2, l2, v2)| { (k1, l1, ValueWrapper(v1)).cmp(&(k2, l2, ValueWrapper(v2))) }); - let max_lsn = all_key_values - .iter() - .map(|(_, lsn, _)| lsn) - .max() - .copied() - .unwrap() - + 1; // Step 2: Produce images+deltas. TODO: ensure newly-produced delta does not overlap with other deltas. // Data of the same key. let mut accumulated_values = Vec::new(); @@ -1043,14 +1067,24 @@ impl Timeline { // We have a list of deltas/images. We want to create image layers while collect garbages. for (key, lsn, val) in accumulated_values.iter().rev() { if *lsn > horizon { - keys_above_horizon.push((*key, *lsn, val.clone())); // TODO: ensure one LSN corresponds to either delta or image instead of both + if let Some((_, prev_lsn, _)) = keys_above_horizon.last_mut() { + if *prev_lsn == *lsn { + // The case that we have an LSN with both data from the delta layer and the image layer. As + // `ValueWrapper` ensures that an image is ordered before a delta at the same LSN, we simply + // drop this delta and keep the image. + // + // For example, we have delta layer key1@0x10, key1@0x20, and image layer key1@0x10, we will + // keep the image for key1@0x10 and the delta for key1@0x20. key1@0x10 delta will be simply + // dropped. + continue; + } + } + keys_above_horizon.push((*key, *lsn, val.clone())); } else if *lsn <= horizon { match val { crate::repository::Value::Image(image) => { - if lsn <= &horizon { - base_image = Some((*lsn, image.clone())); - break; - } + base_image = Some((*lsn, image.clone())); + break; } crate::repository::Value::WalRecord(wal) => { delta_above_base_image.push((*lsn, wal.clone())); @@ -1058,7 +1092,7 @@ impl Timeline { } } } - delta_above_base_image.reverse(); + // do not reverse delta_above_base_image, reconstruct state expects reversely-ordered records keys_above_horizon.reverse(); let state = ValueReconstructState { img: base_image, @@ -1068,15 +1102,59 @@ impl Timeline { Ok((keys_above_horizon, img)) } - let mut delta_layer_writer = DeltaLayerWriter::new( - self.conf, - self.timeline_id, - self.tenant_shard_id, - all_key_values.first().unwrap().0, - gc_cutoff..max_lsn, // TODO: off by one? - ctx, - ) - .await?; + async fn flush_deltas( + deltas: &mut Vec<(Key, Lsn, crate::repository::Value)>, + last_key: Key, + delta_split_points: &[Key], + current_delta_split_point: &mut usize, + tline: &Arc, + gc_cutoff: Lsn, + ctx: &RequestContext, + ) -> anyhow::Result> { + // Check if we need to split the delta layer. We split at the original delta layer boundary to avoid + // overlapping layers. + // + // If we have a structure like this: + // + // | Delta 1 | | Delta 4 | + // |---------| Delta 2 |---------| + // | Delta 3 | | Delta 5 | + // + // And we choose to compact delta 2+3+5. We will get an overlapping delta layer with delta 1+4. + // A simple solution here is to split the delta layers using the original boundary, while this + // might produce a lot of small layers. This should be improved and fixed in the future. + let mut need_split = false; + while *current_delta_split_point < delta_split_points.len() + && last_key >= delta_split_points[*current_delta_split_point] + { + *current_delta_split_point += 1; + need_split = true; + } + if !need_split { + return Ok(None); + } + let deltas = std::mem::take(deltas); + if deltas.is_empty() { + return Ok(None); + } + let end_lsn = deltas.iter().map(|(_, lsn, _)| lsn).max().copied().unwrap() + 1; + let mut delta_layer_writer = DeltaLayerWriter::new( + tline.conf, + tline.timeline_id, + tline.tenant_shard_id, + deltas.first().unwrap().0, + gc_cutoff..end_lsn, + ctx, + ) + .await?; + let key_end = deltas.last().unwrap().0.next(); + for (key, lsn, val) in deltas { + delta_layer_writer.put_value(key, lsn, val, ctx).await?; + } + let delta_layer = delta_layer_writer.finish(key_end, tline, ctx).await?; + Ok(Some(delta_layer)) + } + let mut image_layer_writer = ImageLayerWriter::new( self.conf, self.timeline_id, @@ -1087,6 +1165,10 @@ impl Timeline { ) .await?; + let mut delta_values = Vec::new(); + let delta_split_points = delta_split_points.into_iter().collect_vec(); + let mut current_delta_split_point = 0; + let mut delta_layers = Vec::new(); for item @ (key, _, _) in &all_key_values { if &last_key == key { accumulated_values.push(item); @@ -1094,34 +1176,63 @@ impl Timeline { let (deltas, image) = flush_accumulated_states(self, last_key, &accumulated_values, gc_cutoff) .await?; + // Put the image into the image layer. Currently we have a single big layer for the compaction. image_layer_writer.put_image(last_key, image, ctx).await?; - for (key, lsn, val) in deltas { - delta_layer_writer.put_value(key, lsn, val, ctx).await?; - } + delta_values.extend(deltas); + delta_layers.extend( + flush_deltas( + &mut delta_values, + last_key, + &delta_split_points, + &mut current_delta_split_point, + self, + gc_cutoff, + ctx, + ) + .await?, + ); accumulated_values.clear(); accumulated_values.push(item); last_key = *key; } } + + // TODO: move this part to the loop body let (deltas, image) = flush_accumulated_states(self, last_key, &accumulated_values, gc_cutoff).await?; + // Put the image into the image layer. Currently we have a single big layer for the compaction. image_layer_writer.put_image(last_key, image, ctx).await?; - for (key, lsn, val) in deltas { - delta_layer_writer.put_value(key, lsn, val, ctx).await?; - } - accumulated_values.clear(); - // TODO: split layers - let delta_layer = delta_layer_writer.finish(last_key, self, ctx).await?; + delta_values.extend(deltas); + delta_layers.extend( + flush_deltas( + &mut delta_values, + last_key, + &delta_split_points, + &mut current_delta_split_point, + self, + gc_cutoff, + ctx, + ) + .await?, + ); + let image_layer = image_layer_writer.finish(self, ctx).await?; + info!( + "produced {} delta layers and {} image layers", + delta_layers.len(), + 1 + ); + let mut compact_to = Vec::new(); + compact_to.extend(delta_layers); + compact_to.push(image_layer); // Step 3: Place back to the layer map. { let mut guard = self.layers.write().await; - guard.finish_gc_compaction( - &layer_selection, - &[delta_layer.clone(), image_layer.clone()], - &self.metrics, - ) + guard.finish_gc_compaction(&layer_selection, &compact_to, &self.metrics) }; + + self.remote_client + .schedule_compaction_update(&layer_selection, &compact_to)?; Ok(()) } } diff --git a/pageserver/src/tenant/timeline/delete.rs b/pageserver/src/tenant/timeline/delete.rs index 441298f3e9..6d747d424d 100644 --- a/pageserver/src/tenant/timeline/delete.rs +++ b/pageserver/src/tenant/timeline/delete.rs @@ -255,7 +255,6 @@ impl DeleteTimelineFlow { } /// Shortcut to create Timeline in stopping state and spawn deletion task. - /// See corresponding parts of [`crate::tenant::delete::DeleteTenantFlow`] #[instrument(skip_all, fields(%timeline_id))] pub async fn resume_deletion( tenant: Arc, @@ -420,10 +419,6 @@ impl DeleteTimelineFlow { Ok(()) } - pub(crate) fn is_finished(&self) -> bool { - matches!(self, Self::Finished) - } - pub(crate) fn is_not_started(&self) -> bool { matches!(self, Self::NotStarted) } diff --git a/pageserver/src/tenant/timeline/layer_manager.rs b/pageserver/src/tenant/timeline/layer_manager.rs index 550a9a567a..948237e06a 100644 --- a/pageserver/src/tenant/timeline/layer_manager.rs +++ b/pageserver/src/tenant/timeline/layer_manager.rs @@ -227,7 +227,6 @@ impl LayerManager { } /// Called when a GC-compaction is completed. - #[cfg(test)] pub(crate) fn finish_gc_compaction( &mut self, compact_from: &[Layer], diff --git a/pageserver/src/tenant/vectored_blob_io.rs b/pageserver/src/tenant/vectored_blob_io.rs index 6e825760e3..1241a13902 100644 --- a/pageserver/src/tenant/vectored_blob_io.rs +++ b/pageserver/src/tenant/vectored_blob_io.rs @@ -77,7 +77,7 @@ pub(crate) struct VectoredReadBuilder { start: u64, end: u64, blobs_at: VecMap, - max_read_size: usize, + max_read_size: Option, } impl VectoredReadBuilder { @@ -90,7 +90,7 @@ impl VectoredReadBuilder { start_offset: u64, end_offset: u64, meta: BlobMeta, - max_read_size: usize, + max_read_size: Option, ) -> Self { let mut blobs_at = VecMap::default(); blobs_at @@ -111,7 +111,13 @@ impl VectoredReadBuilder { pub(crate) fn extend(&mut self, start: u64, end: u64, meta: BlobMeta) -> VectoredReadExtended { tracing::trace!(start, end, "trying to extend"); let size = (end - start) as usize; - if self.end == start && self.size() + size <= self.max_read_size { + if self.end == start && { + if let Some(max_read_size) = self.max_read_size { + self.size() + size <= max_read_size + } else { + true + } + } { self.end = end; self.blobs_at .append(start, meta) @@ -157,7 +163,7 @@ pub struct VectoredReadPlanner { // Arguments for previous blob passed into [`VectoredReadPlanner::handle`] prev: Option<(Key, Lsn, u64, BlobFlag)>, - max_read_size: usize, + max_read_size: Option, } impl VectoredReadPlanner { @@ -165,7 +171,20 @@ impl VectoredReadPlanner { Self { blobs: BTreeMap::new(), prev: None, - max_read_size, + max_read_size: Some(max_read_size), + } + } + + /// This function should *only* be used if the caller has a way to control the limit. e.g., in [`StreamingVectoredReadPlanner`], + /// it uses the vectored read planner to avoid duplicated logic on handling blob start/end, while expecting the vectored + /// read planner to give a single read to a continuous range of bytes in the image layer. Therefore, it does not need the + /// code path to split reads into chunks of `max_read_size`, and controls the read size itself. + #[cfg(test)] + pub(crate) fn new_caller_controlled_max_limit() -> Self { + Self { + blobs: BTreeMap::new(), + prev: None, + max_read_size: None, } } @@ -354,6 +373,87 @@ impl<'a> VectoredBlobReader<'a> { } } +/// Read planner used in [`crate::tenant::storage_layer::image_layer::ImageLayerIterator`]. It provides a streaming API for +/// getting read blobs. It returns a batch when `handle` gets called and when the current key would exceed the read_size and +/// max_cnt constraints. Underlying it uses [`VectoredReadPlanner`]. +#[cfg(test)] +pub struct StreamingVectoredReadPlanner { + planner: VectoredReadPlanner, + /// Max read size per batch + max_read_size: u64, + /// Max item count per batch + max_cnt: usize, + /// The first offset of this batch + this_batch_first_offset: Option, + /// Size of the current batch + cnt: usize, +} + +#[cfg(test)] +impl StreamingVectoredReadPlanner { + pub fn new(max_read_size: u64, max_cnt: usize) -> Self { + assert!(max_cnt > 0); + assert!(max_read_size > 0); + Self { + // We want to have exactly one read syscall (plus several others for index lookup) for each `next_batch` call. + // Therefore, we enforce `self.max_read_size` by ourselves instead of using the VectoredReadPlanner's capability, + // to avoid splitting into two I/Os. + planner: VectoredReadPlanner::new_caller_controlled_max_limit(), + max_cnt, + max_read_size, + this_batch_first_offset: None, + cnt: 0, + } + } + + fn emit(&mut self, this_batch_first_offset: u64) -> VectoredRead { + let planner = std::mem::replace( + &mut self.planner, + VectoredReadPlanner::new_caller_controlled_max_limit(), + ); + self.this_batch_first_offset = Some(this_batch_first_offset); + self.cnt = 1; + let mut batch = planner.finish(); + assert_eq!(batch.len(), 1, "should have exactly one read batch"); + batch.pop().unwrap() + } + + pub fn handle( + &mut self, + key: Key, + lsn: Lsn, + offset: u64, + flag: BlobFlag, + ) -> Option { + if let Some(begin_offset) = self.this_batch_first_offset { + // Each batch will have at least one item b/c `self.this_batch_first_offset` is set + // after one item gets processed + if offset - begin_offset > self.max_read_size { + self.planner.handle_range_end(offset); // End the current batch with the offset + let batch = self.emit(offset); // Produce a batch + self.planner.handle(key, lsn, offset, flag); // Add this key to the next batch + return Some(batch); + } + } else { + self.this_batch_first_offset = Some(offset) + } + if self.cnt >= self.max_cnt { + self.planner.handle_range_end(offset); // End the current batch with the offset + let batch = self.emit(offset); // Produce a batch + self.planner.handle(key, lsn, offset, flag); // Add this key to the next batch + return Some(batch); + } + self.planner.handle(key, lsn, offset, flag); // Add this key to the current batch + self.cnt += 1; + None + } + + pub fn handle_range_end(&mut self, offset: u64) -> VectoredRead { + self.planner.handle_range_end(offset); + self.emit(offset) + } +} + #[cfg(test)] mod tests { use super::*; diff --git a/pgxn/neon/neon.c b/pgxn/neon/neon.c index 276d1542fe..b6b2db7e71 100644 --- a/pgxn/neon/neon.c +++ b/pgxn/neon/neon.c @@ -41,7 +41,6 @@ PG_MODULE_MAGIC; void _PG_init(void); static int logical_replication_max_snap_files = 300; -bool primary_is_running = false; static void InitLogicalReplicationMonitor(void) @@ -289,15 +288,6 @@ _PG_init(void) pg_init_extension_server(); - DefineCustomBoolVariable( - "neon.primary_is_running", - "true if the primary was running at replica startup. false otherwise", - NULL, - &primary_is_running, - false, - PGC_POSTMASTER, - 0, - NULL, NULL, NULL); /* * Important: This must happen after other parts of the extension are * loaded, otherwise any settings to GUCs that were set before the diff --git a/pgxn/neon/walproposer.c b/pgxn/neon/walproposer.c index dbc67a24f5..c53257923a 100644 --- a/pgxn/neon/walproposer.c +++ b/pgxn/neon/walproposer.c @@ -1447,7 +1447,7 @@ RecvAppendResponses(Safekeeper *sk) * core as this is kinda expected scenario. */ disable_core_dump(); - wp_log(PANIC, "WAL acceptor %s:%s with term " INT64_FORMAT " rejected our request, our term " INT64_FORMAT "", + wp_log(PANIC, "WAL acceptor %s:%s with term " INT64_FORMAT " rejected our request, our term " INT64_FORMAT ", meaning another compute is running at the same time, and it conflicts with us", sk->host, sk->port, sk->appendResponse.term, wp->propTerm); } diff --git a/pgxn/neon_walredo/walredoproc.c b/pgxn/neon_walredo/walredoproc.c index c4ab22636b..cc545393f5 100644 --- a/pgxn/neon_walredo/walredoproc.c +++ b/pgxn/neon_walredo/walredoproc.c @@ -168,16 +168,15 @@ close_range_syscall(unsigned int start_fd, unsigned int count, unsigned int flag static void enter_seccomp_mode(void) { - /* * The pageserver process relies on us to close all the file descriptors * it potentially leaked to us, _before_ we start processing potentially dangerous * wal records. See the comment in the Rust code that launches this process. */ - int err; - if (err = close_range_syscall(3, ~0U, 0)) { - ereport(FATAL, (errcode(ERRCODE_SYSTEM_ERROR), errmsg("seccomp: could not close files >= fd 3"))); - } + if (close_range_syscall(3, ~0U, 0) != 0) + ereport(FATAL, + (errcode(ERRCODE_SYSTEM_ERROR), + errmsg("seccomp: could not close files >= fd 3"))); PgSeccompRule syscalls[] = { diff --git a/proxy/src/bin/pg_sni_router.rs b/proxy/src/bin/pg_sni_router.rs index e1674049a6..44e880838e 100644 --- a/proxy/src/bin/pg_sni_router.rs +++ b/proxy/src/bin/pg_sni_router.rs @@ -10,7 +10,7 @@ use itertools::Itertools; use proxy::config::TlsServerEndPoint; use proxy::context::RequestMonitoring; use proxy::metrics::{Metrics, ThreadPoolMetrics}; -use proxy::proxy::{copy_bidirectional_client_compute, run_until_cancelled}; +use proxy::proxy::{copy_bidirectional_client_compute, run_until_cancelled, ErrorSource}; use rustls::pki_types::PrivateKeyDer; use tokio::net::TcpListener; @@ -286,7 +286,10 @@ async fn handle_client( // Starting from here we only proxy the client's traffic. info!("performing the proxy pass..."); - let _ = copy_bidirectional_client_compute(&mut tls_stream, &mut client).await?; - Ok(()) + match copy_bidirectional_client_compute(&mut tls_stream, &mut client).await { + Ok(_) => Ok(()), + Err(ErrorSource::Client(err)) => Err(err).context("client"), + Err(ErrorSource::Compute(err)) => Err(err).context("compute"), + } } diff --git a/proxy/src/console/messages.rs b/proxy/src/console/messages.rs index 3b7d681a41..d28d13ba69 100644 --- a/proxy/src/console/messages.rs +++ b/proxy/src/console/messages.rs @@ -5,7 +5,7 @@ use std::fmt::{self, Display}; use crate::auth::IpPattern; use crate::intern::{BranchIdInt, EndpointIdInt, ProjectIdInt}; -use crate::proxy::retry::ShouldRetry; +use crate::proxy::retry::CouldRetry; /// Generic error response with human-readable description. /// Note that we can't always present it to user as is. @@ -64,45 +64,47 @@ impl Display for ConsoleError { } } -impl ShouldRetry for ConsoleError { +impl CouldRetry for ConsoleError { fn could_retry(&self) -> bool { - if self.status.is_none() || self.status.as_ref().unwrap().details.retry_info.is_none() { - // retry some temporary failures because the compute was in a bad state - // (bad request can be returned when the endpoint was in transition) - return match &self { - ConsoleError { - http_status_code: http::StatusCode::BAD_REQUEST, - .. - } => true, - // don't retry when quotas are exceeded - ConsoleError { - http_status_code: http::StatusCode::UNPROCESSABLE_ENTITY, - ref error, - .. - } => !error.contains("compute time quota of non-primary branches is exceeded"), - // locked can be returned when the endpoint was in transition - // or when quotas are exceeded. don't retry when quotas are exceeded - ConsoleError { - http_status_code: http::StatusCode::LOCKED, - ref error, - .. - } => { - !error.contains("quota exceeded") - && !error.contains("the limit for current plan reached") - } - _ => false, - }; + // If the error message does not have a status, + // the error is unknown and probably should not retry automatically + let Some(status) = &self.status else { + return false; + }; + + // retry if the retry info is set. + if status.details.retry_info.is_some() { + return true; } - // retry if the response has a retry delay - if let Some(retry_info) = self - .status - .as_ref() - .and_then(|s| s.details.retry_info.as_ref()) - { - retry_info.retry_delay_ms > 0 - } else { - false + // if no retry info set, attempt to use the error code to guess the retry state. + let reason = status + .details + .error_info + .map_or(Reason::Unknown, |e| e.reason); + match reason { + // not a transitive error + Reason::RoleProtected => false, + // on retry, it will still not be found + Reason::ResourceNotFound + | Reason::ProjectNotFound + | Reason::EndpointNotFound + | Reason::BranchNotFound => false, + // we were asked to go away + Reason::RateLimitExceeded + | Reason::NonDefaultBranchComputeTimeExceeded + | Reason::ActiveTimeQuotaExceeded + | Reason::ComputeTimeQuotaExceeded + | Reason::WrittenDataQuotaExceeded + | Reason::DataTransferQuotaExceeded + | Reason::LogicalSizeQuotaExceeded => false, + // transitive error. control plane is currently busy + // but might be ready soon + Reason::RunningOperations => true, + Reason::ConcurrencyLimitReached => true, + Reason::LockAlreadyTaken => true, + // unknown error. better not retry it. + Reason::Unknown => false, } } } @@ -121,7 +123,7 @@ pub struct Details { pub user_facing_message: Option, } -#[derive(Debug, Deserialize)] +#[derive(Copy, Clone, Debug, Deserialize)] pub struct ErrorInfo { pub reason: Reason, // Schema could also have `metadata` field, but it's not structured. Skip it for now. @@ -129,30 +131,59 @@ pub struct ErrorInfo { #[derive(Clone, Copy, Debug, Deserialize, Default)] pub enum Reason { + /// RoleProtected indicates that the role is protected and the attempted operation is not permitted on protected roles. #[serde(rename = "ROLE_PROTECTED")] RoleProtected, + /// ResourceNotFound indicates that a resource (project, endpoint, branch, etc.) wasn't found, + /// usually due to the provided ID not being correct or because the subject doesn't have enough permissions to + /// access the requested resource. + /// Prefer a more specific reason if possible, e.g., ProjectNotFound, EndpointNotFound, etc. #[serde(rename = "RESOURCE_NOT_FOUND")] ResourceNotFound, + /// ProjectNotFound indicates that the project wasn't found, usually due to the provided ID not being correct, + /// or that the subject doesn't have enough permissions to access the requested project. #[serde(rename = "PROJECT_NOT_FOUND")] ProjectNotFound, + /// EndpointNotFound indicates that the endpoint wasn't found, usually due to the provided ID not being correct, + /// or that the subject doesn't have enough permissions to access the requested endpoint. #[serde(rename = "ENDPOINT_NOT_FOUND")] EndpointNotFound, + /// BranchNotFound indicates that the branch wasn't found, usually due to the provided ID not being correct, + /// or that the subject doesn't have enough permissions to access the requested branch. #[serde(rename = "BRANCH_NOT_FOUND")] BranchNotFound, + /// RateLimitExceeded indicates that the rate limit for the operation has been exceeded. #[serde(rename = "RATE_LIMIT_EXCEEDED")] RateLimitExceeded, + /// NonDefaultBranchComputeTimeExceeded indicates that the compute time quota of non-default branches has been + /// exceeded. #[serde(rename = "NON_PRIMARY_BRANCH_COMPUTE_TIME_EXCEEDED")] - NonPrimaryBranchComputeTimeExceeded, + NonDefaultBranchComputeTimeExceeded, + /// ActiveTimeQuotaExceeded indicates that the active time quota was exceeded. #[serde(rename = "ACTIVE_TIME_QUOTA_EXCEEDED")] ActiveTimeQuotaExceeded, + /// ComputeTimeQuotaExceeded indicates that the compute time quota was exceeded. #[serde(rename = "COMPUTE_TIME_QUOTA_EXCEEDED")] ComputeTimeQuotaExceeded, + /// WrittenDataQuotaExceeded indicates that the written data quota was exceeded. #[serde(rename = "WRITTEN_DATA_QUOTA_EXCEEDED")] WrittenDataQuotaExceeded, + /// DataTransferQuotaExceeded indicates that the data transfer quota was exceeded. #[serde(rename = "DATA_TRANSFER_QUOTA_EXCEEDED")] DataTransferQuotaExceeded, + /// LogicalSizeQuotaExceeded indicates that the logical size quota was exceeded. #[serde(rename = "LOGICAL_SIZE_QUOTA_EXCEEDED")] LogicalSizeQuotaExceeded, + /// RunningOperations indicates that the project already has some running operations + /// and scheduling of new ones is prohibited. + #[serde(rename = "RUNNING_OPERATIONS")] + RunningOperations, + /// ConcurrencyLimitReached indicates that the concurrency limit for an action was reached. + #[serde(rename = "CONCURRENCY_LIMIT_REACHED")] + ConcurrencyLimitReached, + /// LockAlreadyTaken indicates that the we attempted to take a lock that was already taken. + #[serde(rename = "LOCK_ALREADY_TAKEN")] + LockAlreadyTaken, #[default] #[serde(other)] Unknown, @@ -170,7 +201,7 @@ impl Reason { } } -#[derive(Debug, Deserialize)] +#[derive(Copy, Clone, Debug, Deserialize)] pub struct RetryInfo { pub retry_delay_ms: u64, } diff --git a/proxy/src/console/provider.rs b/proxy/src/console/provider.rs index 915c2ee7a6..bec55a8343 100644 --- a/proxy/src/console/provider.rs +++ b/proxy/src/console/provider.rs @@ -25,9 +25,9 @@ use tracing::info; pub mod errors { use crate::{ - console::messages::{self, ConsoleError}, + console::messages::{self, ConsoleError, Reason}, error::{io_error, ReportableError, UserFacingError}, - proxy::retry::ShouldRetry, + proxy::retry::CouldRetry, }; use thiserror::Error; @@ -76,21 +76,22 @@ pub mod errors { ApiError::Console(e) => { use crate::error::ErrorKind::*; match e.get_reason() { - crate::console::messages::Reason::RoleProtected => User, - crate::console::messages::Reason::ResourceNotFound => User, - crate::console::messages::Reason::ProjectNotFound => User, - crate::console::messages::Reason::EndpointNotFound => User, - crate::console::messages::Reason::BranchNotFound => User, - crate::console::messages::Reason::RateLimitExceeded => ServiceRateLimit, - crate::console::messages::Reason::NonPrimaryBranchComputeTimeExceeded => { - User - } - crate::console::messages::Reason::ActiveTimeQuotaExceeded => User, - crate::console::messages::Reason::ComputeTimeQuotaExceeded => User, - crate::console::messages::Reason::WrittenDataQuotaExceeded => User, - crate::console::messages::Reason::DataTransferQuotaExceeded => User, - crate::console::messages::Reason::LogicalSizeQuotaExceeded => User, - crate::console::messages::Reason::Unknown => match &e { + Reason::RoleProtected => User, + Reason::ResourceNotFound => User, + Reason::ProjectNotFound => User, + Reason::EndpointNotFound => User, + Reason::BranchNotFound => User, + Reason::RateLimitExceeded => ServiceRateLimit, + Reason::NonDefaultBranchComputeTimeExceeded => User, + Reason::ActiveTimeQuotaExceeded => User, + Reason::ComputeTimeQuotaExceeded => User, + Reason::WrittenDataQuotaExceeded => User, + Reason::DataTransferQuotaExceeded => User, + Reason::LogicalSizeQuotaExceeded => User, + Reason::ConcurrencyLimitReached => ControlPlane, + Reason::LockAlreadyTaken => ControlPlane, + Reason::RunningOperations => ControlPlane, + Reason::Unknown => match &e { ConsoleError { http_status_code: http::StatusCode::NOT_FOUND | http::StatusCode::NOT_ACCEPTABLE, @@ -128,7 +129,7 @@ pub mod errors { } } - impl ShouldRetry for ApiError { + impl CouldRetry for ApiError { fn could_retry(&self) -> bool { match self { // retry some transport errors @@ -239,6 +240,17 @@ pub mod errors { } } } + + impl CouldRetry for WakeComputeError { + fn could_retry(&self) -> bool { + match self { + WakeComputeError::BadComputeAddress(_) => false, + WakeComputeError::ApiError(e) => e.could_retry(), + WakeComputeError::TooManyConnections => false, + WakeComputeError::TooManyConnectionAttempts(_) => false, + } + } + } } /// Auth secret which is managed by the cloud. diff --git a/proxy/src/context/parquet.rs b/proxy/src/context/parquet.rs index 1355b7e1d8..e72bf199e3 100644 --- a/proxy/src/context/parquet.rs +++ b/proxy/src/context/parquet.rs @@ -543,7 +543,9 @@ mod tests { rx: impl Stream, ) -> Vec<(u64, usize, i64)> { let remote_storage_config = RemoteStorageConfig { - storage: RemoteStorageKind::LocalFs(tmpdir.to_path_buf()), + storage: RemoteStorageKind::LocalFs { + local_path: tmpdir.to_path_buf(), + }, timeout: std::time::Duration::from_secs(120), }; let storage = GenericRemoteStorage::from_config(&remote_storage_config).unwrap(); diff --git a/proxy/src/proxy.rs b/proxy/src/proxy.rs index 072f51958f..3edefcf21a 100644 --- a/proxy/src/proxy.rs +++ b/proxy/src/proxy.rs @@ -8,6 +8,7 @@ pub mod passthrough; pub mod retry; pub mod wake_compute; pub use copy_bidirectional::copy_bidirectional_client_compute; +pub use copy_bidirectional::ErrorSource; use crate::{ auth, @@ -148,8 +149,11 @@ pub async fn task_main( ctx.log_connect(); match p.proxy_pass().instrument(span.clone()).await { Ok(()) => {} - Err(e) => { - error!(parent: &span, "per-client task finished with an error: {e:#}"); + Err(ErrorSource::Client(e)) => { + error!(parent: &span, "per-client task finished with an IO error from the client: {e:#}"); + } + Err(ErrorSource::Compute(e)) => { + error!(parent: &span, "per-client task finished with an IO error from the compute: {e:#}"); } } } diff --git a/proxy/src/proxy/connect_compute.rs b/proxy/src/proxy/connect_compute.rs index 409d45b39a..82180aaee3 100644 --- a/proxy/src/proxy/connect_compute.rs +++ b/proxy/src/proxy/connect_compute.rs @@ -7,7 +7,7 @@ use crate::{ error::ReportableError, metrics::{ConnectOutcome, ConnectionFailureKind, Metrics, RetriesMetricGroup, RetryType}, proxy::{ - retry::{retry_after, ShouldRetry}, + retry::{retry_after, should_retry, CouldRetry}, wake_compute::wake_compute, }, Host, @@ -17,6 +17,8 @@ use pq_proto::StartupMessageParams; use tokio::time; use tracing::{error, info, warn}; +use super::retry::ShouldRetryWakeCompute; + const CONNECT_TIMEOUT: time::Duration = time::Duration::from_secs(2); /// If we couldn't connect, a cached connection info might be to blame @@ -104,7 +106,7 @@ pub async fn connect_to_compute( connect_to_compute_retry_config: RetryConfig, ) -> Result where - M::ConnectError: ShouldRetry + std::fmt::Debug, + M::ConnectError: CouldRetry + ShouldRetryWakeCompute + std::fmt::Debug, M::Error: From, { let mut num_retries = 0; @@ -139,10 +141,10 @@ where error!(error = ?err, "could not connect to compute node"); - let node_info = if !node_info.cached() || !err.should_retry_database_address() { + let node_info = if !node_info.cached() || !err.should_retry_wake_compute() { // If we just recieved this from cplane and dodn't get it from cache, we shouldn't retry. // Do not need to retrieve a new node_info, just return the old one. - if !err.should_retry(num_retries, connect_to_compute_retry_config) { + if should_retry(&err, num_retries, connect_to_compute_retry_config) { Metrics::get().proxy.retries_metric.observe( RetriesMetricGroup { outcome: ConnectOutcome::Failed, @@ -188,9 +190,8 @@ where return Ok(res); } Err(e) => { - let retriable = e.should_retry(num_retries, connect_to_compute_retry_config); - if !retriable { - error!(error = ?e, num_retries, retriable, "couldn't connect to compute node"); + if !should_retry(&e, num_retries, connect_to_compute_retry_config) { + error!(error = ?e, num_retries, retriable = false, "couldn't connect to compute node"); Metrics::get().proxy.retries_metric.observe( RetriesMetricGroup { outcome: ConnectOutcome::Failed, @@ -200,9 +201,10 @@ where ); return Err(e.into()); } - warn!(error = ?e, num_retries, retriable, "couldn't connect to compute node"); + + warn!(error = ?e, num_retries, retriable = true, "couldn't connect to compute node"); } - } + }; let wait_duration = retry_after(num_retries, connect_to_compute_retry_config); num_retries += 1; diff --git a/proxy/src/proxy/copy_bidirectional.rs b/proxy/src/proxy/copy_bidirectional.rs index aaf3688f21..3c45fff969 100644 --- a/proxy/src/proxy/copy_bidirectional.rs +++ b/proxy/src/proxy/copy_bidirectional.rs @@ -13,12 +13,39 @@ enum TransferState { Done(u64), } +#[derive(Debug)] +pub enum ErrorDirection { + Read(io::Error), + Write(io::Error), +} + +impl ErrorSource { + fn from_client(err: ErrorDirection) -> ErrorSource { + match err { + ErrorDirection::Read(client) => Self::Client(client), + ErrorDirection::Write(compute) => Self::Compute(compute), + } + } + fn from_compute(err: ErrorDirection) -> ErrorSource { + match err { + ErrorDirection::Write(client) => Self::Client(client), + ErrorDirection::Read(compute) => Self::Compute(compute), + } + } +} + +#[derive(Debug)] +pub enum ErrorSource { + Client(io::Error), + Compute(io::Error), +} + fn transfer_one_direction( cx: &mut Context<'_>, state: &mut TransferState, r: &mut A, w: &mut B, -) -> Poll> +) -> Poll> where A: AsyncRead + AsyncWrite + Unpin + ?Sized, B: AsyncRead + AsyncWrite + Unpin + ?Sized, @@ -32,7 +59,7 @@ where *state = TransferState::ShuttingDown(count); } TransferState::ShuttingDown(count) => { - ready!(w.as_mut().poll_shutdown(cx))?; + ready!(w.as_mut().poll_shutdown(cx)).map_err(ErrorDirection::Write)?; *state = TransferState::Done(*count); } TransferState::Done(count) => return Poll::Ready(Ok(*count)), @@ -44,7 +71,7 @@ where pub async fn copy_bidirectional_client_compute( client: &mut Client, compute: &mut Compute, -) -> Result<(u64, u64), std::io::Error> +) -> Result<(u64, u64), ErrorSource> where Client: AsyncRead + AsyncWrite + Unpin + ?Sized, Compute: AsyncRead + AsyncWrite + Unpin + ?Sized, @@ -54,9 +81,11 @@ where poll_fn(|cx| { let mut client_to_compute_result = - transfer_one_direction(cx, &mut client_to_compute, client, compute)?; + transfer_one_direction(cx, &mut client_to_compute, client, compute) + .map_err(ErrorSource::from_client)?; let mut compute_to_client_result = - transfer_one_direction(cx, &mut compute_to_client, compute, client)?; + transfer_one_direction(cx, &mut compute_to_client, compute, client) + .map_err(ErrorSource::from_compute)?; // Early termination checks from compute to client. if let TransferState::Done(_) = compute_to_client { @@ -65,18 +94,20 @@ where // Initiate shutdown client_to_compute = TransferState::ShuttingDown(buf.amt); client_to_compute_result = - transfer_one_direction(cx, &mut client_to_compute, client, compute)?; + transfer_one_direction(cx, &mut client_to_compute, client, compute) + .map_err(ErrorSource::from_client)?; } } - // Early termination checks from compute to client. + // Early termination checks from client to compute. if let TransferState::Done(_) = client_to_compute { if let TransferState::Running(buf) = &compute_to_client { info!("Client is done, terminate compute"); // Initiate shutdown compute_to_client = TransferState::ShuttingDown(buf.amt); compute_to_client_result = - transfer_one_direction(cx, &mut compute_to_client, client, compute)?; + transfer_one_direction(cx, &mut compute_to_client, compute, client) + .map_err(ErrorSource::from_compute)?; } } @@ -138,7 +169,7 @@ impl CopyBuffer { cx: &mut Context<'_>, mut reader: Pin<&mut R>, mut writer: Pin<&mut W>, - ) -> Poll> + ) -> Poll> where R: AsyncRead + ?Sized, W: AsyncWrite + ?Sized, @@ -149,11 +180,11 @@ impl CopyBuffer { // Top up the buffer towards full if we can read a bit more // data - this should improve the chances of a large write if !me.read_done && me.cap < me.buf.len() { - ready!(me.poll_fill_buf(cx, reader.as_mut()))?; + ready!(me.poll_fill_buf(cx, reader.as_mut())).map_err(ErrorDirection::Read)?; } Poll::Pending } - res => res, + res => res.map_err(ErrorDirection::Write), } } @@ -162,7 +193,7 @@ impl CopyBuffer { cx: &mut Context<'_>, mut reader: Pin<&mut R>, mut writer: Pin<&mut W>, - ) -> Poll> + ) -> Poll> where R: AsyncRead + ?Sized, W: AsyncWrite + ?Sized, @@ -176,12 +207,13 @@ impl CopyBuffer { match self.poll_fill_buf(cx, reader.as_mut()) { Poll::Ready(Ok(())) => (), - Poll::Ready(Err(err)) => return Poll::Ready(Err(err)), + Poll::Ready(Err(err)) => return Poll::Ready(Err(ErrorDirection::Read(err))), Poll::Pending => { // Try flushing when the reader has no progress to avoid deadlock // when the reader depends on buffered writer. if self.need_flush { - ready!(writer.as_mut().poll_flush(cx))?; + ready!(writer.as_mut().poll_flush(cx)) + .map_err(ErrorDirection::Write)?; self.need_flush = false; } @@ -194,10 +226,10 @@ impl CopyBuffer { while self.pos < self.cap { let i = ready!(self.poll_write_buf(cx, reader.as_mut(), writer.as_mut()))?; if i == 0 { - return Poll::Ready(Err(io::Error::new( + return Poll::Ready(Err(ErrorDirection::Write(io::Error::new( io::ErrorKind::WriteZero, "write zero byte into writer", - ))); + )))); } else { self.pos += i; self.amt += i as u64; @@ -216,7 +248,7 @@ impl CopyBuffer { // If we've written all the data and we've seen EOF, flush out the // data and finish the transfer. if self.pos == self.cap && self.read_done { - ready!(writer.as_mut().poll_flush(cx))?; + ready!(writer.as_mut().poll_flush(cx)).map_err(ErrorDirection::Write)?; return Poll::Ready(Ok(self.amt)); } } diff --git a/proxy/src/proxy/passthrough.rs b/proxy/src/proxy/passthrough.rs index 62de79946f..9942fac383 100644 --- a/proxy/src/proxy/passthrough.rs +++ b/proxy/src/proxy/passthrough.rs @@ -10,13 +10,15 @@ use tokio::io::{AsyncRead, AsyncWrite}; use tracing::info; use utils::measured_stream::MeasuredStream; +use super::copy_bidirectional::ErrorSource; + /// Forward bytes in both directions (client <-> compute). #[tracing::instrument(skip_all)] pub async fn proxy_pass( client: impl AsyncRead + AsyncWrite + Unpin, compute: impl AsyncRead + AsyncWrite + Unpin, aux: MetricsAuxInfo, -) -> anyhow::Result<()> { +) -> Result<(), ErrorSource> { let usage = USAGE_METRICS.register(Ids { endpoint_id: aux.endpoint_id, branch_id: aux.branch_id, @@ -66,9 +68,11 @@ pub struct ProxyPassthrough { } impl ProxyPassthrough { - pub async fn proxy_pass(self) -> anyhow::Result<()> { + pub async fn proxy_pass(self) -> Result<(), ErrorSource> { let res = proxy_pass(self.client, self.compute.stream, self.aux).await; - self.compute.cancel_closure.try_cancel_query().await?; + if let Err(err) = self.compute.cancel_closure.try_cancel_query().await { + tracing::error!(?err, "could not cancel the query in the database"); + } res } } diff --git a/proxy/src/proxy/retry.rs b/proxy/src/proxy/retry.rs index 8dec1f1137..644b183a91 100644 --- a/proxy/src/proxy/retry.rs +++ b/proxy/src/proxy/retry.rs @@ -2,20 +2,22 @@ use crate::{compute, config::RetryConfig}; use std::{error::Error, io}; use tokio::time; -pub trait ShouldRetry { +pub trait CouldRetry { + /// Returns true if the error could be retried fn could_retry(&self) -> bool; - fn should_retry(&self, num_retries: u32, config: RetryConfig) -> bool { - match self { - _ if num_retries >= config.max_retries => false, - err => err.could_retry(), - } - } - fn should_retry_database_address(&self) -> bool { - true - } } -impl ShouldRetry for io::Error { +pub trait ShouldRetryWakeCompute { + /// Returns true if we need to invalidate the cache for this node. + /// If false, we can continue retrying with the current node cache. + fn should_retry_wake_compute(&self) -> bool; +} + +pub fn should_retry(err: &impl CouldRetry, num_retries: u32, config: RetryConfig) -> bool { + num_retries < config.max_retries && err.could_retry() +} + +impl CouldRetry for io::Error { fn could_retry(&self) -> bool { use std::io::ErrorKind; matches!( @@ -25,7 +27,7 @@ impl ShouldRetry for io::Error { } } -impl ShouldRetry for tokio_postgres::error::DbError { +impl CouldRetry for tokio_postgres::error::DbError { fn could_retry(&self) -> bool { use tokio_postgres::error::SqlState; matches!( @@ -36,7 +38,9 @@ impl ShouldRetry for tokio_postgres::error::DbError { | &SqlState::SQLCLIENT_UNABLE_TO_ESTABLISH_SQLCONNECTION, ) } - fn should_retry_database_address(&self) -> bool { +} +impl ShouldRetryWakeCompute for tokio_postgres::error::DbError { + fn should_retry_wake_compute(&self) -> bool { use tokio_postgres::error::SqlState; // Here are errors that happens after the user successfully authenticated to the database. // TODO: there are pgbouncer errors that should be retried, but they are not listed here. @@ -53,7 +57,7 @@ impl ShouldRetry for tokio_postgres::error::DbError { } } -impl ShouldRetry for tokio_postgres::Error { +impl CouldRetry for tokio_postgres::Error { fn could_retry(&self) -> bool { if let Some(io_err) = self.source().and_then(|x| x.downcast_ref()) { io::Error::could_retry(io_err) @@ -63,29 +67,33 @@ impl ShouldRetry for tokio_postgres::Error { false } } - fn should_retry_database_address(&self) -> bool { - if let Some(io_err) = self.source().and_then(|x| x.downcast_ref()) { - io::Error::should_retry_database_address(io_err) - } else if let Some(db_err) = self.source().and_then(|x| x.downcast_ref()) { - tokio_postgres::error::DbError::should_retry_database_address(db_err) +} +impl ShouldRetryWakeCompute for tokio_postgres::Error { + fn should_retry_wake_compute(&self) -> bool { + if let Some(db_err) = self.source().and_then(|x| x.downcast_ref()) { + tokio_postgres::error::DbError::should_retry_wake_compute(db_err) } else { + // likely an IO error. Possible the compute has shutdown and the + // cache is stale. true } } } -impl ShouldRetry for compute::ConnectionError { +impl CouldRetry for compute::ConnectionError { fn could_retry(&self) -> bool { match self { compute::ConnectionError::Postgres(err) => err.could_retry(), compute::ConnectionError::CouldNotConnect(err) => err.could_retry(), + compute::ConnectionError::WakeComputeError(err) => err.could_retry(), _ => false, } } - fn should_retry_database_address(&self) -> bool { +} +impl ShouldRetryWakeCompute for compute::ConnectionError { + fn should_retry_wake_compute(&self) -> bool { match self { - compute::ConnectionError::Postgres(err) => err.should_retry_database_address(), - compute::ConnectionError::CouldNotConnect(err) => err.should_retry_database_address(), + compute::ConnectionError::Postgres(err) => err.should_retry_wake_compute(), // the cache entry was not checked for validity compute::ConnectionError::TooManyConnectionAttempts(_) => false, _ => true, diff --git a/proxy/src/proxy/tests.rs b/proxy/src/proxy/tests.rs index 96683511fe..8119f39fae 100644 --- a/proxy/src/proxy/tests.rs +++ b/proxy/src/proxy/tests.rs @@ -5,21 +5,21 @@ mod mitm; use std::time::Duration; use super::connect_compute::ConnectMechanism; -use super::retry::ShouldRetry; +use super::retry::CouldRetry; use super::*; use crate::auth::backend::{ ComputeCredentialKeys, ComputeCredentials, ComputeUserInfo, MaybeOwned, TestBackend, }; use crate::config::{CertResolver, RetryConfig}; use crate::console::caches::NodeInfoCache; -use crate::console::messages::{ConsoleError, MetricsAuxInfo}; +use crate::console::messages::{ConsoleError, Details, MetricsAuxInfo, Status}; use crate::console::provider::{CachedAllowedIps, CachedRoleSecret, ConsoleBackend}; use crate::console::{self, CachedNodeInfo, NodeInfo}; use crate::error::ErrorKind; -use crate::proxy::retry::retry_after; use crate::{http, sasl, scram, BranchId, EndpointId, ProjectId}; use anyhow::{bail, Context}; use async_trait::async_trait; +use retry::{retry_after, ShouldRetryWakeCompute}; use rstest::rstest; use rustls::pki_types; use tokio_postgres::config::SslMode; @@ -438,11 +438,16 @@ impl std::fmt::Display for TestConnectError { impl std::error::Error for TestConnectError {} -impl ShouldRetry for TestConnectError { +impl CouldRetry for TestConnectError { fn could_retry(&self) -> bool { self.retryable } } +impl ShouldRetryWakeCompute for TestConnectError { + fn should_retry_wake_compute(&self) -> bool { + true + } +} #[async_trait] impl ConnectMechanism for TestConnectMechanism { @@ -485,7 +490,7 @@ impl TestBackend for TestConnectMechanism { ConnectAction::Wake => Ok(helper_create_cached_node_info(self.cache)), ConnectAction::WakeFail => { let err = console::errors::ApiError::Console(ConsoleError { - http_status_code: http::StatusCode::FORBIDDEN, + http_status_code: http::StatusCode::BAD_REQUEST, error: "TEST".into(), status: None, }); @@ -496,7 +501,15 @@ impl TestBackend for TestConnectMechanism { let err = console::errors::ApiError::Console(ConsoleError { http_status_code: http::StatusCode::BAD_REQUEST, error: "TEST".into(), - status: None, + status: Some(Status { + code: "error".into(), + message: "error".into(), + details: Details { + error_info: None, + retry_info: Some(console::messages::RetryInfo { retry_delay_ms: 1 }), + user_facing_message: None, + }, + }), }); assert!(err.could_retry()); Err(console::errors::WakeComputeError::ApiError(err)) diff --git a/proxy/src/proxy/wake_compute.rs b/proxy/src/proxy/wake_compute.rs index c166cf4389..fef349aac0 100644 --- a/proxy/src/proxy/wake_compute.rs +++ b/proxy/src/proxy/wake_compute.rs @@ -1,18 +1,16 @@ use crate::config::RetryConfig; -use crate::console::messages::ConsoleError; +use crate::console::messages::{ConsoleError, Reason}; use crate::console::{errors::WakeComputeError, provider::CachedNodeInfo}; use crate::context::RequestMonitoring; use crate::metrics::{ ConnectOutcome, ConnectionFailuresBreakdownGroup, Metrics, RetriesMetricGroup, RetryType, WakeupFailureKind, }; -use crate::proxy::retry::retry_after; +use crate::proxy::retry::{retry_after, should_retry}; use hyper1::StatusCode; -use std::ops::ControlFlow; use tracing::{error, info, warn}; use super::connect_compute::ComputeConnectBackend; -use super::retry::ShouldRetry; pub async fn wake_compute( num_retries: &mut u32, @@ -22,9 +20,8 @@ pub async fn wake_compute( ) -> Result { let retry_type = RetryType::WakeCompute; loop { - let wake_res = api.wake_compute(ctx).await; - match handle_try_wake(wake_res, *num_retries, config) { - Err(e) => { + match api.wake_compute(ctx).await { + Err(e) if !should_retry(&e, *num_retries, config) => { error!(error = ?e, num_retries, retriable = false, "couldn't wake compute node"); report_error(&e, false); Metrics::get().proxy.retries_metric.observe( @@ -36,11 +33,11 @@ pub async fn wake_compute( ); return Err(e); } - Ok(ControlFlow::Continue(e)) => { + Err(e) => { warn!(error = ?e, num_retries, retriable = true, "couldn't wake compute node"); report_error(&e, true); } - Ok(ControlFlow::Break(n)) => { + Ok(n) => { Metrics::get().proxy.retries_metric.observe( RetriesMetricGroup { outcome: ConnectOutcome::Success, @@ -63,70 +60,28 @@ pub async fn wake_compute( } } -/// Attempts to wake up the compute node. -/// * Returns Ok(Continue(e)) if there was an error waking but retries are acceptable -/// * Returns Ok(Break(node)) if the wakeup succeeded -/// * Returns Err(e) if there was an error -pub fn handle_try_wake( - result: Result, - num_retries: u32, - config: RetryConfig, -) -> Result, WakeComputeError> { - match result { - Err(err) => match &err { - WakeComputeError::ApiError(api) if api.should_retry(num_retries, config) => { - Ok(ControlFlow::Continue(err)) - } - _ => Err(err), - }, - // Ready to try again. - Ok(new) => Ok(ControlFlow::Break(new)), - } -} - fn report_error(e: &WakeComputeError, retry: bool) { use crate::console::errors::ApiError; let kind = match e { WakeComputeError::BadComputeAddress(_) => WakeupFailureKind::BadComputeAddress, WakeComputeError::ApiError(ApiError::Transport(_)) => WakeupFailureKind::ApiTransportError, WakeComputeError::ApiError(ApiError::Console(e)) => match e.get_reason() { - crate::console::messages::Reason::RoleProtected => { - WakeupFailureKind::ApiConsoleBadRequest - } - crate::console::messages::Reason::ResourceNotFound => { - WakeupFailureKind::ApiConsoleBadRequest - } - crate::console::messages::Reason::ProjectNotFound => { - WakeupFailureKind::ApiConsoleBadRequest - } - crate::console::messages::Reason::EndpointNotFound => { - WakeupFailureKind::ApiConsoleBadRequest - } - crate::console::messages::Reason::BranchNotFound => { - WakeupFailureKind::ApiConsoleBadRequest - } - crate::console::messages::Reason::RateLimitExceeded => { - WakeupFailureKind::ApiConsoleLocked - } - crate::console::messages::Reason::NonPrimaryBranchComputeTimeExceeded => { - WakeupFailureKind::QuotaExceeded - } - crate::console::messages::Reason::ActiveTimeQuotaExceeded => { - WakeupFailureKind::QuotaExceeded - } - crate::console::messages::Reason::ComputeTimeQuotaExceeded => { - WakeupFailureKind::QuotaExceeded - } - crate::console::messages::Reason::WrittenDataQuotaExceeded => { - WakeupFailureKind::QuotaExceeded - } - crate::console::messages::Reason::DataTransferQuotaExceeded => { - WakeupFailureKind::QuotaExceeded - } - crate::console::messages::Reason::LogicalSizeQuotaExceeded => { - WakeupFailureKind::QuotaExceeded - } - crate::console::messages::Reason::Unknown => match e { + Reason::RoleProtected => WakeupFailureKind::ApiConsoleBadRequest, + Reason::ResourceNotFound => WakeupFailureKind::ApiConsoleBadRequest, + Reason::ProjectNotFound => WakeupFailureKind::ApiConsoleBadRequest, + Reason::EndpointNotFound => WakeupFailureKind::ApiConsoleBadRequest, + Reason::BranchNotFound => WakeupFailureKind::ApiConsoleBadRequest, + Reason::RateLimitExceeded => WakeupFailureKind::ApiConsoleLocked, + Reason::NonDefaultBranchComputeTimeExceeded => WakeupFailureKind::QuotaExceeded, + Reason::ActiveTimeQuotaExceeded => WakeupFailureKind::QuotaExceeded, + Reason::ComputeTimeQuotaExceeded => WakeupFailureKind::QuotaExceeded, + Reason::WrittenDataQuotaExceeded => WakeupFailureKind::QuotaExceeded, + Reason::DataTransferQuotaExceeded => WakeupFailureKind::QuotaExceeded, + Reason::LogicalSizeQuotaExceeded => WakeupFailureKind::QuotaExceeded, + Reason::ConcurrencyLimitReached => WakeupFailureKind::ApiConsoleLocked, + Reason::LockAlreadyTaken => WakeupFailureKind::ApiConsoleLocked, + Reason::RunningOperations => WakeupFailureKind::ApiConsoleLocked, + Reason::Unknown => match e { ConsoleError { http_status_code: StatusCode::LOCKED, ref error, diff --git a/proxy/src/serverless/backend.rs b/proxy/src/serverless/backend.rs index 86e64c0a38..c73ea8646a 100644 --- a/proxy/src/serverless/backend.rs +++ b/proxy/src/serverless/backend.rs @@ -16,7 +16,10 @@ use crate::{ context::RequestMonitoring, error::{ErrorKind, ReportableError, UserFacingError}, intern::EndpointIdInt, - proxy::{connect_compute::ConnectMechanism, retry::ShouldRetry}, + proxy::{ + connect_compute::ConnectMechanism, + retry::{CouldRetry, ShouldRetryWakeCompute}, + }, rate_limiter::EndpointRateLimiter, Host, }; @@ -179,7 +182,7 @@ impl UserFacingError for HttpConnError { } } -impl ShouldRetry for HttpConnError { +impl CouldRetry for HttpConnError { fn could_retry(&self) -> bool { match self { HttpConnError::ConnectionError(e) => e.could_retry(), @@ -190,9 +193,11 @@ impl ShouldRetry for HttpConnError { HttpConnError::TooManyConnectionAttempts(_) => false, } } - fn should_retry_database_address(&self) -> bool { +} +impl ShouldRetryWakeCompute for HttpConnError { + fn should_retry_wake_compute(&self) -> bool { match self { - HttpConnError::ConnectionError(e) => e.should_retry_database_address(), + HttpConnError::ConnectionError(e) => e.should_retry_wake_compute(), // we never checked cache validity HttpConnError::TooManyConnectionAttempts(_) => false, _ => true, diff --git a/proxy/src/serverless/websocket.rs b/proxy/src/serverless/websocket.rs index 0e9772733d..0d5b88f07b 100644 --- a/proxy/src/serverless/websocket.rs +++ b/proxy/src/serverless/websocket.rs @@ -1,3 +1,4 @@ +use crate::proxy::ErrorSource; use crate::{ cancellation::CancellationHandlerMain, config::ProxyConfig, @@ -7,6 +8,7 @@ use crate::{ proxy::{handle_client, ClientMode}, rate_limiter::EndpointRateLimiter, }; +use anyhow::Context as _; use bytes::{Buf, BufMut, Bytes, BytesMut}; use framed_websockets::{Frame, OpCode, WebSocketServer}; use futures::{Sink, Stream}; @@ -165,7 +167,11 @@ pub async fn serve_websocket( Ok(Some(p)) => { ctx.set_success(); ctx.log_connect(); - p.proxy_pass().await + match p.proxy_pass().await { + Ok(()) => Ok(()), + Err(ErrorSource::Client(err)) => Err(err).context("client"), + Err(ErrorSource::Compute(err)) => Err(err).context("compute"), + } } } } diff --git a/safekeeper/src/bin/safekeeper.rs b/safekeeper/src/bin/safekeeper.rs index 86238c7292..20650490b1 100644 --- a/safekeeper/src/bin/safekeeper.rs +++ b/safekeeper/src/bin/safekeeper.rs @@ -28,8 +28,8 @@ use utils::pid_file; use metrics::set_build_info_metric; use safekeeper::defaults::{ - DEFAULT_HEARTBEAT_TIMEOUT, DEFAULT_HTTP_LISTEN_ADDR, DEFAULT_MAX_OFFLOADER_LAG_BYTES, - DEFAULT_PARTIAL_BACKUP_TIMEOUT, DEFAULT_PG_LISTEN_ADDR, + DEFAULT_CONTROL_FILE_SAVE_INTERVAL, DEFAULT_HEARTBEAT_TIMEOUT, DEFAULT_HTTP_LISTEN_ADDR, + DEFAULT_MAX_OFFLOADER_LAG_BYTES, DEFAULT_PARTIAL_BACKUP_TIMEOUT, DEFAULT_PG_LISTEN_ADDR, }; use safekeeper::http; use safekeeper::wal_service; @@ -172,6 +172,7 @@ struct Args { walsenders_keep_horizon: bool, /// Enable partial backup. If disabled, safekeeper will not upload partial /// segments to remote storage. + /// TODO: now partial backup is always enabled, remove this flag. #[arg(long)] partial_backup_enabled: bool, /// Controls how long backup will wait until uploading the partial segment. @@ -181,6 +182,15 @@ struct Args { /// be used in tests. #[arg(long)] disable_periodic_broker_push: bool, + /// Enable automatic switching to offloaded state. + #[arg(long)] + enable_offload: bool, + /// Delete local WAL files after offloading. When disabled, they will be left on disk. + #[arg(long)] + delete_offloaded_wal: bool, + /// Pending updates to control file will be automatically saved after this interval. + #[arg(long, value_parser = humantime::parse_duration, default_value = DEFAULT_CONTROL_FILE_SAVE_INTERVAL)] + control_file_save_interval: Duration, } // Like PathBufValueParser, but allows empty string. @@ -328,9 +338,12 @@ async fn main() -> anyhow::Result<()> { sk_auth_token, current_thread_runtime: args.current_thread_runtime, walsenders_keep_horizon: args.walsenders_keep_horizon, - partial_backup_enabled: args.partial_backup_enabled, + partial_backup_enabled: true, partial_backup_timeout: args.partial_backup_timeout, disable_periodic_broker_push: args.disable_periodic_broker_push, + enable_offload: args.enable_offload, + delete_offloaded_wal: args.delete_offloaded_wal, + control_file_save_interval: args.control_file_save_interval, }; // initialize sentry if SENTRY_DSN is provided diff --git a/safekeeper/src/control_file.rs b/safekeeper/src/control_file.rs index 9d65187350..cd3c7fe526 100644 --- a/safekeeper/src/control_file.rs +++ b/safekeeper/src/control_file.rs @@ -12,15 +12,16 @@ use std::ops::Deref; use std::path::Path; use std::time::Instant; +use crate::control_file_upgrade::downgrade_v9_to_v8; use crate::metrics::PERSIST_CONTROL_FILE_SECONDS; -use crate::state::TimelinePersistentState; +use crate::state::{EvictionState, TimelinePersistentState}; use crate::{control_file_upgrade::upgrade_control_file, timeline::get_timeline_dir}; use utils::{bin_ser::LeSer, id::TenantTimelineId}; use crate::SafeKeeperConf; pub const SK_MAGIC: u32 = 0xcafeceefu32; -pub const SK_FORMAT_VERSION: u32 = 8; +pub const SK_FORMAT_VERSION: u32 = 9; // contains persistent metadata for safekeeper pub const CONTROL_FILE_NAME: &str = "safekeeper.control"; @@ -71,6 +72,9 @@ impl FileStorage { conf: &SafeKeeperConf, state: TimelinePersistentState, ) -> Result { + // we don't support creating new timelines in offloaded state + assert!(matches!(state.eviction_state, EvictionState::Present)); + let store = FileStorage { timeline_dir, no_sync: conf.no_sync, @@ -102,7 +106,7 @@ impl FileStorage { } /// Load control file from given directory. - pub fn load_control_file_from_dir(timeline_dir: &Utf8Path) -> Result { + fn load_control_file_from_dir(timeline_dir: &Utf8Path) -> Result { let path = timeline_dir.join(CONTROL_FILE_NAME); Self::load_control_file(path) } @@ -178,8 +182,18 @@ impl Storage for FileStorage { })?; let mut buf: Vec = Vec::new(); WriteBytesExt::write_u32::(&mut buf, SK_MAGIC)?; - WriteBytesExt::write_u32::(&mut buf, SK_FORMAT_VERSION)?; - s.ser_into(&mut buf)?; + + if s.eviction_state == EvictionState::Present { + // temp hack for forward compatibility + const PREV_FORMAT_VERSION: u32 = 8; + let prev = downgrade_v9_to_v8(s); + WriteBytesExt::write_u32::(&mut buf, PREV_FORMAT_VERSION)?; + prev.ser_into(&mut buf)?; + } else { + // otherwise, we write the current format version + WriteBytesExt::write_u32::(&mut buf, SK_FORMAT_VERSION)?; + s.ser_into(&mut buf)?; + } // calculate checksum before resize let checksum = crc32c::crc32c(&buf); diff --git a/safekeeper/src/control_file_upgrade.rs b/safekeeper/src/control_file_upgrade.rs index 8f4dfe9b43..a4b4670e42 100644 --- a/safekeeper/src/control_file_upgrade.rs +++ b/safekeeper/src/control_file_upgrade.rs @@ -1,7 +1,7 @@ //! Code to deal with safekeeper control file upgrades use crate::{ safekeeper::{AcceptorState, PgUuid, ServerInfo, Term, TermHistory, TermLsn}, - state::{PersistedPeers, TimelinePersistentState}, + state::{EvictionState, PersistedPeers, TimelinePersistentState}, wal_backup_partial, }; use anyhow::{bail, Result}; @@ -183,6 +183,55 @@ pub struct SafeKeeperStateV7 { pub peers: PersistedPeers, } +/// Persistent information stored on safekeeper node about timeline. +/// On disk data is prefixed by magic and format version and followed by checksum. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +pub struct SafeKeeperStateV8 { + #[serde(with = "hex")] + pub tenant_id: TenantId, + #[serde(with = "hex")] + pub timeline_id: TimelineId, + /// persistent acceptor state + pub acceptor_state: AcceptorState, + /// information about server + pub server: ServerInfo, + /// Unique id of the last *elected* proposer we dealt with. Not needed + /// for correctness, exists for monitoring purposes. + #[serde(with = "hex")] + pub proposer_uuid: PgUuid, + /// Since which LSN this timeline generally starts. Safekeeper might have + /// joined later. + pub timeline_start_lsn: Lsn, + /// Since which LSN safekeeper has (had) WAL for this timeline. + /// All WAL segments next to one containing local_start_lsn are + /// filled with data from the beginning. + pub local_start_lsn: Lsn, + /// Part of WAL acknowledged by quorum *and available locally*. Always points + /// to record boundary. + pub commit_lsn: Lsn, + /// LSN that points to the end of the last backed up segment. Useful to + /// persist to avoid finding out offloading progress on boot. + pub backup_lsn: Lsn, + /// Minimal LSN which may be needed for recovery of some safekeeper (end_lsn + /// of last record streamed to everyone). Persisting it helps skipping + /// recovery in walproposer, generally we compute it from peers. In + /// walproposer proto called 'truncate_lsn'. Updates are currently drived + /// only by walproposer. + pub peer_horizon_lsn: Lsn, + /// LSN of the oldest known checkpoint made by pageserver and successfully + /// pushed to s3. We don't remove WAL beyond it. Persisted only for + /// informational purposes, we receive it from pageserver (or broker). + pub remote_consistent_lsn: Lsn, + /// Peers and their state as we remember it. Knowing peers themselves is + /// fundamental; but state is saved here only for informational purposes and + /// obviously can be stale. (Currently not saved at all, but let's provision + /// place to have less file version upgrades). + pub peers: PersistedPeers, + /// Holds names of partial segments uploaded to remote storage. Used to + /// clean up old objects without leaving garbage in remote storage. + pub partial_backup: wal_backup_partial::State, +} + pub fn upgrade_control_file(buf: &[u8], version: u32) -> Result { // migrate to storing full term history if version == 1 { @@ -213,6 +262,7 @@ pub fn upgrade_control_file(buf: &[u8], version: u32) -> Result Result Result Result Result Result SafeKeeperStateV8 { + assert!(state.eviction_state == EvictionState::Present); + SafeKeeperStateV8 { + tenant_id: state.tenant_id, + timeline_id: state.timeline_id, + acceptor_state: state.acceptor_state.clone(), + server: state.server.clone(), + proposer_uuid: state.proposer_uuid, + timeline_start_lsn: state.timeline_start_lsn, + local_start_lsn: state.local_start_lsn, + commit_lsn: state.commit_lsn, + backup_lsn: state.backup_lsn, + peer_horizon_lsn: state.peer_horizon_lsn, + remote_consistent_lsn: state.remote_consistent_lsn, + peers: state.peers.clone(), + partial_backup: state.partial_backup.clone(), + } +} + #[cfg(test)] mod tests { use std::str::FromStr; diff --git a/safekeeper/src/copy_timeline.rs b/safekeeper/src/copy_timeline.rs index 51cf4db6b5..14bd3c03b8 100644 --- a/safekeeper/src/copy_timeline.rs +++ b/safekeeper/src/copy_timeline.rs @@ -15,7 +15,7 @@ use crate::{ control_file::{FileStorage, Storage}, pull_timeline::{create_temp_timeline_dir, load_temp_timeline, validate_temp_timeline}, state::TimelinePersistentState, - timeline::{FullAccessTimeline, Timeline, TimelineError}, + timeline::{Timeline, TimelineError, WalResidentTimeline}, wal_backup::copy_s3_segments, wal_storage::{wal_file_paths, WalReader}, GlobalTimelines, @@ -46,7 +46,7 @@ pub async fn handle_request(request: Request) -> Result<()> { } } - let source_tli = request.source.full_access_guard().await?; + let source_tli = request.source.wal_residence_guard().await?; let conf = &GlobalTimelines::get_global_config(); let ttid = request.destination_ttid; @@ -159,7 +159,7 @@ pub async fn handle_request(request: Request) -> Result<()> { } async fn copy_disk_segments( - tli: &FullAccessTimeline, + tli: &WalResidentTimeline, wal_seg_size: usize, start_lsn: Lsn, end_lsn: Lsn, @@ -183,7 +183,7 @@ async fn copy_disk_segments( let copy_end = copy_end - segment_start; let wal_file_path = { - let (normal, partial) = wal_file_paths(tli_dir_path, segment, wal_seg_size)?; + let (normal, partial) = wal_file_paths(tli_dir_path, segment, wal_seg_size); if segment == last_segment { partial diff --git a/safekeeper/src/debug_dump.rs b/safekeeper/src/debug_dump.rs index 062ff4b3db..15b0272cd9 100644 --- a/safekeeper/src/debug_dump.rs +++ b/safekeeper/src/debug_dump.rs @@ -28,7 +28,8 @@ use crate::send_wal::WalSenderState; use crate::state::TimelineMemState; use crate::state::TimelinePersistentState; use crate::timeline::get_timeline_dir; -use crate::timeline::FullAccessTimeline; +use crate::timeline::WalResidentTimeline; +use crate::timeline_manager; use crate::GlobalTimelines; use crate::SafeKeeperConf; @@ -168,6 +169,7 @@ pub struct Memory { pub last_removed_segno: XLogSegNo, pub epoch_start_lsn: Lsn, pub mem_state: TimelineMemState, + pub mgr_status: timeline_manager::Status, // PhysicalStorage state. pub write_lsn: Lsn, @@ -326,7 +328,7 @@ pub struct TimelineDigest { } pub async fn calculate_digest( - tli: &FullAccessTimeline, + tli: &WalResidentTimeline, request: TimelineDigestRequest, ) -> Result { if request.from_lsn > request.until_lsn { diff --git a/safekeeper/src/http/routes.rs b/safekeeper/src/http/routes.rs index 3f2cd97ccd..fe6d325cee 100644 --- a/safekeeper/src/http/routes.rs +++ b/safekeeper/src/http/routes.rs @@ -214,10 +214,10 @@ async fn timeline_snapshot_handler(request: Request) -> Result) -> Result) -> Result( async fn prepare_safekeeper( ttid: TenantTimelineId, pg_version: u32, -) -> anyhow::Result { +) -> anyhow::Result { let tli = GlobalTimelines::create( ttid, ServerInfo { @@ -115,11 +115,11 @@ async fn prepare_safekeeper( ) .await?; - tli.full_access_guard().await + tli.wal_residence_guard().await } async fn send_proposer_elected( - tli: &FullAccessTimeline, + tli: &WalResidentTimeline, term: Term, lsn: Lsn, ) -> anyhow::Result<()> { @@ -151,7 +151,7 @@ pub struct InsertedWAL { /// Extend local WAL with new LogicalMessage record. To do that, /// create AppendRequest with new WAL and pass it to safekeeper. pub async fn append_logical_message( - tli: &FullAccessTimeline, + tli: &WalResidentTimeline, msg: &AppendLogicalMessage, ) -> anyhow::Result { let wal_data = encode_logical_message(&msg.lm_prefix, &msg.lm_message); diff --git a/safekeeper/src/lib.rs b/safekeeper/src/lib.rs index cbd67f0064..067e425570 100644 --- a/safekeeper/src/lib.rs +++ b/safekeeper/src/lib.rs @@ -28,6 +28,8 @@ pub mod safekeeper; pub mod send_wal; pub mod state; pub mod timeline; +pub mod timeline_eviction; +pub mod timeline_guard; pub mod timeline_manager; pub mod timelines_set; pub mod wal_backup; @@ -49,6 +51,7 @@ pub mod defaults { pub const DEFAULT_HEARTBEAT_TIMEOUT: &str = "5000ms"; pub const DEFAULT_MAX_OFFLOADER_LAG_BYTES: u64 = 128 * (1 << 20); pub const DEFAULT_PARTIAL_BACKUP_TIMEOUT: &str = "15m"; + pub const DEFAULT_CONTROL_FILE_SAVE_INTERVAL: &str = "300s"; } #[derive(Debug, Clone)] @@ -85,6 +88,9 @@ pub struct SafeKeeperConf { pub partial_backup_enabled: bool, pub partial_backup_timeout: Duration, pub disable_periodic_broker_push: bool, + pub enable_offload: bool, + pub delete_offloaded_wal: bool, + pub control_file_save_interval: Duration, } impl SafeKeeperConf { @@ -124,6 +130,9 @@ impl SafeKeeperConf { partial_backup_enabled: false, partial_backup_timeout: Duration::from_secs(0), disable_periodic_broker_push: false, + enable_offload: false, + delete_offloaded_wal: false, + control_file_save_interval: Duration::from_secs(1), } } } diff --git a/safekeeper/src/pull_timeline.rs b/safekeeper/src/pull_timeline.rs index 66c41f65ff..618c6b278f 100644 --- a/safekeeper/src/pull_timeline.rs +++ b/safekeeper/src/pull_timeline.rs @@ -32,7 +32,7 @@ use crate::{ routes::TimelineStatus, }, safekeeper::Term, - timeline::{get_tenant_dir, get_timeline_dir, FullAccessTimeline, Timeline, TimelineError}, + timeline::{get_tenant_dir, get_timeline_dir, Timeline, TimelineError, WalResidentTimeline}, wal_storage::{self, open_wal_file, Storage}, GlobalTimelines, SafeKeeperConf, }; @@ -46,7 +46,7 @@ use utils::{ /// Stream tar archive of timeline to tx. #[instrument(name = "snapshot", skip_all, fields(ttid = %tli.ttid))] -pub async fn stream_snapshot(tli: FullAccessTimeline, tx: mpsc::Sender>) { +pub async fn stream_snapshot(tli: WalResidentTimeline, tx: mpsc::Sender>) { if let Err(e) = stream_snapshot_guts(tli, tx.clone()).await { // Error type/contents don't matter as they won't can't reach the client // (hyper likely doesn't do anything with it), but http stream will be @@ -66,7 +66,7 @@ pub struct SnapshotContext { pub flush_lsn: Lsn, pub wal_seg_size: usize, // used to remove WAL hold off in Drop. - pub tli: FullAccessTimeline, + pub tli: WalResidentTimeline, } impl Drop for SnapshotContext { @@ -80,7 +80,7 @@ impl Drop for SnapshotContext { } pub async fn stream_snapshot_guts( - tli: FullAccessTimeline, + tli: WalResidentTimeline, tx: mpsc::Sender>, ) -> Result<()> { // tokio-tar wants Write implementor, but we have mpsc tx >; @@ -135,7 +135,7 @@ pub async fn stream_snapshot_guts( Ok(()) } -impl FullAccessTimeline { +impl WalResidentTimeline { /// Start streaming tar archive with timeline: /// 1) stream control file under lock; /// 2) hold off WAL removal; @@ -160,6 +160,7 @@ impl FullAccessTimeline { ar: &mut tokio_tar::Builder, ) -> Result { let mut shared_state = self.write_shared_state().await; + let wal_seg_size = shared_state.get_wal_seg_size(); let cf_path = self.get_timeline_dir().join(CONTROL_FILE_NAME); let mut cf = File::open(cf_path).await?; @@ -173,19 +174,19 @@ impl FullAccessTimeline { // lock and setting `wal_removal_on_hold` later, it guarantees that WAL // won't be removed until we're done. let from_lsn = min( - shared_state.sk.state.remote_consistent_lsn, - shared_state.sk.state.backup_lsn, + shared_state.sk.state().remote_consistent_lsn, + shared_state.sk.state().backup_lsn, ); if from_lsn == Lsn::INVALID { // this is possible if snapshot is called before handling first // elected message bail!("snapshot is called on uninitialized timeline"); } - let from_segno = from_lsn.segment_number(shared_state.get_wal_seg_size()); - let term = shared_state.sk.get_term(); - let last_log_term = shared_state.sk.get_last_log_term(); + let from_segno = from_lsn.segment_number(wal_seg_size); + let term = shared_state.sk.state().acceptor_state.term; + let last_log_term = shared_state.sk.last_log_term(); let flush_lsn = shared_state.sk.flush_lsn(); - let upto_segno = flush_lsn.segment_number(shared_state.get_wal_seg_size()); + let upto_segno = flush_lsn.segment_number(wal_seg_size); // have some limit on max number of segments as a sanity check const MAX_ALLOWED_SEGS: u64 = 1000; let num_segs = upto_segno - from_segno + 1; @@ -206,14 +207,18 @@ impl FullAccessTimeline { } shared_state.wal_removal_on_hold = true; + // Drop shared_state to release the lock, before calling wal_residence_guard(). + drop(shared_state); + + let tli_copy = self.wal_residence_guard().await?; let bctx = SnapshotContext { from_segno, upto_segno, term, last_log_term, flush_lsn, - wal_seg_size: shared_state.get_wal_seg_size(), - tli: self.clone(), + wal_seg_size, + tli: tli_copy, }; Ok(bctx) @@ -225,8 +230,8 @@ impl FullAccessTimeline { /// forget this if snapshotting fails mid the way. pub async fn finish_snapshot(&self, bctx: &SnapshotContext) -> Result<()> { let shared_state = self.read_shared_state().await; - let term = shared_state.sk.get_term(); - let last_log_term = shared_state.sk.get_last_log_term(); + let term = shared_state.sk.state().acceptor_state.term; + let last_log_term = shared_state.sk.last_log_term(); // There are some cases to relax this check (e.g. last_log_term might // change, but as long as older history is strictly part of new that's // fine), but there is no need to do it. diff --git a/safekeeper/src/receive_wal.rs b/safekeeper/src/receive_wal.rs index 7943a2fd86..ab8c76dc17 100644 --- a/safekeeper/src/receive_wal.rs +++ b/safekeeper/src/receive_wal.rs @@ -6,7 +6,7 @@ use crate::handler::SafekeeperPostgresHandler; use crate::safekeeper::AcceptorProposerMessage; use crate::safekeeper::ProposerAcceptorMessage; use crate::safekeeper::ServerInfo; -use crate::timeline::FullAccessTimeline; +use crate::timeline::WalResidentTimeline; use crate::wal_service::ConnectionId; use crate::GlobalTimelines; use anyhow::{anyhow, Context}; @@ -213,7 +213,7 @@ impl SafekeeperPostgresHandler { &mut self, pgb: &mut PostgresBackend, ) -> Result<(), QueryError> { - let mut tli: Option = None; + let mut tli: Option = None; if let Err(end) = self.handle_start_wal_push_guts(pgb, &mut tli).await { // Log the result and probably send it to the client, closing the stream. let handle_end_fut = pgb.handle_copy_stream_end(end); @@ -233,7 +233,7 @@ impl SafekeeperPostgresHandler { pub async fn handle_start_wal_push_guts( &mut self, pgb: &mut PostgresBackend, - tli: &mut Option, + tli: &mut Option, ) -> Result<(), CopyStreamHandlerEnd> { // Notify the libpq client that it's allowed to send `CopyData` messages pgb.write_message(&BeMessage::CopyBothResponse).await?; @@ -269,11 +269,11 @@ impl SafekeeperPostgresHandler { .get_walreceivers() .pageserver_feedback_tx .subscribe(); - *tli = Some(timeline.clone()); + *tli = Some(timeline.wal_residence_guard().await?); tokio::select! { // todo: add read|write .context to these errors - r = network_reader.run(msg_tx, msg_rx, reply_tx, timeline.clone(), next_msg) => r, + r = network_reader.run(msg_tx, msg_rx, reply_tx, timeline, next_msg) => r, r = network_write(pgb, reply_rx, pageserver_feedback_rx) => r, } } else { @@ -323,7 +323,7 @@ struct NetworkReader<'a, IO> { impl<'a, IO: AsyncRead + AsyncWrite + Unpin> NetworkReader<'a, IO> { async fn read_first_message( &mut self, - ) -> Result<(FullAccessTimeline, ProposerAcceptorMessage), CopyStreamHandlerEnd> { + ) -> Result<(WalResidentTimeline, ProposerAcceptorMessage), CopyStreamHandlerEnd> { // Receive information about server to create timeline, if not yet. let next_msg = read_message(self.pgb_reader).await?; let tli = match next_msg { @@ -340,7 +340,7 @@ impl<'a, IO: AsyncRead + AsyncWrite + Unpin> NetworkReader<'a, IO> { let tli = GlobalTimelines::create(self.ttid, server_info, Lsn::INVALID, Lsn::INVALID) .await?; - tli.full_access_guard().await? + tli.wal_residence_guard().await? } _ => { return Err(CopyStreamHandlerEnd::Other(anyhow::anyhow!( @@ -356,7 +356,7 @@ impl<'a, IO: AsyncRead + AsyncWrite + Unpin> NetworkReader<'a, IO> { msg_tx: Sender, msg_rx: Receiver, reply_tx: Sender, - tli: FullAccessTimeline, + tli: WalResidentTimeline, next_msg: ProposerAcceptorMessage, ) -> Result<(), CopyStreamHandlerEnd> { *self.acceptor_handle = Some(WalAcceptor::spawn( @@ -451,7 +451,7 @@ const KEEPALIVE_INTERVAL: Duration = Duration::from_secs(1); /// replies to reply_tx; reading from socket and writing to disk in parallel is /// beneficial for performance, this struct provides writing to disk part. pub struct WalAcceptor { - tli: FullAccessTimeline, + tli: WalResidentTimeline, msg_rx: Receiver, reply_tx: Sender, conn_id: Option, @@ -464,7 +464,7 @@ impl WalAcceptor { /// /// conn_id None means WalAcceptor is used by recovery initiated at this safekeeper. pub fn spawn( - tli: FullAccessTimeline, + tli: WalResidentTimeline, msg_rx: Receiver, reply_tx: Sender, conn_id: Option, diff --git a/safekeeper/src/recovery.rs b/safekeeper/src/recovery.rs index 80a630b1e1..a59ff07b96 100644 --- a/safekeeper/src/recovery.rs +++ b/safekeeper/src/recovery.rs @@ -21,7 +21,7 @@ use utils::{id::NodeId, lsn::Lsn, postgres_client::wal_stream_connection_config} use crate::receive_wal::{WalAcceptor, REPLY_QUEUE_SIZE}; use crate::safekeeper::{AppendRequest, AppendRequestHeader}; -use crate::timeline::FullAccessTimeline; +use crate::timeline::WalResidentTimeline; use crate::{ http::routes::TimelineStatus, receive_wal::MSG_QUEUE_SIZE, @@ -36,7 +36,7 @@ use crate::{ /// Entrypoint for per timeline task which always runs, checking whether /// recovery for this safekeeper is needed and starting it if so. #[instrument(name = "recovery task", skip_all, fields(ttid = %tli.ttid))] -pub async fn recovery_main(tli: FullAccessTimeline, conf: SafeKeeperConf) { +pub async fn recovery_main(tli: WalResidentTimeline, conf: SafeKeeperConf) { info!("started"); let cancel = tli.cancel.clone(); @@ -66,12 +66,12 @@ pub async fn recovery_main(tli: FullAccessTimeline, conf: SafeKeeperConf) { /// depending on assembled quorum (e.g. classic picture 8 from Raft paper). /// Thus we don't try to predict it here. async fn recovery_needed( - tli: &FullAccessTimeline, + tli: &WalResidentTimeline, heartbeat_timeout: Duration, ) -> RecoveryNeededInfo { let ss = tli.read_shared_state().await; - let term = ss.sk.state.acceptor_state.term; - let last_log_term = ss.sk.get_last_log_term(); + let term = ss.sk.state().acceptor_state.term; + let last_log_term = ss.sk.last_log_term(); let flush_lsn = ss.sk.flush_lsn(); // note that peers contain myself, but that's ok -- we are interested only in peers which are strictly ahead of us. let mut peers = ss.get_peers(heartbeat_timeout); @@ -195,7 +195,7 @@ impl From<&PeerInfo> for Donor { const CHECK_INTERVAL_MS: u64 = 2000; /// Check regularly whether we need to start recovery. -async fn recovery_main_loop(tli: FullAccessTimeline, conf: SafeKeeperConf) { +async fn recovery_main_loop(tli: WalResidentTimeline, conf: SafeKeeperConf) { let check_duration = Duration::from_millis(CHECK_INTERVAL_MS); loop { let recovery_needed_info = recovery_needed(&tli, conf.heartbeat_timeout).await; @@ -205,7 +205,12 @@ async fn recovery_main_loop(tli: FullAccessTimeline, conf: SafeKeeperConf) { "starting recovery from donor {}: {}", donor.sk_id, recovery_needed_info ); - match recover(tli.clone(), donor, &conf).await { + let res = tli.wal_residence_guard().await; + if let Err(e) = res { + warn!("failed to obtain guard: {}", e); + continue; + } + match recover(res.unwrap(), donor, &conf).await { // Note: 'write_wal rewrites WAL written before' error is // expected here and might happen if compute and recovery // concurrently write the same data. Eventually compute @@ -228,7 +233,7 @@ async fn recovery_main_loop(tli: FullAccessTimeline, conf: SafeKeeperConf) { /// Recover from the specified donor. Returns message explaining normal finish /// reason or error. async fn recover( - tli: FullAccessTimeline, + tli: WalResidentTimeline, donor: &Donor, conf: &SafeKeeperConf, ) -> anyhow::Result { @@ -314,7 +319,7 @@ async fn recover( // Pull WAL from donor, assuming handshake is already done. async fn recovery_stream( - tli: FullAccessTimeline, + tli: WalResidentTimeline, donor: &Donor, start_streaming_at: Lsn, conf: &SafeKeeperConf, @@ -364,10 +369,10 @@ async fn recovery_stream( // As in normal walreceiver, do networking and writing to disk in parallel. let (msg_tx, msg_rx) = channel(MSG_QUEUE_SIZE); let (reply_tx, reply_rx) = channel(REPLY_QUEUE_SIZE); - let wa = WalAcceptor::spawn(tli.clone(), msg_rx, reply_tx, None); + let wa = WalAcceptor::spawn(tli.wal_residence_guard().await?, msg_rx, reply_tx, None); let res = tokio::select! { - r = network_io(physical_stream, msg_tx, donor.clone(), tli.clone(), conf.clone()) => r, + r = network_io(physical_stream, msg_tx, donor.clone(), tli, conf.clone()) => r, r = read_replies(reply_rx, donor.term) => r.map(|()| None), }; @@ -398,7 +403,7 @@ async fn network_io( physical_stream: ReplicationStream, msg_tx: Sender, donor: Donor, - tli: FullAccessTimeline, + tli: WalResidentTimeline, conf: SafeKeeperConf, ) -> anyhow::Result> { let mut physical_stream = pin!(physical_stream); diff --git a/safekeeper/src/remove_wal.rs b/safekeeper/src/remove_wal.rs index b661e48cb5..16239d847b 100644 --- a/safekeeper/src/remove_wal.rs +++ b/safekeeper/src/remove_wal.rs @@ -8,7 +8,7 @@ use crate::timeline_manager::StateSnapshot; /// While it is safe to use inmem values for determining horizon, /// we use persistent to make possible normal states less surprising. /// All segments covering LSNs before horizon_lsn can be removed. -pub fn calc_horizon_lsn(state: &StateSnapshot, extra_horizon_lsn: Option) -> Lsn { +pub(crate) fn calc_horizon_lsn(state: &StateSnapshot, extra_horizon_lsn: Option) -> Lsn { use std::cmp::min; let mut horizon_lsn = min( diff --git a/safekeeper/src/safekeeper.rs b/safekeeper/src/safekeeper.rs index ae230960ae..4d0992e8bd 100644 --- a/safekeeper/src/safekeeper.rs +++ b/safekeeper/src/safekeeper.rs @@ -499,7 +499,11 @@ where /// Accepts a control file storage containing the safekeeper state. /// State must be initialized, i.e. contain filled `tenant_id`, `timeline_id` /// and `server` (`wal_seg_size` inside it) fields. - pub fn new(state: CTRL, wal_store: WAL, node_id: NodeId) -> Result> { + pub fn new( + state: TimelineState, + wal_store: WAL, + node_id: NodeId, + ) -> Result> { if state.tenant_id == TenantId::from([0u8; 16]) || state.timeline_id == TimelineId::from([0u8; 16]) { @@ -512,7 +516,7 @@ where Ok(SafeKeeper { term_start_lsn: Lsn(0), - state: TimelineState::new(state), + state, wal_store, node_id, }) @@ -526,11 +530,6 @@ where .up_to(self.flush_lsn()) } - /// Get current term. - pub fn get_term(&self) -> Term { - self.state.acceptor_state.term - } - pub fn get_last_log_term(&self) -> Term { self.state .acceptor_state @@ -912,10 +911,8 @@ where ))) } - /// Update timeline state with peer safekeeper data. + /// Update commit_lsn from peer safekeeper data. pub async fn record_safekeeper_info(&mut self, sk_info: &SafekeeperTimelineInfo) -> Result<()> { - let mut sync_control_file = false; - if (Lsn(sk_info.commit_lsn) != Lsn::INVALID) && (sk_info.last_log_term != INVALID_TERM) { // Note: the check is too restrictive, generally we can update local // commit_lsn if our history matches (is part of) history of advanced @@ -924,29 +921,6 @@ where self.update_commit_lsn(Lsn(sk_info.commit_lsn)).await?; } } - - self.state.inmem.backup_lsn = max(Lsn(sk_info.backup_lsn), self.state.inmem.backup_lsn); - sync_control_file |= self.state.backup_lsn + (self.state.server.wal_seg_size as u64) - < self.state.inmem.backup_lsn; - - self.state.inmem.remote_consistent_lsn = max( - Lsn(sk_info.remote_consistent_lsn), - self.state.inmem.remote_consistent_lsn, - ); - sync_control_file |= self.state.remote_consistent_lsn - + (self.state.server.wal_seg_size as u64) - < self.state.inmem.remote_consistent_lsn; - - self.state.inmem.peer_horizon_lsn = max( - Lsn(sk_info.peer_horizon_lsn), - self.state.inmem.peer_horizon_lsn, - ); - sync_control_file |= self.state.peer_horizon_lsn + (self.state.server.wal_seg_size as u64) - < self.state.inmem.peer_horizon_lsn; - - if sync_control_file { - self.state.flush().await?; - } Ok(()) } } @@ -958,7 +932,7 @@ mod tests { use super::*; use crate::{ - state::{PersistedPeers, TimelinePersistentState}, + state::{EvictionState, PersistedPeers, TimelinePersistentState}, wal_storage::Storage, }; use std::{ops::Deref, str::FromStr, time::Instant}; @@ -1039,7 +1013,7 @@ mod tests { persisted_state: test_sk_state(), }; let wal_store = DummyWalStore { lsn: Lsn(0) }; - let mut sk = SafeKeeper::new(storage, wal_store, NodeId(0)).unwrap(); + let mut sk = SafeKeeper::new(TimelineState::new(storage), wal_store, NodeId(0)).unwrap(); // check voting for 1 is ok let vote_request = ProposerAcceptorMessage::VoteRequest(VoteRequest { term: 1 }); @@ -1055,7 +1029,7 @@ mod tests { persisted_state: state, }; - sk = SafeKeeper::new(storage, sk.wal_store, NodeId(0)).unwrap(); + sk = SafeKeeper::new(TimelineState::new(storage), sk.wal_store, NodeId(0)).unwrap(); // and ensure voting second time for 1 is not ok vote_resp = sk.process_msg(&vote_request).await; @@ -1072,7 +1046,7 @@ mod tests { }; let wal_store = DummyWalStore { lsn: Lsn(0) }; - let mut sk = SafeKeeper::new(storage, wal_store, NodeId(0)).unwrap(); + let mut sk = SafeKeeper::new(TimelineState::new(storage), wal_store, NodeId(0)).unwrap(); let mut ar_hdr = AppendRequestHeader { term: 1, @@ -1225,6 +1199,7 @@ mod tests { }, )]), partial_backup: crate::wal_backup_partial::State::default(), + eviction_state: EvictionState::Present, }; let ser = state.ser().unwrap(); @@ -1272,6 +1247,8 @@ mod tests { 0xb0, 0x01, 0x96, 0x49, 0x00, 0x00, 0x00, 0x00, // partial_backup 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + // eviction_state + 0x00, 0x00, 0x00, 0x00, ]; assert_eq!(Hex(&ser), Hex(&expected)); diff --git a/safekeeper/src/send_wal.rs b/safekeeper/src/send_wal.rs index df75893838..90b1604adb 100644 --- a/safekeeper/src/send_wal.rs +++ b/safekeeper/src/send_wal.rs @@ -5,7 +5,7 @@ use crate::handler::SafekeeperPostgresHandler; use crate::metrics::RECEIVED_PS_FEEDBACKS; use crate::receive_wal::WalReceivers; use crate::safekeeper::{Term, TermLsn}; -use crate::timeline::FullAccessTimeline; +use crate::timeline::WalResidentTimeline; use crate::wal_service::ConnectionId; use crate::wal_storage::WalReader; use crate::GlobalTimelines; @@ -387,10 +387,10 @@ impl SafekeeperPostgresHandler { term: Option, ) -> Result<(), QueryError> { let tli = GlobalTimelines::get(self.ttid).map_err(|e| QueryError::Other(e.into()))?; - let full_access = tli.full_access_guard().await?; + let residence_guard = tli.wal_residence_guard().await?; if let Err(end) = self - .handle_start_replication_guts(pgb, start_pos, term, full_access) + .handle_start_replication_guts(pgb, start_pos, term, residence_guard) .await { let info = tli.get_safekeeper_info(&self.conf).await; @@ -407,7 +407,7 @@ impl SafekeeperPostgresHandler { pgb: &mut PostgresBackend, start_pos: Lsn, term: Option, - tli: FullAccessTimeline, + tli: WalResidentTimeline, ) -> Result<(), CopyStreamHandlerEnd> { let appname = self.appname.clone(); @@ -458,7 +458,8 @@ impl SafekeeperPostgresHandler { let mut sender = WalSender { pgb, - tli: tli.clone(), + // should succeed since we're already holding another guard + tli: tli.wal_residence_guard().await?, appname, start_pos, end_pos, @@ -527,7 +528,7 @@ impl EndWatch { /// A half driving sending WAL. struct WalSender<'a, IO> { pgb: &'a mut PostgresBackend, - tli: FullAccessTimeline, + tli: WalResidentTimeline, appname: Option, // Position since which we are sending next chunk. start_pos: Lsn, @@ -736,7 +737,7 @@ impl WalSender<'_, IO> { struct ReplyReader { reader: PostgresBackendReader, ws_guard: Arc, - tli: FullAccessTimeline, + tli: WalResidentTimeline, } impl ReplyReader { diff --git a/safekeeper/src/state.rs b/safekeeper/src/state.rs index be5e516296..e0f7b65aef 100644 --- a/safekeeper/src/state.rs +++ b/safekeeper/src/state.rs @@ -63,11 +63,26 @@ pub struct TimelinePersistentState { /// Holds names of partial segments uploaded to remote storage. Used to /// clean up old objects without leaving garbage in remote storage. pub partial_backup: wal_backup_partial::State, + /// Eviction state of the timeline. If it's Offloaded, we should download + /// WAL files from remote storage to serve the timeline. + pub eviction_state: EvictionState, } #[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] pub struct PersistedPeers(pub Vec<(NodeId, PersistedPeerInfo)>); +/// State of the local WAL files. Used to track current timeline state, +/// that can be either WAL files are present on disk or last partial segment +/// is offloaded to remote storage. +#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq)] +pub enum EvictionState { + /// WAL files are present on disk. + Present, + /// Last partial segment is offloaded to remote storage. + /// Contains flush_lsn of the last offloaded segment. + Offloaded(Lsn), +} + impl TimelinePersistentState { pub fn new( ttid: &TenantTimelineId, @@ -98,6 +113,7 @@ impl TimelinePersistentState { .collect(), ), partial_backup: wal_backup_partial::State::default(), + eviction_state: EvictionState::Present, } } diff --git a/safekeeper/src/timeline.rs b/safekeeper/src/timeline.rs index 544ffdbb36..f632cd6fb3 100644 --- a/safekeeper/src/timeline.rs +++ b/safekeeper/src/timeline.rs @@ -31,12 +31,15 @@ use crate::safekeeper::{ INVALID_TERM, }; use crate::send_wal::WalSenders; -use crate::state::{TimelineMemState, TimelinePersistentState}; +use crate::state::{EvictionState, TimelineMemState, TimelinePersistentState, TimelineState}; +use crate::timeline_guard::ResidenceGuard; +use crate::timeline_manager::{AtomicStatus, ManagerCtl}; use crate::timelines_set::TimelinesSet; use crate::wal_backup::{self}; +use crate::wal_backup_partial::PartialRemoteSegment; use crate::{control_file, safekeeper::UNKNOWN_SERVER_VERSION}; -use crate::metrics::FullTimelineInfo; +use crate::metrics::{FullTimelineInfo, WalStorageMetrics}; use crate::wal_storage::{Storage as wal_storage_iface, WalReader}; use crate::{debug_dump, timeline_manager, wal_storage}; use crate::{GlobalTimelines, SafeKeeperConf}; @@ -132,8 +135,9 @@ impl<'a> DerefMut for WriteGuardSharedState<'a> { impl<'a> Drop for WriteGuardSharedState<'a> { fn drop(&mut self) { - let term_flush_lsn = TermLsn::from((self.guard.sk.get_term(), self.guard.sk.flush_lsn())); - let commit_lsn = self.guard.sk.state.inmem.commit_lsn; + let term_flush_lsn = + TermLsn::from((self.guard.sk.last_log_term(), self.guard.sk.flush_lsn())); + let commit_lsn = self.guard.sk.state().inmem.commit_lsn; let _ = self.tli.term_flush_lsn_watch_tx.send_if_modified(|old| { if *old != term_flush_lsn { @@ -162,10 +166,150 @@ impl<'a> Drop for WriteGuardSharedState<'a> { } } +/// This structure is stored in shared state and represents the state of the timeline. +/// Usually it holds SafeKeeper, but it also supports offloaded timeline state. In this +/// case, SafeKeeper is not available (because WAL is not present on disk) and all +/// operations can be done only with control file. +pub enum StateSK { + Loaded(SafeKeeper), + Offloaded(Box>), + // Not used, required for moving between states. + Empty, +} + +impl StateSK { + pub fn flush_lsn(&self) -> Lsn { + match self { + StateSK::Loaded(sk) => sk.wal_store.flush_lsn(), + StateSK::Offloaded(state) => match state.eviction_state { + EvictionState::Offloaded(flush_lsn) => flush_lsn, + _ => panic!("StateSK::Offloaded mismatches with eviction_state from control_file"), + }, + StateSK::Empty => unreachable!(), + } + } + + /// Get a reference to the control file's timeline state. + pub fn state(&self) -> &TimelineState { + match self { + StateSK::Loaded(sk) => &sk.state, + StateSK::Offloaded(ref s) => s, + StateSK::Empty => unreachable!(), + } + } + + pub fn state_mut(&mut self) -> &mut TimelineState { + match self { + StateSK::Loaded(sk) => &mut sk.state, + StateSK::Offloaded(ref mut s) => s, + StateSK::Empty => unreachable!(), + } + } + + pub fn last_log_term(&self) -> Term { + self.state() + .acceptor_state + .get_last_log_term(self.flush_lsn()) + } + + /// Close open WAL files to release FDs. + fn close_wal_store(&mut self) { + if let StateSK::Loaded(sk) = self { + sk.wal_store.close(); + } + } + + /// Update timeline state with peer safekeeper data. + pub async fn record_safekeeper_info(&mut self, sk_info: &SafekeeperTimelineInfo) -> Result<()> { + // update commit_lsn if safekeeper is loaded + match self { + StateSK::Loaded(sk) => sk.record_safekeeper_info(sk_info).await?, + StateSK::Offloaded(_) => {} + StateSK::Empty => unreachable!(), + } + + // update everything else, including remote_consistent_lsn and backup_lsn + let mut sync_control_file = false; + let state = self.state_mut(); + let wal_seg_size = state.server.wal_seg_size as u64; + + state.inmem.backup_lsn = max(Lsn(sk_info.backup_lsn), state.inmem.backup_lsn); + sync_control_file |= state.backup_lsn + wal_seg_size < state.inmem.backup_lsn; + + state.inmem.remote_consistent_lsn = max( + Lsn(sk_info.remote_consistent_lsn), + state.inmem.remote_consistent_lsn, + ); + sync_control_file |= + state.remote_consistent_lsn + wal_seg_size < state.inmem.remote_consistent_lsn; + + state.inmem.peer_horizon_lsn = + max(Lsn(sk_info.peer_horizon_lsn), state.inmem.peer_horizon_lsn); + sync_control_file |= state.peer_horizon_lsn + wal_seg_size < state.inmem.peer_horizon_lsn; + + if sync_control_file { + state.flush().await?; + } + Ok(()) + } + + /// Previously known as epoch_start_lsn. Needed only for reference in some APIs. + pub fn term_start_lsn(&self) -> Lsn { + match self { + StateSK::Loaded(sk) => sk.term_start_lsn, + StateSK::Offloaded(_) => Lsn(0), + StateSK::Empty => unreachable!(), + } + } + + /// Used for metrics only. + pub fn wal_storage_metrics(&self) -> WalStorageMetrics { + match self { + StateSK::Loaded(sk) => sk.wal_store.get_metrics(), + StateSK::Offloaded(_) => WalStorageMetrics::default(), + StateSK::Empty => unreachable!(), + } + } + + /// Returns WAL storage internal LSNs for debug dump. + pub fn wal_storage_internal_state(&self) -> (Lsn, Lsn, Lsn, bool) { + match self { + StateSK::Loaded(sk) => sk.wal_store.internal_state(), + StateSK::Offloaded(_) => { + let flush_lsn = self.flush_lsn(); + (flush_lsn, flush_lsn, flush_lsn, false) + } + StateSK::Empty => unreachable!(), + } + } + + /// Access to SafeKeeper object. Panics if offloaded, should be good to use from WalResidentTimeline. + pub fn safekeeper( + &mut self, + ) -> &mut SafeKeeper { + match self { + StateSK::Loaded(sk) => sk, + StateSK::Offloaded(_) => { + panic!("safekeeper is offloaded, cannot be used") + } + StateSK::Empty => unreachable!(), + } + } + + /// Moves control file's state structure out of the enum. Used to switch states. + fn take_state(self) -> TimelineState { + match self { + StateSK::Loaded(sk) => sk.state, + StateSK::Offloaded(state) => *state, + StateSK::Empty => unreachable!(), + } + } +} + /// Shared state associated with database instance pub struct SharedState { /// Safekeeper object - pub(crate) sk: SafeKeeper, + pub(crate) sk: StateSK, /// In memory list containing state of peers sent in latest messages from them. pub(crate) peers_info: PeersInfo, // True value hinders old WAL removal; this is used by snapshotting. We @@ -203,10 +347,10 @@ impl SharedState { control_file::FileStorage::create_new(timeline_dir.clone(), conf, state)?; let wal_store = wal_storage::PhysicalStorage::new(ttid, timeline_dir, conf, &control_store)?; - let sk = SafeKeeper::new(control_store, wal_store, conf.my_id)?; + let sk = SafeKeeper::new(TimelineState::new(control_store), wal_store, conf.my_id)?; Ok(Self { - sk, + sk: StateSK::Loaded(sk), peers_info: PeersInfo(vec![]), wal_removal_on_hold: false, }) @@ -220,18 +364,30 @@ impl SharedState { bail!(TimelineError::UninitializedWalSegSize(*ttid)); } - let wal_store = - wal_storage::PhysicalStorage::new(ttid, timeline_dir, conf, &control_store)?; + let sk = match control_store.eviction_state { + EvictionState::Present => { + let wal_store = + wal_storage::PhysicalStorage::new(ttid, timeline_dir, conf, &control_store)?; + StateSK::Loaded(SafeKeeper::new( + TimelineState::new(control_store), + wal_store, + conf.my_id, + )?) + } + EvictionState::Offloaded(_) => { + StateSK::Offloaded(Box::new(TimelineState::new(control_store))) + } + }; Ok(Self { - sk: SafeKeeper::new(control_store, wal_store, conf.my_id)?, + sk, peers_info: PeersInfo(vec![]), wal_removal_on_hold: false, }) } pub(crate) fn get_wal_seg_size(&self) -> usize { - self.sk.state.server.wal_seg_size as usize + self.sk.state().server.wal_seg_size as usize } fn get_safekeeper_info( @@ -246,20 +402,20 @@ impl SharedState { tenant_id: ttid.tenant_id.as_ref().to_owned(), timeline_id: ttid.timeline_id.as_ref().to_owned(), }), - term: self.sk.state.acceptor_state.term, - last_log_term: self.sk.get_last_log_term(), + term: self.sk.state().acceptor_state.term, + last_log_term: self.sk.last_log_term(), flush_lsn: self.sk.flush_lsn().0, // note: this value is not flushed to control file yet and can be lost - commit_lsn: self.sk.state.inmem.commit_lsn.0, - remote_consistent_lsn: self.sk.state.inmem.remote_consistent_lsn.0, - peer_horizon_lsn: self.sk.state.inmem.peer_horizon_lsn.0, + commit_lsn: self.sk.state().inmem.commit_lsn.0, + remote_consistent_lsn: self.sk.state().inmem.remote_consistent_lsn.0, + peer_horizon_lsn: self.sk.state().inmem.peer_horizon_lsn.0, safekeeper_connstr: conf .advertise_pg_addr .to_owned() .unwrap_or(conf.listen_pg_addr.clone()), http_connstr: conf.listen_http_addr.to_owned(), - backup_lsn: self.sk.state.inmem.backup_lsn.0, - local_start_lsn: self.sk.state.local_start_lsn.0, + backup_lsn: self.sk.state().inmem.backup_lsn.0, + local_start_lsn: self.sk.state().local_start_lsn.0, availability_zone: conf.availability_zone.clone(), standby_horizon: standby_apply_lsn.0, } @@ -335,6 +491,7 @@ pub struct Timeline { walsenders: Arc, walreceivers: Arc, timeline_dir: Utf8PathBuf, + manager_ctl: ManagerCtl, /// Delete/cancel will trigger this, background tasks should drop out as soon as it fires pub(crate) cancel: CancellationToken, @@ -343,6 +500,7 @@ pub struct Timeline { pub(crate) broker_active: AtomicBool, pub(crate) wal_backup_active: AtomicBool, pub(crate) last_removed_segno: AtomicU64, + pub(crate) mgr_status: AtomicStatus, } impl Timeline { @@ -352,9 +510,9 @@ impl Timeline { let shared_state = SharedState::restore(conf, &ttid)?; let (commit_lsn_watch_tx, commit_lsn_watch_rx) = - watch::channel(shared_state.sk.state.commit_lsn); + watch::channel(shared_state.sk.state().commit_lsn); let (term_flush_lsn_watch_tx, term_flush_lsn_watch_rx) = watch::channel(TermLsn::from(( - shared_state.sk.get_term(), + shared_state.sk.last_log_term(), shared_state.sk.flush_lsn(), ))); let (shared_state_version_tx, shared_state_version_rx) = watch::channel(0); @@ -373,9 +531,11 @@ impl Timeline { walreceivers, cancel: CancellationToken::default(), timeline_dir: get_timeline_dir(conf, &ttid), + manager_ctl: ManagerCtl::new(), broker_active: AtomicBool::new(false), wal_backup_active: AtomicBool::new(false), last_removed_segno: AtomicU64::new(0), + mgr_status: AtomicStatus::new(), }) } @@ -409,9 +569,11 @@ impl Timeline { walreceivers, cancel: CancellationToken::default(), timeline_dir: get_timeline_dir(conf, &ttid), + manager_ctl: ManagerCtl::new(), broker_active: AtomicBool::new(false), wal_backup_active: AtomicBool::new(false), last_removed_segno: AtomicU64::new(0), + mgr_status: AtomicStatus::new(), }) } @@ -442,7 +604,7 @@ impl Timeline { fs::create_dir_all(&self.timeline_dir).await?; // Write timeline to disk and start background tasks. - if let Err(e) = shared_state.sk.state.flush().await { + if let Err(e) = shared_state.sk.state_mut().flush().await { // Bootstrap failed, cancel timeline and remove timeline directory. self.cancel(shared_state); @@ -465,12 +627,16 @@ impl Timeline { conf: &SafeKeeperConf, broker_active_set: Arc, ) { + let (tx, rx) = self.manager_ctl.bootstrap_manager(); + // Start manager task which will monitor timeline state and update // background tasks. tokio::spawn(timeline_manager::main_task( - self.clone(), + ManagerTimeline { tli: self.clone() }, conf.clone(), broker_active_set, + tx, + rx, )); } @@ -507,7 +673,7 @@ impl Timeline { self.cancel.cancel(); // Close associated FDs. Nobody will be able to touch timeline data once // it is cancelled, so WAL storage won't be opened again. - shared_state.sk.wal_store.close(); + shared_state.sk.close_wal_store(); } /// Returns if timeline is cancelled. @@ -547,12 +713,15 @@ impl Timeline { /// Returns state of the timeline. pub async fn get_state(&self) -> (TimelineMemState, TimelinePersistentState) { let state = self.read_shared_state().await; - (state.sk.state.inmem.clone(), state.sk.state.clone()) + ( + state.sk.state().inmem.clone(), + TimelinePersistentState::clone(state.sk.state()), + ) } /// Returns latest backup_lsn. pub async fn get_wal_backup_lsn(&self) -> Lsn { - self.read_shared_state().await.sk.state.inmem.backup_lsn + self.read_shared_state().await.sk.state().inmem.backup_lsn } /// Sets backup_lsn to the given value. @@ -562,7 +731,7 @@ impl Timeline { } let mut state = self.write_shared_state().await; - state.sk.state.inmem.backup_lsn = max(state.sk.state.inmem.backup_lsn, backup_lsn); + state.sk.state_mut().inmem.backup_lsn = max(state.sk.state().inmem.backup_lsn, backup_lsn); // we should check whether to shut down offloader, but this will be done // soon by peer communication anyway. Ok(()) @@ -604,7 +773,7 @@ impl Timeline { /// Returns flush_lsn. pub async fn get_flush_lsn(&self) -> Lsn { - self.read_shared_state().await.sk.wal_store.flush_lsn() + self.read_shared_state().await.sk.flush_lsn() } /// Gather timeline data for metrics. @@ -623,11 +792,11 @@ impl Timeline { timeline_is_active: self.broker_active.load(Ordering::Relaxed), num_computes: self.walreceivers.get_num() as u32, last_removed_segno: self.last_removed_segno.load(Ordering::Relaxed), - epoch_start_lsn: state.sk.term_start_lsn, - mem_state: state.sk.state.inmem.clone(), - persisted_state: state.sk.state.clone(), - flush_lsn: state.sk.wal_store.flush_lsn(), - wal_storage: state.sk.wal_store.get_metrics(), + epoch_start_lsn: state.sk.term_start_lsn(), + mem_state: state.sk.state().inmem.clone(), + persisted_state: TimelinePersistentState::clone(state.sk.state()), + flush_lsn: state.sk.flush_lsn(), + wal_storage: state.sk.wal_storage_metrics(), }) } @@ -636,7 +805,7 @@ impl Timeline { let state = self.read_shared_state().await; let (write_lsn, write_record_lsn, flush_lsn, file_open) = - state.sk.wal_store.internal_state(); + state.sk.wal_storage_internal_state(); debug_dump::Memory { is_cancelled: self.is_cancelled(), @@ -646,8 +815,9 @@ impl Timeline { active: self.broker_active.load(Ordering::Relaxed), num_computes: self.walreceivers.get_num() as u32, last_removed_segno: self.last_removed_segno.load(Ordering::Relaxed), - epoch_start_lsn: state.sk.term_start_lsn, - mem_state: state.sk.state.inmem.clone(), + epoch_start_lsn: state.sk.term_start_lsn(), + mem_state: state.sk.state().inmem.clone(), + mgr_status: self.mgr_status.get(), write_lsn, write_record_lsn, flush_lsn, @@ -661,34 +831,77 @@ impl Timeline { f: impl FnOnce(&mut TimelinePersistentState) -> Result, ) -> Result { let mut state = self.write_shared_state().await; - let mut persistent_state = state.sk.state.start_change(); + let mut persistent_state = state.sk.state_mut().start_change(); // If f returns error, we abort the change and don't persist anything. let res = f(&mut persistent_state)?; // If persisting fails, we abort the change and return error. - state.sk.state.finish_change(&persistent_state).await?; + state + .sk + .state_mut() + .finish_change(&persistent_state) + .await?; Ok(res) } /// Get the timeline guard for reading/writing WAL files. - /// TODO: if WAL files are not present on disk (evicted), they will be - /// downloaded from S3. Also there will logic for preventing eviction - /// while someone is holding FullAccessTimeline guard. - pub async fn full_access_guard(self: &Arc) -> Result { + /// If WAL files are not present on disk (evicted), they will be automatically + /// downloaded from remote storage. This is done in the manager task, which is + /// responsible for issuing all guards. + /// + /// NB: don't use this function from timeline_manager, it will deadlock. + /// NB: don't use this function while holding shared_state lock. + pub async fn wal_residence_guard(self: &Arc) -> Result { if self.is_cancelled() { bail!(TimelineError::Cancelled(self.ttid)); } - Ok(FullAccessTimeline { tli: self.clone() }) + + debug!("requesting WalResidentTimeline guard"); + + // Wait 5 seconds for the guard to be acquired, should be enough for uneviction. + // If it times out, most likely there is a deadlock in the manager task. + let res = tokio::time::timeout( + Duration::from_secs(5), + self.manager_ctl.wal_residence_guard(), + ) + .await; + + let guard = match res { + Ok(Ok(guard)) => guard, + Ok(Err(e)) => { + warn!( + "error while acquiring WalResidentTimeline guard (current state {:?}): {}", + self.mgr_status.get(), + e + ); + return Err(e); + } + Err(_) => { + warn!( + "timeout while acquiring WalResidentTimeline guard (current state {:?})", + self.mgr_status.get() + ); + anyhow::bail!("timeout while acquiring WalResidentTimeline guard"); + } + }; + + Ok(WalResidentTimeline::new(self.clone(), guard)) } } /// This is a guard that allows to read/write disk timeline state. -/// All tasks that are using the disk should use this guard. -#[derive(Clone)] -pub struct FullAccessTimeline { +/// All tasks that are trying to read/write WAL from disk should use this guard. +pub struct WalResidentTimeline { pub tli: Arc, + _guard: ResidenceGuard, } -impl Deref for FullAccessTimeline { +impl WalResidentTimeline { + pub fn new(tli: Arc, _guard: ResidenceGuard) -> Self { + WalResidentTimeline { tli, _guard } + } +} + +impl Deref for WalResidentTimeline { type Target = Arc; fn deref(&self) -> &Self::Target { @@ -696,7 +909,7 @@ impl Deref for FullAccessTimeline { } } -impl FullAccessTimeline { +impl WalResidentTimeline { /// Returns true if walsender should stop sending WAL to pageserver. We /// terminate it if remote_consistent_lsn reached commit_lsn and there is no /// computes. While there might be nothing to stream already, we learn about @@ -708,8 +921,8 @@ impl FullAccessTimeline { } let shared_state = self.read_shared_state().await; if self.walreceivers.get_num() == 0 { - return shared_state.sk.state.inmem.commit_lsn == Lsn(0) || // no data at all yet - reported_remote_consistent_lsn >= shared_state.sk.state.inmem.commit_lsn; + return shared_state.sk.state().inmem.commit_lsn == Lsn(0) || // no data at all yet + reported_remote_consistent_lsn >= shared_state.sk.state().inmem.commit_lsn; } false } @@ -717,11 +930,11 @@ impl FullAccessTimeline { /// Ensure that current term is t, erroring otherwise, and lock the state. pub async fn acquire_term(&self, t: Term) -> Result { let ss = self.read_shared_state().await; - if ss.sk.state.acceptor_state.term != t { + if ss.sk.state().acceptor_state.term != t { bail!( "failed to acquire term {}, current term {}", t, - ss.sk.state.acceptor_state.term + ss.sk.state().acceptor_state.term ); } Ok(ss) @@ -739,7 +952,7 @@ impl FullAccessTimeline { let mut rmsg: Option; { let mut shared_state = self.write_shared_state().await; - rmsg = shared_state.sk.process_msg(msg).await?; + rmsg = shared_state.sk.safekeeper().process_msg(msg).await?; // if this is AppendResponse, fill in proper hot standby feedback. if let Some(AcceptorProposerMessage::AppendResponse(ref mut resp)) = rmsg { @@ -769,8 +982,141 @@ impl FullAccessTimeline { /// Update in memory remote consistent lsn. pub async fn update_remote_consistent_lsn(&self, candidate: Lsn) { let mut shared_state = self.write_shared_state().await; - shared_state.sk.state.inmem.remote_consistent_lsn = - max(shared_state.sk.state.inmem.remote_consistent_lsn, candidate); + shared_state.sk.state_mut().inmem.remote_consistent_lsn = max( + shared_state.sk.state().inmem.remote_consistent_lsn, + candidate, + ); + } +} + +/// This struct contains methods that are used by timeline manager task. +pub(crate) struct ManagerTimeline { + pub(crate) tli: Arc, +} + +impl Deref for ManagerTimeline { + type Target = Arc; + + fn deref(&self) -> &Self::Target { + &self.tli + } +} + +impl ManagerTimeline { + pub(crate) fn timeline_dir(&self) -> &Utf8PathBuf { + &self.tli.timeline_dir + } + + /// Manager requests this state on startup. + pub(crate) async fn bootstrap_mgr(&self) -> (bool, Option) { + let shared_state = self.read_shared_state().await; + let is_offloaded = matches!( + shared_state.sk.state().eviction_state, + EvictionState::Offloaded(_) + ); + let partial_backup_uploaded = shared_state.sk.state().partial_backup.uploaded_segment(); + + (is_offloaded, partial_backup_uploaded) + } + + /// Try to switch state Present->Offloaded. + pub(crate) async fn switch_to_offloaded( + &self, + partial: &PartialRemoteSegment, + ) -> anyhow::Result<()> { + let mut shared = self.write_shared_state().await; + + // updating control file + let mut pstate = shared.sk.state_mut().start_change(); + + if !matches!(pstate.eviction_state, EvictionState::Present) { + bail!( + "cannot switch to offloaded state, current state is {:?}", + pstate.eviction_state + ); + } + + if partial.flush_lsn != shared.sk.flush_lsn() { + bail!( + "flush_lsn mismatch in partial backup, expected {}, got {}", + shared.sk.flush_lsn(), + partial.flush_lsn + ); + } + + if partial.commit_lsn != pstate.commit_lsn { + bail!( + "commit_lsn mismatch in partial backup, expected {}, got {}", + pstate.commit_lsn, + partial.commit_lsn + ); + } + + if partial.term != shared.sk.last_log_term() { + bail!( + "term mismatch in partial backup, expected {}, got {}", + shared.sk.last_log_term(), + partial.term + ); + } + + pstate.eviction_state = EvictionState::Offloaded(shared.sk.flush_lsn()); + shared.sk.state_mut().finish_change(&pstate).await?; + // control file is now switched to Offloaded state + + // now we can switch shared.sk to Offloaded, shouldn't fail + let prev_sk = std::mem::replace(&mut shared.sk, StateSK::Empty); + let cfile_state = prev_sk.take_state(); + shared.sk = StateSK::Offloaded(Box::new(cfile_state)); + + Ok(()) + } + + /// Try to switch state Offloaded->Present. + pub(crate) async fn switch_to_present(&self) -> anyhow::Result<()> { + let conf = GlobalTimelines::get_global_config(); + let mut shared = self.write_shared_state().await; + + // trying to restore WAL storage + let wal_store = wal_storage::PhysicalStorage::new( + &self.ttid, + self.timeline_dir.clone(), + &conf, + shared.sk.state(), + )?; + + // updating control file + let mut pstate = shared.sk.state_mut().start_change(); + + if !matches!(pstate.eviction_state, EvictionState::Offloaded(_)) { + bail!( + "cannot switch to present state, current state is {:?}", + pstate.eviction_state + ); + } + + if wal_store.flush_lsn() != shared.sk.flush_lsn() { + bail!( + "flush_lsn mismatch in restored WAL, expected {}, got {}", + shared.sk.flush_lsn(), + wal_store.flush_lsn() + ); + } + + pstate.eviction_state = EvictionState::Present; + shared.sk.state_mut().finish_change(&pstate).await?; + + // now we can switch shared.sk to Present, shouldn't fail + let prev_sk = std::mem::replace(&mut shared.sk, StateSK::Empty); + let cfile_state = prev_sk.take_state(); + shared.sk = StateSK::Loaded(SafeKeeper::new(cfile_state, wal_store, conf.my_id)?); + + Ok(()) + } + + /// Update current manager state, useful for debugging manager deadlocks. + pub(crate) fn set_status(&self, status: timeline_manager::Status) { + self.mgr_status.store(status, Ordering::Relaxed); } } @@ -784,13 +1130,13 @@ async fn delete_dir(path: &Utf8PathBuf) -> Result { } /// Get a path to the tenant directory. If you just need to get a timeline directory, -/// use FullAccessTimeline::get_timeline_dir instead. +/// use WalResidentTimeline::get_timeline_dir instead. pub(crate) fn get_tenant_dir(conf: &SafeKeeperConf, tenant_id: &TenantId) -> Utf8PathBuf { conf.workdir.join(tenant_id.to_string()) } /// Get a path to the timeline directory. If you need to read WAL files from disk, -/// use FullAccessTimeline::get_timeline_dir instead. This function does not check +/// use WalResidentTimeline::get_timeline_dir instead. This function does not check /// timeline eviction status and WAL files might not be present on disk. pub(crate) fn get_timeline_dir(conf: &SafeKeeperConf, ttid: &TenantTimelineId) -> Utf8PathBuf { get_tenant_dir(conf, &ttid.tenant_id).join(ttid.timeline_id.to_string()) diff --git a/safekeeper/src/timeline_eviction.rs b/safekeeper/src/timeline_eviction.rs new file mode 100644 index 0000000000..b303d41b7b --- /dev/null +++ b/safekeeper/src/timeline_eviction.rs @@ -0,0 +1,366 @@ +//! Code related to evicting WAL files to remote storage. The actual upload is done by the +//! partial WAL backup code. This file has code to delete and re-download WAL files, +//! cross-validate with partial WAL backup if local file is still present. + +use anyhow::Context; +use camino::Utf8PathBuf; +use remote_storage::RemotePath; +use tokio::{ + fs::File, + io::{AsyncRead, AsyncWriteExt}, +}; +use tracing::{debug, info, instrument, warn}; +use utils::crashsafe::durable_rename; + +use crate::{ + timeline_manager::{Manager, StateSnapshot}, + wal_backup, + wal_backup_partial::{self, PartialRemoteSegment}, + wal_storage::wal_file_paths, +}; + +impl Manager { + /// Returns true if the timeline is ready for eviction. + /// Current criteria: + /// - no active tasks + /// - control file is flushed (no next event scheduled) + /// - no WAL residence guards + /// - no pushes to the broker + /// - partial WAL backup is uploaded + pub(crate) fn ready_for_eviction( + &self, + next_event: &Option, + state: &StateSnapshot, + ) -> bool { + self.backup_task.is_none() + && self.recovery_task.is_none() + && self.wal_removal_task.is_none() + && self.partial_backup_task.is_none() + && self.partial_backup_uploaded.is_some() + && next_event.is_none() + && self.access_service.is_empty() + && !self.tli_broker_active.get() + && !wal_backup_partial::needs_uploading(state, &self.partial_backup_uploaded) + && self + .partial_backup_uploaded + .as_ref() + .unwrap() + .flush_lsn + .segment_number(self.wal_seg_size) + == self.last_removed_segno + 1 + } + + /// Evict the timeline to remote storage. + #[instrument(name = "evict_timeline", skip_all)] + pub(crate) async fn evict_timeline(&mut self) { + assert!(!self.is_offloaded); + let partial_backup_uploaded = match &self.partial_backup_uploaded { + Some(p) => p.clone(), + None => { + warn!("no partial backup uploaded, skipping eviction"); + return; + } + }; + + info!("starting eviction, using {:?}", partial_backup_uploaded); + + if let Err(e) = do_eviction(self, &partial_backup_uploaded).await { + warn!("failed to evict timeline: {:?}", e); + return; + } + + info!("successfully evicted timeline"); + } + + /// Restore evicted timeline from remote storage. + #[instrument(name = "unevict_timeline", skip_all)] + pub(crate) async fn unevict_timeline(&mut self) { + assert!(self.is_offloaded); + let partial_backup_uploaded = match &self.partial_backup_uploaded { + Some(p) => p.clone(), + None => { + warn!("no partial backup uploaded, cannot unevict"); + return; + } + }; + + info!("starting uneviction, using {:?}", partial_backup_uploaded); + + if let Err(e) = do_uneviction(self, &partial_backup_uploaded).await { + warn!("failed to unevict timeline: {:?}", e); + return; + } + + info!("successfully restored evicted timeline"); + } +} + +/// Ensure that content matches the remote partial backup, if local segment exists. +/// Then change state in control file and in-memory. If `delete_offloaded_wal` is set, +/// delete the local segment. +async fn do_eviction(mgr: &mut Manager, partial: &PartialRemoteSegment) -> anyhow::Result<()> { + compare_local_segment_with_remote(mgr, partial).await?; + + mgr.tli.switch_to_offloaded(partial).await?; + // switch manager state as soon as possible + mgr.is_offloaded = true; + + if mgr.conf.delete_offloaded_wal { + delete_local_segment(mgr, partial).await?; + } + + Ok(()) +} + +/// Ensure that content matches the remote partial backup, if local segment exists. +/// Then download segment to local disk and change state in control file and in-memory. +async fn do_uneviction(mgr: &mut Manager, partial: &PartialRemoteSegment) -> anyhow::Result<()> { + // if the local segment is present, validate it + compare_local_segment_with_remote(mgr, partial).await?; + + // atomically download the partial segment + redownload_partial_segment(mgr, partial).await?; + + mgr.tli.switch_to_present().await?; + // switch manager state as soon as possible + mgr.is_offloaded = false; + + Ok(()) +} + +/// Delete local WAL segment. +async fn delete_local_segment(mgr: &Manager, partial: &PartialRemoteSegment) -> anyhow::Result<()> { + let local_path = local_segment_path(mgr, partial); + + info!("deleting WAL file to evict: {}", local_path); + tokio::fs::remove_file(&local_path).await?; + Ok(()) +} + +/// Redownload partial segment from remote storage. +/// The segment is downloaded to a temporary file and then renamed to the final path. +async fn redownload_partial_segment( + mgr: &Manager, + partial: &PartialRemoteSegment, +) -> anyhow::Result<()> { + let tmp_file = mgr.tli.timeline_dir().join("remote_partial.tmp"); + let remote_segfile = remote_segment_path(mgr, partial)?; + + debug!( + "redownloading partial segment: {} -> {}", + remote_segfile, tmp_file + ); + + let mut reader = wal_backup::read_object(&remote_segfile, 0).await?; + let mut file = File::create(&tmp_file).await?; + + let actual_len = tokio::io::copy(&mut reader, &mut file).await?; + let expected_len = partial.flush_lsn.segment_offset(mgr.wal_seg_size); + + if actual_len != expected_len as u64 { + anyhow::bail!( + "partial downloaded {} bytes, expected {}", + actual_len, + expected_len + ); + } + + if actual_len > mgr.wal_seg_size as u64 { + anyhow::bail!( + "remote segment is too long: {} bytes, expected {}", + actual_len, + mgr.wal_seg_size + ); + } + file.set_len(mgr.wal_seg_size as u64).await?; + file.flush().await?; + + let final_path = local_segment_path(mgr, partial); + info!( + "downloaded {} bytes, renaming to {}", + final_path, final_path, + ); + if let Err(e) = durable_rename(&tmp_file, &final_path, !mgr.conf.no_sync).await { + // Probably rename succeeded, but fsync of it failed. Remove + // the file then to avoid using it. + tokio::fs::remove_file(tmp_file) + .await + .or_else(utils::fs_ext::ignore_not_found)?; + return Err(e.into()); + } + + Ok(()) +} + +/// Compare local WAL segment with partial WAL backup in remote storage. +/// If the local segment is not present, the function does nothing. +/// If the local segment is present, it compares the local segment with the remote one. +async fn compare_local_segment_with_remote( + mgr: &Manager, + partial: &PartialRemoteSegment, +) -> anyhow::Result<()> { + let local_path = local_segment_path(mgr, partial); + + match File::open(&local_path).await { + Ok(mut local_file) => do_validation(mgr, &mut local_file, mgr.wal_seg_size, partial) + .await + .context("validation failed"), + Err(_) => { + info!( + "local WAL file {} is not present, skipping validation", + local_path + ); + Ok(()) + } + } +} + +/// Compare opened local WAL segment with partial WAL backup in remote storage. +/// Validate full content of both files. +async fn do_validation( + mgr: &Manager, + file: &mut File, + wal_seg_size: usize, + partial: &PartialRemoteSegment, +) -> anyhow::Result<()> { + let local_size = file.metadata().await?.len() as usize; + if local_size != wal_seg_size { + anyhow::bail!( + "local segment size is invalid: found {}, expected {}", + local_size, + wal_seg_size + ); + } + + let remote_segfile = remote_segment_path(mgr, partial)?; + let mut remote_reader: std::pin::Pin> = + wal_backup::read_object(&remote_segfile, 0).await?; + + // remote segment should have bytes excatly up to `flush_lsn` + let expected_remote_size = partial.flush_lsn.segment_offset(mgr.wal_seg_size); + // let's compare the first `expected_remote_size` bytes + compare_n_bytes(&mut remote_reader, file, expected_remote_size).await?; + // and check that the remote segment ends here + check_end(&mut remote_reader).await?; + + // if local segment is longer, the rest should be zeroes + read_n_zeroes(file, mgr.wal_seg_size - expected_remote_size).await?; + // and check that the local segment ends here + check_end(file).await?; + + Ok(()) +} + +fn local_segment_path(mgr: &Manager, partial: &PartialRemoteSegment) -> Utf8PathBuf { + let flush_lsn = partial.flush_lsn; + let segno = flush_lsn.segment_number(mgr.wal_seg_size); + let (_, local_partial_segfile) = + wal_file_paths(mgr.tli.timeline_dir(), segno, mgr.wal_seg_size); + local_partial_segfile +} + +fn remote_segment_path( + mgr: &Manager, + partial: &PartialRemoteSegment, +) -> anyhow::Result { + let remote_timeline_path = wal_backup::remote_timeline_path(&mgr.tli.ttid)?; + Ok(partial.remote_path(&remote_timeline_path)) +} + +/// Compare first `n` bytes of two readers. If the bytes differ, return an error. +/// If the readers are shorter than `n`, return an error. +async fn compare_n_bytes(reader1: &mut R1, reader2: &mut R2, n: usize) -> anyhow::Result<()> +where + R1: AsyncRead + Unpin, + R2: AsyncRead + Unpin, +{ + use tokio::io::AsyncReadExt; + + const BUF_SIZE: usize = 32 * 1024; + + let mut buffer1 = vec![0u8; BUF_SIZE]; + let mut buffer2 = vec![0u8; BUF_SIZE]; + + let mut offset = 0; + + while offset < n { + let bytes_to_read = std::cmp::min(BUF_SIZE, n - offset); + + let bytes_read1 = reader1 + .read(&mut buffer1[..bytes_to_read]) + .await + .with_context(|| format!("failed to read from reader1 at offset {}", offset))?; + if bytes_read1 == 0 { + anyhow::bail!("unexpected EOF from reader1 at offset {}", offset); + } + + let bytes_read2 = reader2 + .read_exact(&mut buffer2[..bytes_read1]) + .await + .with_context(|| { + format!( + "failed to read {} bytes from reader2 at offset {}", + bytes_read1, offset + ) + })?; + assert!(bytes_read2 == bytes_read1); + + if buffer1[..bytes_read1] != buffer2[..bytes_read2] { + let diff_offset = buffer1[..bytes_read1] + .iter() + .zip(buffer2[..bytes_read2].iter()) + .position(|(a, b)| a != b) + .expect("mismatched buffers, but no difference found"); + anyhow::bail!("mismatch at offset {}", offset + diff_offset); + } + + offset += bytes_read1; + } + + Ok(()) +} + +async fn check_end(mut reader: R) -> anyhow::Result<()> +where + R: AsyncRead + Unpin, +{ + use tokio::io::AsyncReadExt; + + let mut buffer = [0u8; 1]; + let bytes_read = reader.read(&mut buffer).await?; + if bytes_read != 0 { + anyhow::bail!("expected EOF, found bytes"); + } + Ok(()) +} + +async fn read_n_zeroes(reader: &mut R, n: usize) -> anyhow::Result<()> +where + R: AsyncRead + Unpin, +{ + use tokio::io::AsyncReadExt; + + const BUF_SIZE: usize = 32 * 1024; + let mut buffer = vec![0u8; BUF_SIZE]; + let mut offset = 0; + + while offset < n { + let bytes_to_read = std::cmp::min(BUF_SIZE, n - offset); + + let bytes_read = reader + .read(&mut buffer[..bytes_to_read]) + .await + .context("expected zeroes, got read error")?; + if bytes_read == 0 { + anyhow::bail!("expected zeroes, got EOF"); + } + + if buffer[..bytes_read].iter().all(|&b| b == 0) { + offset += bytes_read; + } else { + anyhow::bail!("non-zero byte found"); + } + } + + Ok(()) +} diff --git a/safekeeper/src/timeline_guard.rs b/safekeeper/src/timeline_guard.rs new file mode 100644 index 0000000000..e249c859b4 --- /dev/null +++ b/safekeeper/src/timeline_guard.rs @@ -0,0 +1,71 @@ +//! Timeline residence guard is needed to ensure that WAL segments are present on disk, +//! as long as the code is holding the guard. This file implements guard logic, to issue +//! and drop guards, and to notify the manager when the guard is dropped. + +use std::collections::HashSet; + +use tracing::{debug, warn}; + +use crate::timeline_manager::ManagerCtlMessage; + +#[derive(Debug, Clone, Copy)] +pub struct GuardId(u64); + +pub struct ResidenceGuard { + manager_tx: tokio::sync::mpsc::UnboundedSender, + guard_id: GuardId, +} + +impl Drop for ResidenceGuard { + fn drop(&mut self) { + // notify the manager that the guard is dropped + let res = self + .manager_tx + .send(ManagerCtlMessage::GuardDrop(self.guard_id)); + if let Err(e) = res { + warn!("failed to send GuardDrop message: {:?}", e); + } + } +} + +/// AccessService is responsible for issuing and dropping residence guards. +/// All guards are stored in the `guards` set. +/// TODO: it's possible to add `String` name to each guard, for better observability. +pub(crate) struct AccessService { + next_guard_id: u64, + guards: HashSet, + manager_tx: tokio::sync::mpsc::UnboundedSender, +} + +impl AccessService { + pub(crate) fn new(manager_tx: tokio::sync::mpsc::UnboundedSender) -> Self { + Self { + next_guard_id: 0, + guards: HashSet::new(), + manager_tx, + } + } + + pub(crate) fn is_empty(&self) -> bool { + self.guards.is_empty() + } + + pub(crate) fn create_guard(&mut self) -> ResidenceGuard { + let guard_id = self.next_guard_id; + self.next_guard_id += 1; + self.guards.insert(guard_id); + + let guard_id = GuardId(guard_id); + debug!("issued a new guard {:?}", guard_id); + + ResidenceGuard { + manager_tx: self.manager_tx.clone(), + guard_id, + } + } + + pub(crate) fn drop_guard(&mut self, guard_id: GuardId) { + debug!("dropping guard {:?}", guard_id); + assert!(self.guards.remove(&guard_id.0)); + } +} diff --git a/safekeeper/src/timeline_manager.rs b/safekeeper/src/timeline_manager.rs index 592426bba3..c3abeac644 100644 --- a/safekeeper/src/timeline_manager.rs +++ b/safekeeper/src/timeline_manager.rs @@ -2,66 +2,83 @@ //! It is spawned alongside each timeline and exits when the timeline is deleted. //! It watches for changes in the timeline state and decides when to spawn or kill background tasks. //! It also can manage some reactive state, like should the timeline be active for broker pushes or not. +//! +//! Be aware that you need to be extra careful with manager code, because it is not respawned on panic. +//! Also, if it will stuck in some branch, it will prevent any further progress in the timeline. use std::{ - sync::Arc, - time::{Duration, Instant}, + sync::{atomic::AtomicUsize, Arc}, + time::Duration, }; use postgres_ffi::XLogSegNo; -use tokio::task::{JoinError, JoinHandle}; -use tracing::{info, info_span, instrument, warn, Instrument}; +use serde::{Deserialize, Serialize}; +use tokio::{ + task::{JoinError, JoinHandle}, + time::Instant, +}; +use tracing::{debug, info, info_span, instrument, warn, Instrument}; use utils::lsn::Lsn; use crate::{ - control_file::Storage, + control_file::{FileStorage, Storage}, metrics::{MANAGER_ACTIVE_CHANGES, MANAGER_ITERATIONS_TOTAL}, recovery::recovery_main, remove_wal::calc_horizon_lsn, + safekeeper::Term, send_wal::WalSenders, - timeline::{PeerInfo, ReadGuardSharedState, Timeline}, + state::TimelineState, + timeline::{ManagerTimeline, PeerInfo, ReadGuardSharedState, StateSK, WalResidentTimeline}, + timeline_guard::{AccessService, GuardId, ResidenceGuard}, timelines_set::{TimelineSetGuard, TimelinesSet}, wal_backup::{self, WalBackupTaskHandle}, - wal_backup_partial, SafeKeeperConf, + wal_backup_partial::{self, PartialRemoteSegment}, + SafeKeeperConf, }; -pub struct StateSnapshot { +pub(crate) struct StateSnapshot { // inmem values - pub commit_lsn: Lsn, - pub backup_lsn: Lsn, - pub remote_consistent_lsn: Lsn, + pub(crate) commit_lsn: Lsn, + pub(crate) backup_lsn: Lsn, + pub(crate) remote_consistent_lsn: Lsn, // persistent control file values - pub cfile_peer_horizon_lsn: Lsn, - pub cfile_remote_consistent_lsn: Lsn, - pub cfile_backup_lsn: Lsn, + pub(crate) cfile_peer_horizon_lsn: Lsn, + pub(crate) cfile_remote_consistent_lsn: Lsn, + pub(crate) cfile_backup_lsn: Lsn, + + // latest state + pub(crate) flush_lsn: Lsn, + pub(crate) last_log_term: Term, // misc - pub cfile_last_persist_at: Instant, - pub inmem_flush_pending: bool, - pub wal_removal_on_hold: bool, - pub peers: Vec, + pub(crate) cfile_last_persist_at: std::time::Instant, + pub(crate) inmem_flush_pending: bool, + pub(crate) wal_removal_on_hold: bool, + pub(crate) peers: Vec, } impl StateSnapshot { /// Create a new snapshot of the timeline state. fn new(read_guard: ReadGuardSharedState, heartbeat_timeout: Duration) -> Self { + let state = read_guard.sk.state(); Self { - commit_lsn: read_guard.sk.state.inmem.commit_lsn, - backup_lsn: read_guard.sk.state.inmem.backup_lsn, - remote_consistent_lsn: read_guard.sk.state.inmem.remote_consistent_lsn, - cfile_peer_horizon_lsn: read_guard.sk.state.peer_horizon_lsn, - cfile_remote_consistent_lsn: read_guard.sk.state.remote_consistent_lsn, - cfile_backup_lsn: read_guard.sk.state.backup_lsn, - cfile_last_persist_at: read_guard.sk.state.pers.last_persist_at(), - inmem_flush_pending: Self::has_unflushed_inmem_state(&read_guard), + commit_lsn: state.inmem.commit_lsn, + backup_lsn: state.inmem.backup_lsn, + remote_consistent_lsn: state.inmem.remote_consistent_lsn, + cfile_peer_horizon_lsn: state.peer_horizon_lsn, + cfile_remote_consistent_lsn: state.remote_consistent_lsn, + cfile_backup_lsn: state.backup_lsn, + flush_lsn: read_guard.sk.flush_lsn(), + last_log_term: read_guard.sk.last_log_term(), + cfile_last_persist_at: state.pers.last_persist_at(), + inmem_flush_pending: Self::has_unflushed_inmem_state(state), wal_removal_on_hold: read_guard.wal_removal_on_hold, peers: read_guard.get_peers(heartbeat_timeout), } } - fn has_unflushed_inmem_state(read_guard: &ReadGuardSharedState) -> bool { - let state = &read_guard.sk.state; + fn has_unflushed_inmem_state(state: &TimelineState) -> bool { state.inmem.commit_lsn > state.commit_lsn || state.inmem.backup_lsn > state.backup_lsn || state.inmem.peer_horizon_lsn > state.peer_horizon_lsn @@ -73,314 +90,560 @@ impl StateSnapshot { /// There is no need to check for updates more often than this. const REFRESH_INTERVAL: Duration = Duration::from_millis(300); -/// How often to save the control file if the is no other activity. -const CF_SAVE_INTERVAL: Duration = Duration::from_secs(300); +pub enum ManagerCtlMessage { + /// Request to get a guard for WalResidentTimeline, with WAL files available locally. + GuardRequest(tokio::sync::oneshot::Sender>), + /// Request to drop the guard. + GuardDrop(GuardId), +} + +impl std::fmt::Debug for ManagerCtlMessage { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + ManagerCtlMessage::GuardRequest(_) => write!(f, "GuardRequest"), + ManagerCtlMessage::GuardDrop(id) => write!(f, "GuardDrop({:?})", id), + } + } +} + +pub struct ManagerCtl { + manager_tx: tokio::sync::mpsc::UnboundedSender, + + // this is used to initialize manager, it will be moved out in bootstrap(). + init_manager_rx: + std::sync::Mutex>>, +} + +impl Default for ManagerCtl { + fn default() -> Self { + Self::new() + } +} + +impl ManagerCtl { + pub fn new() -> Self { + let (tx, rx) = tokio::sync::mpsc::unbounded_channel(); + Self { + manager_tx: tx, + init_manager_rx: std::sync::Mutex::new(Some(rx)), + } + } + + /// Issue a new guard and wait for manager to prepare the timeline. + /// Sends a message to the manager and waits for the response. + /// Can be blocked indefinitely if the manager is stuck. + pub async fn wal_residence_guard(&self) -> anyhow::Result { + let (tx, rx) = tokio::sync::oneshot::channel(); + self.manager_tx.send(ManagerCtlMessage::GuardRequest(tx))?; + + // wait for the manager to respond with the guard + rx.await + .map_err(|e| anyhow::anyhow!("response read fail: {:?}", e)) + .and_then(std::convert::identity) + } + + /// Must be called exactly once to bootstrap the manager. + pub fn bootstrap_manager( + &self, + ) -> ( + tokio::sync::mpsc::UnboundedSender, + tokio::sync::mpsc::UnboundedReceiver, + ) { + let rx = self + .init_manager_rx + .lock() + .expect("mutex init_manager_rx poisoned") + .take() + .expect("manager already bootstrapped"); + + (self.manager_tx.clone(), rx) + } +} + +pub(crate) struct Manager { + // configuration & dependencies + pub(crate) tli: ManagerTimeline, + pub(crate) conf: SafeKeeperConf, + pub(crate) wal_seg_size: usize, + pub(crate) walsenders: Arc, + + // current state + pub(crate) state_version_rx: tokio::sync::watch::Receiver, + pub(crate) num_computes_rx: tokio::sync::watch::Receiver, + pub(crate) tli_broker_active: TimelineSetGuard, + pub(crate) last_removed_segno: XLogSegNo, + pub(crate) is_offloaded: bool, + + // background tasks + pub(crate) backup_task: Option, + pub(crate) recovery_task: Option>, + pub(crate) wal_removal_task: Option>>, + + // partial backup + pub(crate) partial_backup_task: Option>>, + pub(crate) partial_backup_uploaded: Option, + + // misc + pub(crate) access_service: AccessService, +} /// This task gets spawned alongside each timeline and is responsible for managing the timeline's /// background tasks. /// Be careful, this task is not respawned on panic, so it should not panic. #[instrument(name = "manager", skip_all, fields(ttid = %tli.ttid))] pub async fn main_task( - tli: Arc, + tli: ManagerTimeline, conf: SafeKeeperConf, broker_active_set: Arc, + manager_tx: tokio::sync::mpsc::UnboundedSender, + mut manager_rx: tokio::sync::mpsc::UnboundedReceiver, ) { + tli.set_status(Status::Started); + + let defer_tli = tli.tli.clone(); scopeguard::defer! { - if tli.is_cancelled() { + if defer_tli.is_cancelled() { info!("manager task finished"); } else { warn!("manager task finished prematurely"); } }; - // configuration & dependencies - let wal_seg_size = tli.get_wal_seg_size().await; - let heartbeat_timeout = conf.heartbeat_timeout; - let walsenders = tli.get_walsenders(); - let walreceivers = tli.get_walreceivers(); - - // current state - let mut state_version_rx = tli.get_state_version_rx(); - let mut num_computes_rx = walreceivers.get_num_rx(); - let mut tli_broker_active = broker_active_set.guard(tli.clone()); - let mut last_removed_segno = 0 as XLogSegNo; - - // list of background tasks - let mut backup_task: Option = None; - let mut recovery_task: Option> = None; - let mut partial_backup_task: Option> = None; - let mut wal_removal_task: Option>> = None; + let mut mgr = Manager::new(tli, conf, broker_active_set, manager_tx).await; // Start recovery task which always runs on the timeline. - if conf.peer_recovery_enabled { - match tli.full_access_guard().await { - Ok(tli) => { - recovery_task = Some(tokio::spawn(recovery_main(tli, conf.clone()))); - } - Err(e) => { - warn!("failed to start recovery task: {:?}", e); - } - } - } - - // Start partial backup task which always runs on the timeline. - if conf.is_wal_backup_enabled() && conf.partial_backup_enabled { - match tli.full_access_guard().await { - Ok(tli) => { - partial_backup_task = Some(tokio::spawn(wal_backup_partial::main_task( - tli, - conf.clone(), - ))); - } - Err(e) => { - warn!("failed to start partial backup task: {:?}", e); - } - } + if !mgr.is_offloaded && mgr.conf.peer_recovery_enabled { + let tli = mgr.wal_resident_timeline(); + mgr.recovery_task = Some(tokio::spawn(recovery_main(tli, mgr.conf.clone()))); } let last_state = 'outer: loop { MANAGER_ITERATIONS_TOTAL.inc(); - let state_snapshot = StateSnapshot::new(tli.read_shared_state().await, heartbeat_timeout); - let num_computes = *num_computes_rx.borrow(); + mgr.set_status(Status::StateSnapshot); + let state_snapshot = mgr.state_snapshot().await; - let is_wal_backup_required = update_backup( - &conf, - &tli, - wal_seg_size, - num_computes, - &state_snapshot, - &mut backup_task, - ) - .await; + let mut next_event: Option = None; + if !mgr.is_offloaded { + let num_computes = *mgr.num_computes_rx.borrow(); - let _is_active = update_is_active( - is_wal_backup_required, - num_computes, - &state_snapshot, - &mut tli_broker_active, - &tli, - ); + mgr.set_status(Status::UpdateBackup); + let is_wal_backup_required = mgr.update_backup(num_computes, &state_snapshot).await; + mgr.update_is_active(is_wal_backup_required, num_computes, &state_snapshot); - let next_cfile_save = update_control_file_save(&state_snapshot, &tli).await; + mgr.set_status(Status::UpdateControlFile); + mgr.update_control_file_save(&state_snapshot, &mut next_event) + .await; - update_wal_removal( - &conf, - walsenders, - &tli, - wal_seg_size, - &state_snapshot, - last_removed_segno, - &mut wal_removal_task, - ) - .await; + mgr.set_status(Status::UpdateWalRemoval); + mgr.update_wal_removal(&state_snapshot).await; + mgr.set_status(Status::UpdatePartialBackup); + mgr.update_partial_backup(&state_snapshot).await; + + if mgr.conf.enable_offload && mgr.ready_for_eviction(&next_event, &state_snapshot) { + mgr.set_status(Status::EvictTimeline); + mgr.evict_timeline().await; + } + } + + mgr.set_status(Status::Wait); // wait until something changes. tx channels are stored under Arc, so they will not be // dropped until the manager task is finished. tokio::select! { - _ = tli.cancel.cancelled() => { + _ = mgr.tli.cancel.cancelled() => { // timeline was deleted break 'outer state_snapshot; } _ = async { // don't wake up on every state change, but at most every REFRESH_INTERVAL tokio::time::sleep(REFRESH_INTERVAL).await; - let _ = state_version_rx.changed().await; + let _ = mgr.state_version_rx.changed().await; } => { // state was updated } - _ = num_computes_rx.changed() => { + _ = mgr.num_computes_rx.changed() => { // number of connected computes was updated } - _ = async { - if let Some(timeout) = next_cfile_save { - tokio::time::sleep_until(timeout).await - } else { - futures::future::pending().await - } - } => { - // it's time to save the control file + _ = sleep_until(&next_event) => { + // we were waiting for some event (e.g. cfile save) } - res = async { - if let Some(task) = &mut wal_removal_task { - task.await - } else { - futures::future::pending().await - } - } => { + res = await_task_finish(&mut mgr.wal_removal_task) => { // WAL removal task finished - wal_removal_task = None; - update_wal_removal_end(res, &tli, &mut last_removed_segno); + mgr.wal_removal_task = None; + mgr.update_wal_removal_end(res); + } + res = await_task_finish(&mut mgr.partial_backup_task) => { + // partial backup task finished + mgr.partial_backup_task = None; + mgr.update_partial_backup_end(res); + } + + msg = manager_rx.recv() => { + mgr.set_status(Status::HandleMessage); + mgr.handle_message(msg).await; } } }; + mgr.set_status(Status::Exiting); // remove timeline from the broker active set sooner, before waiting for background tasks - tli_broker_active.set(false); + mgr.tli_broker_active.set(false); // shutdown background tasks - if conf.is_wal_backup_enabled() { - wal_backup::update_task(&conf, &tli, false, &last_state, &mut backup_task).await; + if mgr.conf.is_wal_backup_enabled() { + wal_backup::update_task(&mut mgr, false, &last_state).await; } - if let Some(recovery_task) = recovery_task { + if let Some(recovery_task) = &mut mgr.recovery_task { if let Err(e) = recovery_task.await { warn!("recovery task failed: {:?}", e); } } - if let Some(partial_backup_task) = partial_backup_task { + if let Some(partial_backup_task) = &mut mgr.partial_backup_task { if let Err(e) = partial_backup_task.await { warn!("partial backup task failed: {:?}", e); } } - if let Some(wal_removal_task) = wal_removal_task { + if let Some(wal_removal_task) = &mut mgr.wal_removal_task { let res = wal_removal_task.await; - update_wal_removal_end(res, &tli, &mut last_removed_segno); + mgr.update_wal_removal_end(res); } + + mgr.set_status(Status::Finished); } -/// Spawns/kills backup task and returns true if backup is required. -async fn update_backup( - conf: &SafeKeeperConf, - tli: &Arc, - wal_seg_size: usize, - num_computes: usize, - state: &StateSnapshot, - backup_task: &mut Option, -) -> bool { - let is_wal_backup_required = - wal_backup::is_wal_backup_required(wal_seg_size, num_computes, state); - - if conf.is_wal_backup_enabled() { - wal_backup::update_task(conf, tli, is_wal_backup_required, state, backup_task).await; +impl Manager { + async fn new( + tli: ManagerTimeline, + conf: SafeKeeperConf, + broker_active_set: Arc, + manager_tx: tokio::sync::mpsc::UnboundedSender, + ) -> Manager { + let (is_offloaded, partial_backup_uploaded) = tli.bootstrap_mgr().await; + Manager { + conf, + wal_seg_size: tli.get_wal_seg_size().await, + walsenders: tli.get_walsenders().clone(), + state_version_rx: tli.get_state_version_rx(), + num_computes_rx: tli.get_walreceivers().get_num_rx(), + tli_broker_active: broker_active_set.guard(tli.clone()), + last_removed_segno: 0, + is_offloaded, + backup_task: None, + recovery_task: None, + wal_removal_task: None, + partial_backup_task: None, + partial_backup_uploaded, + access_service: AccessService::new(manager_tx), + tli, + } } - // update the state in Arc - tli.wal_backup_active - .store(backup_task.is_some(), std::sync::atomic::Ordering::Relaxed); - is_wal_backup_required -} - -/// Update is_active flag and returns its value. -fn update_is_active( - is_wal_backup_required: bool, - num_computes: usize, - state: &StateSnapshot, - tli_broker_active: &mut TimelineSetGuard, - tli: &Arc, -) -> bool { - let is_active = is_wal_backup_required - || num_computes > 0 - || state.remote_consistent_lsn < state.commit_lsn; - - // update the broker timeline set - if tli_broker_active.set(is_active) { - // write log if state has changed - info!( - "timeline active={} now, remote_consistent_lsn={}, commit_lsn={}", - is_active, state.remote_consistent_lsn, state.commit_lsn, - ); - - MANAGER_ACTIVE_CHANGES.inc(); + fn set_status(&self, status: Status) { + self.tli.set_status(status); } - // update the state in Arc - tli.broker_active - .store(is_active, std::sync::atomic::Ordering::Relaxed); - is_active -} - -/// Save control file if needed. Returns Instant if we should persist the control file in the future. -async fn update_control_file_save( - state: &StateSnapshot, - tli: &Arc, -) -> Option { - if !state.inmem_flush_pending { - return None; + /// Get a WalResidentTimeline. + /// Manager code must use this function instead of one from `Timeline` + /// directly, because it will deadlock. + pub(crate) fn wal_resident_timeline(&mut self) -> WalResidentTimeline { + assert!(!self.is_offloaded); + let guard = self.access_service.create_guard(); + WalResidentTimeline::new(self.tli.clone(), guard) } - if state.cfile_last_persist_at.elapsed() > CF_SAVE_INTERVAL { - let mut write_guard = tli.write_shared_state().await; - // this can be done in the background because it blocks manager task, but flush() should - // be fast enough not to be a problem now - if let Err(e) = write_guard.sk.state.flush().await { - warn!("failed to save control file: {:?}", e); + /// Get a snapshot of the timeline state. + async fn state_snapshot(&self) -> StateSnapshot { + StateSnapshot::new( + self.tli.read_shared_state().await, + self.conf.heartbeat_timeout, + ) + } + + /// Spawns/kills backup task and returns true if backup is required. + async fn update_backup(&mut self, num_computes: usize, state: &StateSnapshot) -> bool { + let is_wal_backup_required = + wal_backup::is_wal_backup_required(self.wal_seg_size, num_computes, state); + + if self.conf.is_wal_backup_enabled() { + wal_backup::update_task(self, is_wal_backup_required, state).await; } - None - } else { - // we should wait until next CF_SAVE_INTERVAL - Some((state.cfile_last_persist_at + CF_SAVE_INTERVAL).into()) - } -} - -/// Spawns WAL removal task if needed. -async fn update_wal_removal( - conf: &SafeKeeperConf, - walsenders: &Arc, - tli: &Arc, - wal_seg_size: usize, - state: &StateSnapshot, - last_removed_segno: u64, - wal_removal_task: &mut Option>>, -) { - if wal_removal_task.is_some() || state.wal_removal_on_hold { - // WAL removal is already in progress or hold off - return; - } - - // If enabled, we use LSN of the most lagging walsender as a WAL removal horizon. - // This allows to get better read speed for pageservers that are lagging behind, - // at the cost of keeping more WAL on disk. - let replication_horizon_lsn = if conf.walsenders_keep_horizon { - walsenders.laggard_lsn() - } else { - None - }; - - let removal_horizon_lsn = calc_horizon_lsn(state, replication_horizon_lsn); - let removal_horizon_segno = removal_horizon_lsn - .segment_number(wal_seg_size) - .saturating_sub(1); - - if removal_horizon_segno > last_removed_segno { - // we need to remove WAL - let remover = crate::wal_storage::Storage::remove_up_to( - &tli.read_shared_state().await.sk.wal_store, - removal_horizon_segno, + // update the state in Arc + self.tli.wal_backup_active.store( + self.backup_task.is_some(), + std::sync::atomic::Ordering::Relaxed, ); - *wal_removal_task = Some(tokio::spawn( - async move { - remover.await?; - Ok(removal_horizon_segno) + is_wal_backup_required + } + + /// Update is_active flag and returns its value. + fn update_is_active( + &mut self, + is_wal_backup_required: bool, + num_computes: usize, + state: &StateSnapshot, + ) { + let is_active = is_wal_backup_required + || num_computes > 0 + || state.remote_consistent_lsn < state.commit_lsn; + + // update the broker timeline set + if self.tli_broker_active.set(is_active) { + // write log if state has changed + info!( + "timeline active={} now, remote_consistent_lsn={}, commit_lsn={}", + is_active, state.remote_consistent_lsn, state.commit_lsn, + ); + + MANAGER_ACTIVE_CHANGES.inc(); + } + + // update the state in Arc + self.tli + .broker_active + .store(is_active, std::sync::atomic::Ordering::Relaxed); + } + + /// Save control file if needed. Returns Instant if we should persist the control file in the future. + async fn update_control_file_save( + &self, + state: &StateSnapshot, + next_event: &mut Option, + ) { + if !state.inmem_flush_pending { + return; + } + + if state.cfile_last_persist_at.elapsed() > self.conf.control_file_save_interval { + let mut write_guard = self.tli.write_shared_state().await; + // it should be done in the background because it blocks manager task, but flush() should + // be fast enough not to be a problem now + if let Err(e) = write_guard.sk.state_mut().flush().await { + warn!("failed to save control file: {:?}", e); } - .instrument(info_span!("WAL removal", ttid=%tli.ttid)), - )); + } else { + // we should wait until some time passed until the next save + update_next_event( + next_event, + (state.cfile_last_persist_at + self.conf.control_file_save_interval).into(), + ); + } + } + + /// Spawns WAL removal task if needed. + async fn update_wal_removal(&mut self, state: &StateSnapshot) { + if self.wal_removal_task.is_some() || state.wal_removal_on_hold { + // WAL removal is already in progress or hold off + return; + } + + // If enabled, we use LSN of the most lagging walsender as a WAL removal horizon. + // This allows to get better read speed for pageservers that are lagging behind, + // at the cost of keeping more WAL on disk. + let replication_horizon_lsn = if self.conf.walsenders_keep_horizon { + self.walsenders.laggard_lsn() + } else { + None + }; + + let removal_horizon_lsn = calc_horizon_lsn(state, replication_horizon_lsn); + let removal_horizon_segno = removal_horizon_lsn + .segment_number(self.wal_seg_size) + .saturating_sub(1); + + if removal_horizon_segno > self.last_removed_segno { + // we need to remove WAL + let remover = match self.tli.read_shared_state().await.sk { + StateSK::Loaded(ref sk) => { + crate::wal_storage::Storage::remove_up_to(&sk.wal_store, removal_horizon_segno) + } + StateSK::Offloaded(_) => { + // we can't remove WAL if it's not loaded + warn!("unexpectedly trying to run WAL removal on offloaded timeline"); + return; + } + StateSK::Empty => unreachable!(), + }; + + self.wal_removal_task = Some(tokio::spawn( + async move { + remover.await?; + Ok(removal_horizon_segno) + } + .instrument(info_span!("WAL removal", ttid=%self.tli.ttid)), + )); + } + } + + /// Update the state after WAL removal task finished. + fn update_wal_removal_end(&mut self, res: Result, JoinError>) { + let new_last_removed_segno = match res { + Ok(Ok(segno)) => segno, + Err(e) => { + warn!("WAL removal task failed: {:?}", e); + return; + } + Ok(Err(e)) => { + warn!("WAL removal task failed: {:?}", e); + return; + } + }; + + self.last_removed_segno = new_last_removed_segno; + // update the state in Arc + self.tli + .last_removed_segno + .store(new_last_removed_segno, std::sync::atomic::Ordering::Relaxed); + } + + /// Spawns partial WAL backup task if needed. + async fn update_partial_backup(&mut self, state: &StateSnapshot) { + // check if partial backup is enabled and should be started + if !self.conf.is_wal_backup_enabled() || !self.conf.partial_backup_enabled { + return; + } + + if self.partial_backup_task.is_some() { + // partial backup is already running + return; + } + + if !wal_backup_partial::needs_uploading(state, &self.partial_backup_uploaded) { + // nothing to upload + return; + } + + // Get WalResidentTimeline and start partial backup task. + self.partial_backup_task = Some(tokio::spawn(wal_backup_partial::main_task( + self.wal_resident_timeline(), + self.conf.clone(), + ))); + } + + /// Update the state after partial WAL backup task finished. + fn update_partial_backup_end(&mut self, res: Result, JoinError>) { + match res { + Ok(new_upload_state) => { + self.partial_backup_uploaded = new_upload_state; + } + Err(e) => { + warn!("partial backup task panicked: {:?}", e); + } + } + } + + /// Handle message arrived from ManagerCtl. + async fn handle_message(&mut self, msg: Option) { + debug!("received manager message: {:?}", msg); + match msg { + Some(ManagerCtlMessage::GuardRequest(tx)) => { + if self.is_offloaded { + // trying to unevict timeline, but without gurarantee that it will be successful + self.unevict_timeline().await; + } + + let guard = if self.is_offloaded { + Err(anyhow::anyhow!("timeline is offloaded, can't get a guard")) + } else { + Ok(self.access_service.create_guard()) + }; + + if tx.send(guard).is_err() { + warn!("failed to reply with a guard, receiver dropped"); + } + } + Some(ManagerCtlMessage::GuardDrop(guard_id)) => { + self.access_service.drop_guard(guard_id); + } + None => { + // can't happen, we're holding the sender + unreachable!(); + } + } } } -/// Update the state after WAL removal task finished. -fn update_wal_removal_end( - res: Result, JoinError>, - tli: &Arc, - last_removed_segno: &mut u64, -) { - let new_last_removed_segno = match res { - Ok(Ok(segno)) => segno, - Err(e) => { - warn!("WAL removal task failed: {:?}", e); - return; - } - Ok(Err(e)) => { - warn!("WAL removal task failed: {:?}", e); - return; - } - }; - - *last_removed_segno = new_last_removed_segno; - // update the state in Arc - tli.last_removed_segno - .store(new_last_removed_segno, std::sync::atomic::Ordering::Relaxed); +// utility functions +async fn sleep_until(option: &Option) { + if let Some(timeout) = option { + tokio::time::sleep_until(*timeout).await; + } else { + futures::future::pending::<()>().await; + } +} + +async fn await_task_finish(option: &mut Option>) -> Result { + if let Some(task) = option { + task.await + } else { + futures::future::pending().await + } +} + +/// Update next_event if candidate is earlier. +fn update_next_event(next_event: &mut Option, candidate: Instant) { + if let Some(next) = next_event { + if candidate < *next { + *next = candidate; + } + } else { + *next_event = Some(candidate); + } +} + +#[repr(usize)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +pub enum Status { + NotStarted, + Started, + StateSnapshot, + UpdateBackup, + UpdateControlFile, + UpdateWalRemoval, + UpdatePartialBackup, + EvictTimeline, + Wait, + HandleMessage, + Exiting, + Finished, +} + +/// AtomicStatus is a wrapper around AtomicUsize adapted for the Status enum. +pub struct AtomicStatus { + inner: AtomicUsize, +} + +impl Default for AtomicStatus { + fn default() -> Self { + Self::new() + } +} + +impl AtomicStatus { + pub fn new() -> Self { + AtomicStatus { + inner: AtomicUsize::new(Status::NotStarted as usize), + } + } + + pub fn load(&self, order: std::sync::atomic::Ordering) -> Status { + // Safety: This line of code uses `std::mem::transmute` to reinterpret the loaded value as `Status`. + // It is safe to use `transmute` in this context because `Status` is a repr(usize) enum, + // which means it has the same memory layout as usize. + // However, it is important to ensure that the loaded value is a valid variant of `Status`, + // otherwise, the behavior will be undefined. + unsafe { std::mem::transmute(self.inner.load(order)) } + } + + pub fn get(&self) -> Status { + self.load(std::sync::atomic::Ordering::Relaxed) + } + + pub fn store(&self, val: Status, order: std::sync::atomic::Ordering) { + self.inner.store(val as usize, order); + } } diff --git a/safekeeper/src/timelines_set.rs b/safekeeper/src/timelines_set.rs index ea8e23bb72..d6eea79f82 100644 --- a/safekeeper/src/timelines_set.rs +++ b/safekeeper/src/timelines_set.rs @@ -80,6 +80,10 @@ impl TimelineSetGuard { self.timelines_set.set_present(self.tli.clone(), present); true } + + pub fn get(&self) -> bool { + self.is_present + } } impl Drop for TimelineSetGuard { diff --git a/safekeeper/src/wal_backup.rs b/safekeeper/src/wal_backup.rs index 58591aecfa..9ea048a3c7 100644 --- a/safekeeper/src/wal_backup.rs +++ b/safekeeper/src/wal_backup.rs @@ -12,7 +12,6 @@ use std::cmp::min; use std::collections::HashSet; use std::num::NonZeroU32; use std::pin::Pin; -use std::sync::Arc; use std::time::Duration; use postgres_ffi::v14::xlog_utils::XLogSegNoOffsetToRecPtr; @@ -30,8 +29,8 @@ use tracing::*; use utils::{id::TenantTimelineId, lsn::Lsn}; use crate::metrics::{BACKED_UP_SEGMENTS, BACKUP_ERRORS, WAL_BACKUP_TASKS}; -use crate::timeline::{FullAccessTimeline, PeerInfo, Timeline}; -use crate::timeline_manager::StateSnapshot; +use crate::timeline::{PeerInfo, WalResidentTimeline}; +use crate::timeline_manager::{Manager, StateSnapshot}; use crate::{SafeKeeperConf, WAL_BACKUP_RUNTIME}; use once_cell::sync::OnceCell; @@ -48,7 +47,7 @@ pub struct WalBackupTaskHandle { } /// Do we have anything to upload to S3, i.e. should safekeepers run backup activity? -pub fn is_wal_backup_required( +pub(crate) fn is_wal_backup_required( wal_seg_size: usize, num_computes: usize, state: &StateSnapshot, @@ -61,35 +60,33 @@ pub fn is_wal_backup_required( /// Based on peer information determine which safekeeper should offload; if it /// is me, run (per timeline) task, if not yet. OTOH, if it is not me and task /// is running, kill it. -pub async fn update_task( - conf: &SafeKeeperConf, - tli: &Arc, - need_backup: bool, - state: &StateSnapshot, - entry: &mut Option, -) { +pub(crate) async fn update_task(mgr: &mut Manager, need_backup: bool, state: &StateSnapshot) { let (offloader, election_dbg_str) = - determine_offloader(&state.peers, state.backup_lsn, tli.ttid, conf); - let elected_me = Some(conf.my_id) == offloader; + determine_offloader(&state.peers, state.backup_lsn, mgr.tli.ttid, &mgr.conf); + let elected_me = Some(mgr.conf.my_id) == offloader; let should_task_run = need_backup && elected_me; // start or stop the task - if should_task_run != (entry.is_some()) { + if should_task_run != (mgr.backup_task.is_some()) { if should_task_run { info!("elected for backup: {}", election_dbg_str); let (shutdown_tx, shutdown_rx) = mpsc::channel(1); - let async_task = backup_task_main(tli.clone(), conf.backup_parallel_jobs, shutdown_rx); + let async_task = backup_task_main( + mgr.wal_resident_timeline(), + mgr.conf.backup_parallel_jobs, + shutdown_rx, + ); - let handle = if conf.current_thread_runtime { + let handle = if mgr.conf.current_thread_runtime { tokio::spawn(async_task) } else { WAL_BACKUP_RUNTIME.spawn(async_task) }; - *entry = Some(WalBackupTaskHandle { + mgr.backup_task = Some(WalBackupTaskHandle { shutdown_tx, handle, }); @@ -101,7 +98,7 @@ pub async fn update_task( // someone else has been elected info!("stepping down from backup: {}", election_dbg_str); } - shut_down_task(entry).await; + shut_down_task(&mut mgr.backup_task).await; } } } @@ -191,7 +188,7 @@ pub fn init_remote_storage(conf: &SafeKeeperConf) { } struct WalBackupTask { - timeline: FullAccessTimeline, + timeline: WalResidentTimeline, timeline_dir: Utf8PathBuf, wal_seg_size: usize, parallel_jobs: usize, @@ -200,16 +197,12 @@ struct WalBackupTask { /// Offload single timeline. #[instrument(name = "WAL backup", skip_all, fields(ttid = %tli.ttid))] -async fn backup_task_main(tli: Arc, parallel_jobs: usize, mut shutdown_rx: Receiver<()>) { +async fn backup_task_main( + tli: WalResidentTimeline, + parallel_jobs: usize, + mut shutdown_rx: Receiver<()>, +) { let _guard = WAL_BACKUP_TASKS.guard(); - - let tli = match tli.full_access_guard().await { - Ok(tli) => tli, - Err(e) => { - error!("backup error: {}", e); - return; - } - }; info!("started"); let mut wb = WalBackupTask { @@ -304,7 +297,7 @@ impl WalBackupTask { } async fn backup_lsn_range( - timeline: &FullAccessTimeline, + timeline: &WalResidentTimeline, backup_lsn: &mut Lsn, end_lsn: Lsn, wal_seg_size: usize, diff --git a/safekeeper/src/wal_backup_partial.rs b/safekeeper/src/wal_backup_partial.rs index ed5ddb71f5..9c7cd0888d 100644 --- a/safekeeper/src/wal_backup_partial.rs +++ b/safekeeper/src/wal_backup_partial.rs @@ -29,18 +29,22 @@ use utils::lsn::Lsn; use crate::{ metrics::{PARTIAL_BACKUP_UPLOADED_BYTES, PARTIAL_BACKUP_UPLOADS}, safekeeper::Term, - timeline::FullAccessTimeline, + timeline::WalResidentTimeline, + timeline_manager::StateSnapshot, wal_backup::{self, remote_timeline_path}, SafeKeeperConf, }; #[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] pub enum UploadStatus { - /// Upload is in progress + /// Upload is in progress. This status should be used only for garbage collection, + /// don't read data from the remote storage with this status. InProgress, - /// Upload is finished + /// Upload is finished. There is always at most one segment with this status. + /// It means that the segment is actual and can be used. Uploaded, - /// Deletion is in progress + /// Deletion is in progress. This status should be used only for garbage collection, + /// don't read data from the remote storage with this status. Deleting, } @@ -50,6 +54,10 @@ pub struct PartialRemoteSegment { pub name: String, pub commit_lsn: Lsn, pub flush_lsn: Lsn, + // We should use last_log_term here, otherwise it's possible to have inconsistent data in the + // remote storage. + // + // More info here: https://github.com/neondatabase/neon/pull/8022#discussion_r1654738405 pub term: Term, } @@ -60,6 +68,10 @@ impl PartialRemoteSegment { && self.flush_lsn == other.flush_lsn && self.term == other.term } + + pub(crate) fn remote_path(&self, remote_timeline_path: &RemotePath) -> RemotePath { + remote_timeline_path.join(&self.name) + } } // NB: these structures are a part of a control_file, you can't change them without @@ -71,7 +83,7 @@ pub struct State { impl State { /// Find an Uploaded segment. There should be only one Uploaded segment at a time. - fn uploaded_segment(&self) -> Option { + pub(crate) fn uploaded_segment(&self) -> Option { self.segments .iter() .find(|seg| seg.status == UploadStatus::Uploaded) @@ -81,7 +93,7 @@ impl State { struct PartialBackup { wal_seg_size: usize, - tli: FullAccessTimeline, + tli: WalResidentTimeline, conf: SafeKeeperConf, local_prefix: Utf8PathBuf, remote_timeline_path: RemotePath, @@ -128,17 +140,17 @@ impl PartialBackup { let sk_info = self.tli.get_safekeeper_info(&self.conf).await; let flush_lsn = Lsn(sk_info.flush_lsn); let commit_lsn = Lsn(sk_info.commit_lsn); - let term = sk_info.term; + let last_log_term = sk_info.last_log_term; let segno = self.segno(flush_lsn); - let name = self.remote_segment_name(segno, term, commit_lsn, flush_lsn); + let name = self.remote_segment_name(segno, last_log_term, commit_lsn, flush_lsn); PartialRemoteSegment { status: UploadStatus::InProgress, name, commit_lsn, flush_lsn, - term, + term: last_log_term, } } @@ -151,7 +163,7 @@ impl PartialBackup { let backup_bytes = flush_lsn.segment_offset(self.wal_seg_size); let local_path = self.local_prefix.join(self.local_segment_name(segno)); - let remote_path = self.remote_timeline_path.join(&prepared.name); + let remote_path = prepared.remote_path(&self.remote_timeline_path); // Upload first `backup_bytes` bytes of the segment to the remote storage. wal_backup::backup_partial_segment(&local_path, &remote_path, backup_bytes).await?; @@ -161,7 +173,7 @@ impl PartialBackup { // If the term changed, we cannot guarantee the validity of the uploaded data. // If the term is the same, we know the data is not corrupted. let sk_info = self.tli.get_safekeeper_info(&self.conf).await; - if sk_info.term != prepared.term { + if sk_info.last_log_term != prepared.term { anyhow::bail!("term changed during upload"); } assert!(prepared.commit_lsn <= Lsn(sk_info.commit_lsn)); @@ -270,8 +282,32 @@ impl PartialBackup { } } +/// Check if everything is uploaded and partial backup task doesn't need to run. +pub(crate) fn needs_uploading( + state: &StateSnapshot, + uploaded: &Option, +) -> bool { + match uploaded { + Some(uploaded) => { + uploaded.status != UploadStatus::Uploaded + || uploaded.flush_lsn != state.flush_lsn + || uploaded.commit_lsn != state.commit_lsn + || uploaded.term != state.last_log_term + } + None => true, + } +} + +/// Main task for partial backup. It waits for the flush_lsn to change and then uploads the +/// partial segment to the remote storage. It also does garbage collection of old segments. +/// +/// When there is nothing more to do and the last segment was successfully uploaded, the task +/// returns PartialRemoteSegment, to signal readiness for offloading the timeline. #[instrument(name = "Partial backup", skip_all, fields(ttid = %tli.ttid))] -pub async fn main_task(tli: FullAccessTimeline, conf: SafeKeeperConf) { +pub async fn main_task( + tli: WalResidentTimeline, + conf: SafeKeeperConf, +) -> Option { debug!("started"); let await_duration = conf.partial_backup_timeout; @@ -285,7 +321,7 @@ pub async fn main_task(tli: FullAccessTimeline, conf: SafeKeeperConf) { Ok(path) => path, Err(e) => { error!("failed to create remote path: {:?}", e); - return; + return None; } }; @@ -320,19 +356,13 @@ pub async fn main_task(tli: FullAccessTimeline, conf: SafeKeeperConf) { // wait until we have something to upload let uploaded_segment = backup.state.uploaded_segment(); if let Some(seg) = &uploaded_segment { - // if we already uploaded something, wait until we have something new - while flush_lsn_rx.borrow().lsn == seg.flush_lsn + // check if uploaded segment matches the current state + if flush_lsn_rx.borrow().lsn == seg.flush_lsn && *commit_lsn_rx.borrow() == seg.commit_lsn && flush_lsn_rx.borrow().term == seg.term { - tokio::select! { - _ = backup.tli.cancel.cancelled() => { - info!("timeline canceled"); - return; - } - _ = commit_lsn_rx.changed() => {} - _ = flush_lsn_rx.changed() => {} - } + // we have nothing to do, the last segment is already uploaded + return Some(seg.clone()); } } @@ -341,7 +371,7 @@ pub async fn main_task(tli: FullAccessTimeline, conf: SafeKeeperConf) { tokio::select! { _ = backup.tli.cancel.cancelled() => { info!("timeline canceled"); - return; + return None; } _ = flush_lsn_rx.changed() => {} } @@ -358,7 +388,7 @@ pub async fn main_task(tli: FullAccessTimeline, conf: SafeKeeperConf) { tokio::select! { _ = backup.tli.cancel.cancelled() => { info!("timeline canceled"); - return; + return None; } _ = commit_lsn_rx.changed() => {} _ = flush_lsn_rx.changed() => { diff --git a/safekeeper/src/wal_storage.rs b/safekeeper/src/wal_storage.rs index 0c1731937c..74c4693ccd 100644 --- a/safekeeper/src/wal_storage.rs +++ b/safekeeper/src/wal_storage.rs @@ -211,7 +211,7 @@ impl PhysicalStorage { /// Returns `file` and `is_partial`. async fn open_or_create(&mut self, segno: XLogSegNo) -> Result<(File, bool)> { let (wal_file_path, wal_file_partial_path) = - wal_file_paths(&self.timeline_dir, segno, self.wal_seg_size)?; + wal_file_paths(&self.timeline_dir, segno, self.wal_seg_size); // Try to open already completed segment if let Ok(file) = OpenOptions::new().write(true).open(&wal_file_path).await { @@ -231,11 +231,7 @@ impl PhysicalStorage { // half initialized segment, first bake it under tmp filename and // then rename. let tmp_path = self.timeline_dir.join("waltmp"); - #[allow(clippy::suspicious_open_options)] - let mut file = OpenOptions::new() - .create(true) - .write(true) - .open(&tmp_path) + let mut file = File::create(&tmp_path) .await .with_context(|| format!("Failed to open tmp wal file {:?}", &tmp_path))?; @@ -280,7 +276,7 @@ impl PhysicalStorage { // Rename partial file to completed file let (wal_file_path, wal_file_partial_path) = - wal_file_paths(&self.timeline_dir, segno, self.wal_seg_size)?; + wal_file_paths(&self.timeline_dir, segno, self.wal_seg_size); fs::rename(wal_file_partial_path, wal_file_path).await?; } else { // otherwise, file can be reused later @@ -465,7 +461,7 @@ impl Storage for PhysicalStorage { if !is_partial { // Make segment partial once again let (wal_file_path, wal_file_partial_path) = - wal_file_paths(&self.timeline_dir, segno, self.wal_seg_size)?; + wal_file_paths(&self.timeline_dir, segno, self.wal_seg_size); fs::rename(wal_file_path, wal_file_partial_path).await?; } @@ -745,7 +741,7 @@ pub(crate) async fn open_wal_file( segno: XLogSegNo, wal_seg_size: usize, ) -> Result<(tokio::fs::File, bool)> { - let (wal_file_path, wal_file_partial_path) = wal_file_paths(timeline_dir, segno, wal_seg_size)?; + let (wal_file_path, wal_file_partial_path) = wal_file_paths(timeline_dir, segno, wal_seg_size); // First try to open the .partial file. let mut partial_path = wal_file_path.to_owned(); @@ -771,9 +767,9 @@ pub fn wal_file_paths( timeline_dir: &Utf8Path, segno: XLogSegNo, wal_seg_size: usize, -) -> Result<(Utf8PathBuf, Utf8PathBuf)> { +) -> (Utf8PathBuf, Utf8PathBuf) { let wal_file_name = XLogFileName(PG_TLI, segno, wal_seg_size); let wal_file_path = timeline_dir.join(wal_file_name.clone()); let wal_file_partial_path = timeline_dir.join(wal_file_name + ".partial"); - Ok((wal_file_path, wal_file_partial_path)) + (wal_file_path, wal_file_partial_path) } diff --git a/safekeeper/tests/random_test.rs b/safekeeper/tests/random_test.rs index 6c6f6a8c96..7bdee35cd7 100644 --- a/safekeeper/tests/random_test.rs +++ b/safekeeper/tests/random_test.rs @@ -10,7 +10,7 @@ use crate::walproposer_sim::{ pub mod walproposer_sim; // Generates 2000 random seeds and runs a schedule for each of them. -// If you seed this test fail, please report the last seed to the +// If you see this test fail, please report the last seed to the // @safekeeper team. #[test] fn test_random_schedules() -> anyhow::Result<()> { diff --git a/safekeeper/tests/walproposer_sim/safekeeper.rs b/safekeeper/tests/walproposer_sim/safekeeper.rs index 47539872a6..43835c7f44 100644 --- a/safekeeper/tests/walproposer_sim/safekeeper.rs +++ b/safekeeper/tests/walproposer_sim/safekeeper.rs @@ -16,12 +16,12 @@ use desim::{ use hyper::Uri; use safekeeper::{ safekeeper::{ProposerAcceptorMessage, SafeKeeper, ServerInfo, UNKNOWN_SERVER_VERSION}, - state::TimelinePersistentState, + state::{TimelinePersistentState, TimelineState}, timeline::TimelineError, wal_storage::Storage, SafeKeeperConf, }; -use tracing::{debug, info_span}; +use tracing::{debug, info_span, warn}; use utils::{ id::{NodeId, TenantId, TenantTimelineId, TimelineId}, lsn::Lsn, @@ -68,7 +68,7 @@ impl GlobalMap { let control_store = DiskStateStorage::new(disk.clone()); let wal_store = DiskWALStorage::new(disk.clone(), &control_store)?; - let sk = SafeKeeper::new(control_store, wal_store, conf.my_id)?; + let sk = SafeKeeper::new(TimelineState::new(control_store), wal_store, conf.my_id)?; timelines.insert( ttid, SharedState { @@ -118,7 +118,11 @@ impl GlobalMap { let control_store = DiskStateStorage::new(disk_timeline.clone()); let wal_store = DiskWALStorage::new(disk_timeline.clone(), &control_store)?; - let sk = SafeKeeper::new(control_store, wal_store, self.conf.my_id)?; + let sk = SafeKeeper::new( + TimelineState::new(control_store), + wal_store, + self.conf.my_id, + )?; self.timelines.insert( ttid, @@ -180,6 +184,9 @@ pub fn run_server(os: NodeOs, disk: Arc) -> Result<()> { partial_backup_enabled: false, partial_backup_timeout: Duration::from_secs(0), disable_periodic_broker_push: false, + enable_offload: false, + delete_offloaded_wal: false, + control_file_save_interval: Duration::from_secs(1), }; let mut global = GlobalMap::new(disk, conf.clone())?; @@ -247,7 +254,12 @@ pub fn run_server(os: NodeOs, disk: Arc) -> Result<()> { NetEvent::Message(msg) => { let res = conn.process_any(msg, &mut global); if res.is_err() { - debug!("conn {:?} error: {:#}", connection_id, res.unwrap_err()); + let e = res.unwrap_err(); + let estr = e.to_string(); + if !estr.contains("finished processing START_REPLICATION") { + warn!("conn {:?} error: {:?}", connection_id, e); + panic!("unexpected error at safekeeper: {:#}", e); + } conns.remove(&connection_id); break; } diff --git a/storage_controller/src/compute_hook.rs b/storage_controller/src/compute_hook.rs index a1d051f150..4d0f8006aa 100644 --- a/storage_controller/src/compute_hook.rs +++ b/storage_controller/src/compute_hook.rs @@ -146,6 +146,9 @@ pub(crate) enum NotifyError { // A response indicates we will never succeed, such as 400 or 404 #[error("Non-retryable error {0}")] Fatal(StatusCode), + + #[error("neon_local error: {0}")] + NeonLocal(anyhow::Error), } enum MaybeSendResult { @@ -278,7 +281,7 @@ impl ComputeHook { async fn do_notify_local( &self, reconfigure_request: &ComputeHookNotifyRequest, - ) -> anyhow::Result<()> { + ) -> Result<(), NotifyError> { // neon_local updates are not safe to call concurrently, use a lock to serialize // all calls to this function let _locked = self.neon_local_lock.lock().await; @@ -321,7 +324,8 @@ impl ComputeHook { tracing::info!("Reconfiguring endpoint {}", endpoint_name,); endpoint .reconfigure(compute_pageservers.clone(), *stripe_size) - .await?; + .await + .map_err(NotifyError::NeonLocal)?; } } @@ -510,7 +514,7 @@ impl ComputeHook { } else { self.do_notify_local(&request).await.map_err(|e| { // This path is for testing only, so munge the error into our prod-style error type. - tracing::error!("Local notification hook failed: {e}"); + tracing::error!("neon_local notification hook failed: {e}"); NotifyError::Fatal(StatusCode::INTERNAL_SERVER_ERROR) }) }; diff --git a/storage_controller/src/http.rs b/storage_controller/src/http.rs index 3e9951fb9e..680e6f09c4 100644 --- a/storage_controller/src/http.rs +++ b/storage_controller/src/http.rs @@ -502,6 +502,17 @@ async fn handle_node_drain(req: Request) -> Result, ApiErro json_response(StatusCode::ACCEPTED, ()) } +async fn handle_cancel_node_drain(req: Request) -> Result, ApiError> { + check_permissions(&req, Scope::Admin)?; + + let state = get_state(&req); + let node_id: NodeId = parse_request_param(&req, "node_id")?; + + state.service.cancel_node_drain(node_id).await?; + + json_response(StatusCode::ACCEPTED, ()) +} + async fn handle_node_fill(req: Request) -> Result, ApiError> { check_permissions(&req, Scope::Admin)?; @@ -513,6 +524,17 @@ async fn handle_node_fill(req: Request) -> Result, ApiError json_response(StatusCode::ACCEPTED, ()) } +async fn handle_cancel_node_fill(req: Request) -> Result, ApiError> { + check_permissions(&req, Scope::Admin)?; + + let state = get_state(&req); + let node_id: NodeId = parse_request_param(&req, "node_id")?; + + state.service.cancel_node_fill(node_id).await?; + + json_response(StatusCode::ACCEPTED, ()) +} + async fn handle_tenant_shard_split( service: Arc, mut req: Request, @@ -871,9 +893,23 @@ pub fn make_router( .put("/control/v1/node/:node_id/drain", |r| { named_request_span(r, handle_node_drain, RequestName("control_v1_node_drain")) }) + .delete("/control/v1/node/:node_id/drain", |r| { + named_request_span( + r, + handle_cancel_node_drain, + RequestName("control_v1_cancel_node_drain"), + ) + }) .put("/control/v1/node/:node_id/fill", |r| { named_request_span(r, handle_node_fill, RequestName("control_v1_node_fill")) }) + .delete("/control/v1/node/:node_id/fill", |r| { + named_request_span( + r, + handle_cancel_node_fill, + RequestName("control_v1_cancel_node_fill"), + ) + }) // TODO(vlad): endpoint for cancelling drain and fill // Tenant Shard operations .put("/control/v1/tenant/:tenant_shard_id/migrate", |r| { diff --git a/storage_controller/src/id_lock_map.rs b/storage_controller/src/id_lock_map.rs index dff793289f..fcd3eb57e2 100644 --- a/storage_controller/src/id_lock_map.rs +++ b/storage_controller/src/id_lock_map.rs @@ -8,14 +8,15 @@ use crate::service::RECONCILE_TIMEOUT; const LOCK_TIMEOUT_ALERT_THRESHOLD: Duration = RECONCILE_TIMEOUT; -/// A wrapper around `OwnedRwLockWriteGuard` that when dropped changes the -/// current holding operation in lock. -pub struct WrappedWriteGuard { +/// A wrapper around `OwnedRwLockWriteGuard` used for tracking the +/// operation that holds the lock, and print a warning if it exceeds +/// the LOCK_TIMEOUT_ALERT_THRESHOLD time +pub struct TracingExclusiveGuard { guard: tokio::sync::OwnedRwLockWriteGuard>, start: Instant, } -impl WrappedWriteGuard { +impl TracingExclusiveGuard { pub fn new(guard: tokio::sync::OwnedRwLockWriteGuard>) -> Self { Self { guard, @@ -24,12 +25,12 @@ impl WrappedWriteGuard { } } -impl Drop for WrappedWriteGuard { +impl Drop for TracingExclusiveGuard { fn drop(&mut self) { let duration = self.start.elapsed(); if duration > LOCK_TIMEOUT_ALERT_THRESHOLD { tracing::warn!( - "Lock on {} was held for {:?}", + "Exclusive lock by {} was held for {:?}", self.guard.as_ref().unwrap(), duration ); @@ -38,6 +39,38 @@ impl Drop for WrappedWriteGuard { } } +// A wrapper around `OwnedRwLockReadGuard` used for tracking the +/// operation that holds the lock, and print a warning if it exceeds +/// the LOCK_TIMEOUT_ALERT_THRESHOLD time +pub struct TracingSharedGuard { + _guard: tokio::sync::OwnedRwLockReadGuard>, + operation: T, + start: Instant, +} + +impl TracingSharedGuard { + pub fn new(guard: tokio::sync::OwnedRwLockReadGuard>, operation: T) -> Self { + Self { + _guard: guard, + operation, + start: Instant::now(), + } + } +} + +impl Drop for TracingSharedGuard { + fn drop(&mut self) { + let duration = self.start.elapsed(); + if duration > LOCK_TIMEOUT_ALERT_THRESHOLD { + tracing::warn!( + "Shared lock by {} was held for {:?}", + self.operation, + duration + ); + } + } +} + /// A map of locks covering some arbitrary identifiers. Useful if you have a collection of objects but don't /// want to embed a lock in each one, or if your locking granularity is different to your object granularity. /// For example, used in the storage controller where the objects are tenant shards, but sometimes locking @@ -58,21 +91,22 @@ where pub(crate) fn shared( &self, key: T, - ) -> impl std::future::Future>> { + operation: I, + ) -> impl std::future::Future> { let mut locked = self.entities.lock().unwrap(); - let entry = locked.entry(key).or_default(); - entry.clone().read_owned() + let entry = locked.entry(key).or_default().clone(); + async move { TracingSharedGuard::new(entry.read_owned().await, operation) } } pub(crate) fn exclusive( &self, key: T, operation: I, - ) -> impl std::future::Future> { + ) -> impl std::future::Future> { let mut locked = self.entities.lock().unwrap(); let entry = locked.entry(key).or_default().clone(); async move { - let mut guard = WrappedWriteGuard::new(entry.clone().write_owned().await); + let mut guard = TracingExclusiveGuard::new(entry.write_owned().await); *guard.guard = Some(operation); guard } @@ -99,12 +133,12 @@ where pub async fn trace_exclusive_lock< T: Clone + Display + Eq + PartialEq + std::hash::Hash, - I: Display + Clone, + I: Clone + Display, >( op_locks: &IdLockMap, key: T, operation: I, -) -> WrappedWriteGuard { +) -> TracingExclusiveGuard { let start = Instant::now(); let guard = op_locks.exclusive(key.clone(), operation.clone()).await; @@ -123,14 +157,14 @@ pub async fn trace_exclusive_lock< pub async fn trace_shared_lock< T: Clone + Display + Eq + PartialEq + std::hash::Hash, - I: Display, + I: Clone + Display, >( op_locks: &IdLockMap, key: T, operation: I, -) -> tokio::sync::OwnedRwLockReadGuard> { +) -> TracingSharedGuard { let start = Instant::now(); - let guard = op_locks.shared(key.clone()).await; + let guard = op_locks.shared(key.clone(), operation.clone()).await; let duration = start.elapsed(); if duration > LOCK_TIMEOUT_ALERT_THRESHOLD { @@ -159,11 +193,11 @@ mod tests { async fn multiple_shared_locks() { let id_lock_map: IdLockMap = IdLockMap::default(); - let shared_lock_1 = id_lock_map.shared(1).await; - let shared_lock_2 = id_lock_map.shared(1).await; + let shared_lock_1 = id_lock_map.shared(1, Operations::Op1).await; + let shared_lock_2 = id_lock_map.shared(1, Operations::Op2).await; - assert!(shared_lock_1.is_none()); - assert!(shared_lock_2.is_none()); + assert_eq!(shared_lock_1.operation, Operations::Op1); + assert_eq!(shared_lock_2.operation, Operations::Op2); } #[tokio::test] @@ -183,7 +217,7 @@ mod tests { assert!(_ex_lock_2.is_err()); } - let shared_lock_1 = id_lock_map.shared(resource_id).await; - assert!(shared_lock_1.is_none()); + let shared_lock_1 = id_lock_map.shared(resource_id, Operations::Op1).await; + assert_eq!(shared_lock_1.operation, Operations::Op1); } } diff --git a/storage_controller/src/scheduler.rs b/storage_controller/src/scheduler.rs index 0bd2eeac35..843159010d 100644 --- a/storage_controller/src/scheduler.rs +++ b/storage_controller/src/scheduler.rs @@ -391,7 +391,7 @@ impl Scheduler { return Err(ScheduleError::NoPageservers); } - let mut scores: Vec<(NodeId, AffinityScore, usize)> = self + let mut scores: Vec<(NodeId, AffinityScore, usize, usize)> = self .nodes .iter() .filter_map(|(k, v)| { @@ -402,6 +402,7 @@ impl Scheduler { *k, context.nodes.get(k).copied().unwrap_or(AffinityScore::FREE), v.shard_count, + v.attached_shard_count, )) } }) @@ -409,9 +410,12 @@ impl Scheduler { // Sort by, in order of precedence: // 1st: Affinity score. We should never pick a higher-score node if a lower-score node is available - // 2nd: Utilization. Within nodes with the same affinity, use the least loaded nodes. - // 3rd: Node ID. This is a convenience to make selection deterministic in tests and empty systems. - scores.sort_by_key(|i| (i.1, i.2, i.0)); + // 2nd: Attached shard count. Within nodes with the same affinity, we always pick the node with + // the least number of attached shards. + // 3rd: Total shard count. Within nodes with the same affinity and attached shard count, use nodes + // with the lower total shard count. + // 4th: Node ID. This is a convenience to make selection deterministic in tests and empty systems. + scores.sort_by_key(|i| (i.1, i.3, i.2, i.0)); if scores.is_empty() { // After applying constraints, no pageservers were left. diff --git a/storage_controller/src/service.rs b/storage_controller/src/service.rs index 8475bf46d2..e329f42dd6 100644 --- a/storage_controller/src/service.rs +++ b/storage_controller/src/service.rs @@ -13,7 +13,7 @@ use crate::{ Drain, Fill, Operation, OperationError, OperationHandler, MAX_RECONCILES_PER_OPERATION, }, compute_hook::NotifyError, - id_lock_map::{trace_exclusive_lock, trace_shared_lock, IdLockMap, WrappedWriteGuard}, + id_lock_map::{trace_exclusive_lock, trace_shared_lock, IdLockMap, TracingExclusiveGuard}, persistence::{AbortShardSplitStatus, TenantFilter}, reconciler::{ReconcileError, ReconcileUnits}, scheduler::{MaySchedule, ScheduleContext, ScheduleMode}, @@ -359,7 +359,7 @@ struct TenantShardSplitAbort { new_shard_count: ShardCount, new_stripe_size: Option, /// Until this abort op is complete, no other operations may be done on the tenant - _tenant_lock: WrappedWriteGuard, + _tenant_lock: TracingExclusiveGuard, } #[derive(thiserror::Error, Debug)] @@ -1231,6 +1231,13 @@ impl Service { &self, attach_req: AttachHookRequest, ) -> anyhow::Result { + let _tenant_lock = trace_exclusive_lock( + &self.tenant_op_locks, + attach_req.tenant_shard_id.tenant_id, + TenantOperations::ShardSplit, + ) + .await; + // This is a test hook. To enable using it on tenants that were created directly with // the pageserver API (not via this service), we will auto-create any missing tenant // shards with default state. @@ -1429,7 +1436,7 @@ impl Service { async fn node_activate_reconcile( &self, mut node: Node, - _lock: &WrappedWriteGuard, + _lock: &TracingExclusiveGuard, ) -> Result<(), ApiError> { // This Node is a mutable local copy: we will set it active so that we can use its // API client to reconcile with the node. The Node in [`Self::nodes`] will get updated @@ -2658,6 +2665,7 @@ impl Service { TenantOperations::TimelineCreate, ) .await; + failpoint_support::sleep_millis_async!("tenant-create-timeline-shared-lock"); self.ensure_attached_wait(tenant_id).await?; @@ -4540,7 +4548,8 @@ impl Service { self.node_configure(node_id, None, Some(NodeSchedulingPolicy::Draining)) .await?; - let cancel = CancellationToken::new(); + let cancel = self.cancel.child_token(); + let gate_guard = self.gate.enter().map_err(|_| ApiError::ShuttingDown)?; self.inner.write().unwrap().ongoing_operation = Some(OperationHandler { operation: Operation::Drain(Drain { node_id }), @@ -4551,6 +4560,8 @@ impl Service { let service = self.clone(); let cancel = cancel.clone(); async move { + let _gate_guard = gate_guard; + scopeguard::defer! { let prev = service.inner.write().unwrap().ongoing_operation.take(); @@ -4592,6 +4603,44 @@ impl Service { Ok(()) } + pub(crate) async fn cancel_node_drain(&self, node_id: NodeId) -> Result<(), ApiError> { + let (node_available, node_policy) = { + let locked = self.inner.read().unwrap(); + let nodes = &locked.nodes; + let node = nodes.get(&node_id).ok_or(ApiError::NotFound( + anyhow::anyhow!("Node {} not registered", node_id).into(), + ))?; + + (node.is_available(), node.get_scheduling()) + }; + + if !node_available { + return Err(ApiError::ResourceUnavailable( + format!("Node {node_id} is currently unavailable").into(), + )); + } + + if !matches!(node_policy, NodeSchedulingPolicy::Draining) { + return Err(ApiError::PreconditionFailed( + format!("Node {node_id} has no drain in progress").into(), + )); + } + + if let Some(op_handler) = self.inner.read().unwrap().ongoing_operation.as_ref() { + if let Operation::Drain(drain) = op_handler.operation { + if drain.node_id == node_id { + tracing::info!("Cancelling background drain operation for node {node_id}"); + op_handler.cancel.cancel(); + return Ok(()); + } + } + } + + Err(ApiError::PreconditionFailed( + format!("Node {node_id} has no drain in progress").into(), + )) + } + pub(crate) async fn start_node_fill(self: &Arc, node_id: NodeId) -> Result<(), ApiError> { let (ongoing_op, node_available, node_policy, total_nodes_count) = { let locked = self.inner.read().unwrap(); @@ -4634,7 +4683,8 @@ impl Service { self.node_configure(node_id, None, Some(NodeSchedulingPolicy::Filling)) .await?; - let cancel = CancellationToken::new(); + let cancel = self.cancel.child_token(); + let gate_guard = self.gate.enter().map_err(|_| ApiError::ShuttingDown)?; self.inner.write().unwrap().ongoing_operation = Some(OperationHandler { operation: Operation::Fill(Fill { node_id }), @@ -4645,6 +4695,8 @@ impl Service { let service = self.clone(); let cancel = cancel.clone(); async move { + let _gate_guard = gate_guard; + scopeguard::defer! { let prev = service.inner.write().unwrap().ongoing_operation.take(); @@ -4686,6 +4738,44 @@ impl Service { Ok(()) } + pub(crate) async fn cancel_node_fill(&self, node_id: NodeId) -> Result<(), ApiError> { + let (node_available, node_policy) = { + let locked = self.inner.read().unwrap(); + let nodes = &locked.nodes; + let node = nodes.get(&node_id).ok_or(ApiError::NotFound( + anyhow::anyhow!("Node {} not registered", node_id).into(), + ))?; + + (node.is_available(), node.get_scheduling()) + }; + + if !node_available { + return Err(ApiError::ResourceUnavailable( + format!("Node {node_id} is currently unavailable").into(), + )); + } + + if !matches!(node_policy, NodeSchedulingPolicy::Filling) { + return Err(ApiError::PreconditionFailed( + format!("Node {node_id} has no fill in progress").into(), + )); + } + + if let Some(op_handler) = self.inner.read().unwrap().ongoing_operation.as_ref() { + if let Operation::Fill(fill) = op_handler.operation { + if fill.node_id == node_id { + tracing::info!("Cancelling background drain operation for node {node_id}"); + op_handler.cancel.cancel(); + return Ok(()); + } + } + } + + Err(ApiError::PreconditionFailed( + format!("Node {node_id} has no fill in progress").into(), + )) + } + /// Helper for methods that will try and call pageserver APIs for /// a tenant, such as timeline CRUD: they cannot proceed unless the tenant /// is attached somewhere. @@ -5281,11 +5371,24 @@ impl Service { let mut last_inspected_shard: Option = None; let mut inspected_all_shards = false; let mut waiters = Vec::new(); - let mut schedule_context = ScheduleContext::default(); while !inspected_all_shards { if cancel.is_cancelled() { - return Err(OperationError::Cancelled); + match self + .node_configure(node_id, None, Some(NodeSchedulingPolicy::Active)) + .await + { + Ok(()) => return Err(OperationError::Cancelled), + Err(err) => { + return Err(OperationError::FinalizeError( + format!( + "Failed to finalise drain cancel of {} by setting scheduling policy to Active: {}", + node_id, err + ) + .into(), + )); + } + } } { @@ -5322,28 +5425,32 @@ impl Service { } }; - if tenant_shard.intent.demote_attached(scheduler, node_id) { - match tenant_shard.schedule(scheduler, &mut schedule_context) { - Err(e) => { - tracing::warn!( - tenant_id=%tid.tenant_id, shard_id=%tid.shard_slug(), - "Scheduling error when draining pageserver {} : {e}", node_id - ); - } - Ok(()) => { - let scheduled_to = tenant_shard.intent.get_attached(); - tracing::info!( - tenant_id=%tid.tenant_id, shard_id=%tid.shard_slug(), - "Rescheduled shard while draining node {}: {} -> {:?}", - node_id, - node_id, - scheduled_to - ); + // If the shard is not attached to the node being drained, skip it. + if *tenant_shard.intent.get_attached() != Some(node_id) { + last_inspected_shard = Some(*tid); + continue; + } - let waiter = self.maybe_reconcile_shard(tenant_shard, nodes); - if let Some(some) = waiter { - waiters.push(some); - } + match tenant_shard.reschedule_to_secondary(None, scheduler) { + Err(e) => { + tracing::warn!( + tenant_id=%tid.tenant_id, shard_id=%tid.shard_slug(), + "Scheduling error when draining pageserver {} : {e}", node_id + ); + } + Ok(()) => { + let scheduled_to = tenant_shard.intent.get_attached(); + tracing::info!( + tenant_id=%tid.tenant_id, shard_id=%tid.shard_slug(), + "Rescheduled shard while draining node {}: {} -> {:?}", + node_id, + node_id, + scheduled_to + ); + + let waiter = self.maybe_reconcile_shard(tenant_shard, nodes); + if let Some(some) = waiter { + waiters.push(some); } } } @@ -5355,9 +5462,29 @@ impl Service { waiters = self .await_waiters_remainder(waiters, SHORT_RECONCILE_TIMEOUT) .await; + + failpoint_support::sleep_millis_async!("sleepy-drain-loop"); } while !waiters.is_empty() { + if cancel.is_cancelled() { + match self + .node_configure(node_id, None, Some(NodeSchedulingPolicy::Active)) + .await + { + Ok(()) => return Err(OperationError::Cancelled), + Err(err) => { + return Err(OperationError::FinalizeError( + format!( + "Failed to finalise drain cancel of {} by setting scheduling policy to Active: {}", + node_id, err + ) + .into(), + )); + } + } + } + tracing::info!("Awaiting {} pending drain reconciliations", waiters.len()); waiters = self @@ -5394,6 +5521,9 @@ impl Service { /// throughout the cluster. We achieve this by picking tenant shards from each node, /// starting from the ones with the largest number of attached shards, until the node /// reaches the expected cluster average. + /// 3. Avoid promoting more shards of the same tenant than required. The upper bound + /// for the number of tenants from the same shard promoted to the node being filled is: + /// shard count for the tenant divided by the number of nodes in the cluster. fn fill_node_plan(&self, node_id: NodeId) -> Vec { let mut locked = self.inner.write().unwrap(); let fill_requirement = locked.scheduler.compute_fill_requirement(node_id); @@ -5415,8 +5545,18 @@ impl Service { let expected_attached = locked.scheduler.expected_attached_shard_count(); let nodes_by_load = locked.scheduler.nodes_by_attached_shard_count(); + let mut promoted_per_tenant: HashMap = HashMap::new(); let mut plan = Vec::new(); + for (node_id, attached) in nodes_by_load { + let available = locked + .nodes + .get(&node_id) + .map_or(false, |n| n.is_available()); + if !available { + continue; + } + if plan.len() >= fill_requirement || tids_by_node.is_empty() || attached <= expected_attached @@ -5424,13 +5564,22 @@ impl Service { break; } - let can_take = attached - expected_attached; + let mut can_take = attached - expected_attached; let mut remove_node = false; - for _ in 0..can_take { + while can_take > 0 { match tids_by_node.get_mut(&node_id) { Some(tids) => match tids.pop() { Some(tid) => { - plan.push(tid); + let max_promote_for_tenant = std::cmp::max( + tid.shard_count.count() as usize / locked.nodes.len(), + 1, + ); + let promoted = promoted_per_tenant.entry(tid.tenant_id).or_default(); + if *promoted < max_promote_for_tenant { + plan.push(tid); + *promoted += 1; + can_take -= 1; + } } None => { remove_node = true; @@ -5464,15 +5613,27 @@ impl Service { // secondaries are warm. This is not always true (e.g. we just migrated the // tenant). Take that into consideration by checking the secondary status. let mut tids_to_promote = self.fill_node_plan(node_id); - let mut waiters = Vec::new(); - let mut schedule_context = ScheduleContext::default(); // Execute the plan we've composed above. Before aplying each move from the plan, // we validate to ensure that it has not gone stale in the meantime. while !tids_to_promote.is_empty() { if cancel.is_cancelled() { - return Err(OperationError::Cancelled); + match self + .node_configure(node_id, None, Some(NodeSchedulingPolicy::Active)) + .await + { + Ok(()) => return Err(OperationError::Cancelled), + Err(err) => { + return Err(OperationError::FinalizeError( + format!( + "Failed to finalise drain cancel of {} by setting scheduling policy to Active: {}", + node_id, err + ) + .into(), + )); + } + } } { @@ -5502,9 +5663,7 @@ impl Service { } let previously_attached_to = *tenant_shard.intent.get_attached(); - - tenant_shard.intent.promote_attached(scheduler, node_id); - match tenant_shard.schedule(scheduler, &mut schedule_context) { + match tenant_shard.reschedule_to_secondary(Some(node_id), scheduler) { Err(e) => { tracing::warn!( tenant_id=%tid.tenant_id, shard_id=%tid.shard_slug(), @@ -5540,6 +5699,24 @@ impl Service { } while !waiters.is_empty() { + if cancel.is_cancelled() { + match self + .node_configure(node_id, None, Some(NodeSchedulingPolicy::Active)) + .await + { + Ok(()) => return Err(OperationError::Cancelled), + Err(err) => { + return Err(OperationError::FinalizeError( + format!( + "Failed to finalise drain cancel of {} by setting scheduling policy to Active: {}", + node_id, err + ) + .into(), + )); + } + } + } + tracing::info!("Awaiting {} pending fill reconciliations", waiters.len()); waiters = self diff --git a/storage_controller/src/tenant_shard.rs b/storage_controller/src/tenant_shard.rs index d1b632755f..45295bc59b 100644 --- a/storage_controller/src/tenant_shard.rs +++ b/storage_controller/src/tenant_shard.rs @@ -646,6 +646,48 @@ impl TenantShard { Ok(()) } + /// Reschedule this tenant shard to one of its secondary locations. Returns a scheduling error + /// if the swap is not possible and leaves the intent state in its original state. + /// + /// Arguments: + /// `attached_to`: the currently attached location matching the intent state (may be None if the + /// shard is not attached) + /// `promote_to`: an optional secondary location of this tenant shard. If set to None, we ask + /// the scheduler to recommend a node + pub(crate) fn reschedule_to_secondary( + &mut self, + promote_to: Option, + scheduler: &mut Scheduler, + ) -> Result<(), ScheduleError> { + let promote_to = match promote_to { + Some(node) => node, + None => match scheduler.node_preferred(self.intent.get_secondary()) { + Some(node) => node, + None => { + return Err(ScheduleError::ImpossibleConstraint); + } + }, + }; + + assert!(self.intent.get_secondary().contains(&promote_to)); + + if let Some(node) = self.intent.get_attached() { + let demoted = self.intent.demote_attached(scheduler, *node); + if !demoted { + return Err(ScheduleError::ImpossibleConstraint); + } + } + + self.intent.promote_attached(scheduler, promote_to); + + // Increment the sequence number for the edge case where a + // reconciler is already running to avoid waiting on the + // current reconcile instead of spawning a new one. + self.sequence = self.sequence.next(); + + Ok(()) + } + /// Optimize attachments: if a shard has a secondary location that is preferable to /// its primary location based on soft constraints, switch that secondary location /// to be attached. @@ -1632,14 +1674,10 @@ pub(crate) mod tests { // We should see equal number of locations on the two nodes. assert_eq!(scheduler.get_node_shard_count(NodeId(1)), 4); - // Scheduling does not consider the number of attachments picking the initial - // pageserver to attach to (hence the assertion that all primaries are on the - // same node) - // TODO: Tweak the scheduling to evenly distribute attachments for new shards. - assert_eq!(scheduler.get_node_attached_shard_count(NodeId(1)), 4); + assert_eq!(scheduler.get_node_attached_shard_count(NodeId(1)), 2); assert_eq!(scheduler.get_node_shard_count(NodeId(2)), 4); - assert_eq!(scheduler.get_node_attached_shard_count(NodeId(2)), 0); + assert_eq!(scheduler.get_node_attached_shard_count(NodeId(2)), 2); // Add another two nodes: we should see the shards spread out when their optimize // methods are called diff --git a/test_runner/fixtures/metrics.py b/test_runner/fixtures/metrics.py index 8b8075f8c1..41fa8e679f 100644 --- a/test_runner/fixtures/metrics.py +++ b/test_runner/fixtures/metrics.py @@ -118,8 +118,6 @@ PAGESERVER_GLOBAL_METRICS: Tuple[str, ...] = ( "libmetrics_launch_timestamp", "libmetrics_build_info", "libmetrics_tracing_event_count_total", - "pageserver_materialized_cache_hits_total", - "pageserver_materialized_cache_hits_direct_total", "pageserver_page_cache_read_hits_total", "pageserver_page_cache_read_accesses_total", "pageserver_page_cache_size_current_bytes", @@ -151,6 +149,7 @@ PAGESERVER_PER_TENANT_METRICS: Tuple[str, ...] = ( "pageserver_evictions_total", "pageserver_evictions_with_low_residence_duration_total", "pageserver_aux_file_estimated_size", + "pageserver_valid_lsn_lease_count", *PAGESERVER_PER_TENANT_REMOTE_TIMELINE_CLIENT_METRICS, # "pageserver_directory_entries_count", -- only used if above a certain threshold # "pageserver_broken_tenants_count" -- used only for broken diff --git a/test_runner/fixtures/neon_fixtures.py b/test_runner/fixtures/neon_fixtures.py index b5d9a69d55..6bfe1afd1f 100644 --- a/test_runner/fixtures/neon_fixtures.py +++ b/test_runner/fixtures/neon_fixtures.py @@ -581,7 +581,7 @@ class NeonEnvBuilder: timeline_id=env.initial_timeline, shard_count=initial_tenant_shard_count, shard_stripe_size=initial_tenant_shard_stripe_size, - aux_file_v2=self.pageserver_aux_file_policy, + aux_file_policy=self.pageserver_aux_file_policy, ) assert env.initial_tenant == initial_tenant assert env.initial_timeline == initial_timeline @@ -1177,10 +1177,10 @@ class NeonEnv: force=config.config_init_force, ) - def start(self): + def start(self, timeout_in_seconds: Optional[int] = None): # Storage controller starts first, so that pageserver /re-attach calls don't # bounce through retries on startup - self.storage_controller.start() + self.storage_controller.start(timeout_in_seconds=timeout_in_seconds) # Wait for storage controller readiness to prevent unnecessary post start-up # reconcile. @@ -1196,10 +1196,18 @@ class NeonEnv: ) # The `or None` is for the linter for pageserver in self.pageservers: - futs.append(executor.submit(lambda ps=pageserver: ps.start())) + futs.append( + executor.submit( + lambda ps=pageserver: ps.start(timeout_in_seconds=timeout_in_seconds) + ) + ) for safekeeper in self.safekeepers: - futs.append(executor.submit(lambda sk=safekeeper: sk.start())) + futs.append( + executor.submit( + lambda sk=safekeeper: sk.start(timeout_in_seconds=timeout_in_seconds) + ) + ) for f in futs: f.result() @@ -1596,7 +1604,7 @@ class NeonCli(AbstractNeonCli): shard_stripe_size: Optional[int] = None, placement_policy: Optional[str] = None, set_default: bool = False, - aux_file_v2: Optional[AuxFileStore] = None, + aux_file_policy: Optional[AuxFileStore] = None, ) -> Tuple[TenantId, TimelineId]: """ Creates a new tenant, returns its id and its initial timeline's id. @@ -1621,13 +1629,11 @@ class NeonCli(AbstractNeonCli): ) ) - if aux_file_v2 is AuxFileStore.V2: + if aux_file_policy is AuxFileStore.V2: args.extend(["-c", "switch_aux_file_policy:v2"]) - - if aux_file_v2 is AuxFileStore.V1: + elif aux_file_policy is AuxFileStore.V1: args.extend(["-c", "switch_aux_file_policy:v1"]) - - if aux_file_v2 is AuxFileStore.CrossValidation: + elif aux_file_policy is AuxFileStore.CrossValidation: args.extend(["-c", "switch_aux_file_policy:cross-validation"]) if set_default: @@ -1783,8 +1789,13 @@ class NeonCli(AbstractNeonCli): res.check_returncode() return res - def storage_controller_start(self): + def storage_controller_start( + self, + timeout_in_seconds: Optional[int] = None, + ): cmd = ["storage_controller", "start"] + if timeout_in_seconds is not None: + cmd.append(f"--start-timeout={timeout_in_seconds}s") return self.raw_cli(cmd) def storage_controller_stop(self, immediate: bool): @@ -1797,8 +1808,11 @@ class NeonCli(AbstractNeonCli): self, id: int, extra_env_vars: Optional[Dict[str, str]] = None, + timeout_in_seconds: Optional[int] = None, ) -> "subprocess.CompletedProcess[str]": start_args = ["pageserver", "start", f"--id={id}"] + if timeout_in_seconds is not None: + start_args.append(f"--start-timeout={timeout_in_seconds}s") storage = self.env.pageserver_remote_storage if isinstance(storage, S3Storage): @@ -1816,7 +1830,10 @@ class NeonCli(AbstractNeonCli): return self.raw_cli(cmd) def safekeeper_start( - self, id: int, extra_opts: Optional[List[str]] = None + self, + id: int, + extra_opts: Optional[List[str]] = None, + timeout_in_seconds: Optional[int] = None, ) -> "subprocess.CompletedProcess[str]": s3_env_vars = None if isinstance(self.env.safekeepers_remote_storage, S3Storage): @@ -1826,6 +1843,8 @@ class NeonCli(AbstractNeonCli): extra_opts = [f"-e={opt}" for opt in extra_opts] else: extra_opts = [] + if timeout_in_seconds is not None: + extra_opts.append(f"--start-timeout={timeout_in_seconds}s") return self.raw_cli( ["safekeeper", "start", str(id), *extra_opts], extra_env_vars=s3_env_vars ) @@ -2077,9 +2096,9 @@ class NeonStorageController(MetricsGetter, LogUtils): self.allowed_errors: list[str] = DEFAULT_STORAGE_CONTROLLER_ALLOWED_ERRORS self.logfile = self.workdir / "storage_controller.log" - def start(self): + def start(self, timeout_in_seconds: Optional[int] = None): assert not self.running - self.env.neon_cli.storage_controller_start() + self.env.neon_cli.storage_controller_start(timeout_in_seconds) self.running = True return self @@ -2228,6 +2247,14 @@ class NeonStorageController(MetricsGetter, LogUtils): headers=self.headers(TokenScope.ADMIN), ) + def cancel_node_drain(self, node_id): + log.info(f"cancel_node_drain({node_id})") + self.request( + "DELETE", + f"{self.env.storage_controller_api}/control/v1/node/{node_id}/drain", + headers=self.headers(TokenScope.ADMIN), + ) + def node_fill(self, node_id): log.info(f"node_fill({node_id})") self.request( @@ -2236,6 +2263,14 @@ class NeonStorageController(MetricsGetter, LogUtils): headers=self.headers(TokenScope.ADMIN), ) + def cancel_node_fill(self, node_id): + log.info(f"cancel_node_fill({node_id})") + self.request( + "DELETE", + f"{self.env.storage_controller_api}/control/v1/node/{node_id}/fill", + headers=self.headers(TokenScope.ADMIN), + ) + def node_status(self, node_id): response = self.request( "GET", @@ -2531,6 +2566,7 @@ class NeonPageserver(PgProtocol, LogUtils): def start( self, extra_env_vars: Optional[Dict[str, str]] = None, + timeout_in_seconds: Optional[int] = None, ) -> "NeonPageserver": """ Start the page server. @@ -2539,7 +2575,9 @@ class NeonPageserver(PgProtocol, LogUtils): """ assert self.running is False - self.env.neon_cli.pageserver_start(self.id, extra_env_vars=extra_env_vars) + self.env.neon_cli.pageserver_start( + self.id, extra_env_vars=extra_env_vars, timeout_in_seconds=timeout_in_seconds + ) self.running = True return self @@ -2553,13 +2591,17 @@ class NeonPageserver(PgProtocol, LogUtils): self.running = False return self - def restart(self, immediate: bool = False): + def restart( + self, + immediate: bool = False, + timeout_in_seconds: Optional[int] = None, + ): """ High level wrapper for restart: restarts the process, and waits for tenant state to stabilize. """ self.stop(immediate=immediate) - self.start() + self.start(timeout_in_seconds=timeout_in_seconds) self.quiesce_tenants() def quiesce_tenants(self): @@ -2640,7 +2682,6 @@ class NeonPageserver(PgProtocol, LogUtils): self, tenant_id: TenantId, config: None | Dict[str, Any] = None, - config_null: bool = False, generation: Optional[int] = None, override_storage_controller_generation: bool = False, ): @@ -2658,7 +2699,6 @@ class NeonPageserver(PgProtocol, LogUtils): return client.tenant_attach( tenant_id, config, - config_null, generation=generation, ) @@ -2700,12 +2740,6 @@ class NeonPageserver(PgProtocol, LogUtils): client = self.http_client(auth_token=auth_token) return client.tenant_create(tenant_id, conf, generation=generation) - def tenant_load(self, tenant_id: TenantId): - client = self.http_client() - return client.tenant_load( - tenant_id, generation=self.env.storage_controller.attach_hook_issue(tenant_id, self.id) - ) - def list_layers( self, tenant_id: Union[TenantId, TenantShardId], timeline_id: TimelineId ) -> list[Path]: @@ -3039,9 +3073,16 @@ class PSQL: host: str = "127.0.0.1", port: int = 5432, ): - assert shutil.which(path) + search_path = None + if (d := os.getenv("POSTGRES_DISTRIB_DIR")) is not None and ( + v := os.getenv("DEFAULT_PG_VERSION") + ) is not None: + search_path = Path(d) / f"v{v}" / "bin" - self.path = path + full_path = shutil.which(path, path=search_path) + assert full_path is not None + + self.path = full_path self.database_url = f"postgres://{host}:{port}/main?options=project%3Dgeneric-project-name" async def run(self, query: Optional[str] = None) -> asyncio.subprocess.Process: @@ -3503,7 +3544,6 @@ class Endpoint(PgProtocol, LogUtils): # and make tests more stable. config_lines = ["max_replication_write_lag=15MB"] + config_lines - config_lines = ["neon.primary_is_running=on"] + config_lines self.config(config_lines) return self @@ -3841,9 +3881,13 @@ class Safekeeper(LogUtils): self.running = running self.logfile = Path(self.data_dir) / f"safekeeper-{id}.log" - def start(self, extra_opts: Optional[List[str]] = None) -> "Safekeeper": + def start( + self, extra_opts: Optional[List[str]] = None, timeout_in_seconds: Optional[int] = None + ) -> "Safekeeper": assert self.running is False - self.env.neon_cli.safekeeper_start(self.id, extra_opts=extra_opts) + self.env.neon_cli.safekeeper_start( + self.id, extra_opts=extra_opts, timeout_in_seconds=timeout_in_seconds + ) self.running = True # wait for wal acceptor start by checking its status started_at = time.time() @@ -3870,6 +3914,8 @@ class Safekeeper(LogUtils): def assert_no_errors(self): assert not self.log_contains("manager task finished prematurely") + assert not self.log_contains("error while acquiring WalResidentTimeline guard") + assert not self.log_contains("timeout while acquiring WalResidentTimeline guard") def append_logical_message( self, tenant_id: TenantId, timeline_id: TimelineId, request: Dict[str, Any] @@ -4613,6 +4659,70 @@ def fork_at_current_lsn( return env.neon_cli.create_branch(new_branch_name, ancestor_branch_name, tenant_id, current_lsn) +def import_timeline_from_vanilla_postgres( + test_output_dir: Path, + env: NeonEnv, + pg_bin: PgBin, + tenant_id: TenantId, + timeline_id: TimelineId, + branch_name: str, + vanilla_pg_connstr: str, +): + """ + Create a new timeline, by importing an existing PostgreSQL cluster. + + This works by taking a physical backup of the running PostgreSQL cluster, and importing that. + """ + + # Take backup of the existing PostgreSQL server with pg_basebackup + basebackup_dir = os.path.join(test_output_dir, "basebackup") + base_tar = os.path.join(basebackup_dir, "base.tar") + wal_tar = os.path.join(basebackup_dir, "pg_wal.tar") + os.mkdir(basebackup_dir) + pg_bin.run( + [ + "pg_basebackup", + "-F", + "tar", + "-d", + vanilla_pg_connstr, + "-D", + basebackup_dir, + ] + ) + + # Extract start_lsn and end_lsn form the backup manifest file + with open(os.path.join(basebackup_dir, "backup_manifest")) as f: + manifest = json.load(f) + start_lsn = manifest["WAL-Ranges"][0]["Start-LSN"] + end_lsn = manifest["WAL-Ranges"][0]["End-LSN"] + + # Import the backup tarballs into the pageserver + env.neon_cli.raw_cli( + [ + "timeline", + "import", + "--tenant-id", + str(tenant_id), + "--timeline-id", + str(timeline_id), + "--branch-name", + branch_name, + "--base-lsn", + start_lsn, + "--base-tarfile", + base_tar, + "--end-lsn", + end_lsn, + "--wal-tarfile", + wal_tar, + "--pg-version", + env.pg_version, + ] + ) + wait_for_last_record_lsn(env.pageserver.http_client(), tenant_id, timeline_id, Lsn(end_lsn)) + + def last_flush_lsn_upload( env: NeonEnv, endpoint: Endpoint, diff --git a/test_runner/fixtures/pageserver/allowed_errors.py b/test_runner/fixtures/pageserver/allowed_errors.py index 147d5705d3..c5b09e3608 100755 --- a/test_runner/fixtures/pageserver/allowed_errors.py +++ b/test_runner/fixtures/pageserver/allowed_errors.py @@ -106,6 +106,11 @@ DEFAULT_STORAGE_CONTROLLER_ALLOWED_ERRORS = [ ".*startup_reconcile: Could not scan node.*", # Tests run in dev mode ".*Starting in dev mode.*", + # Tests that stop endpoints & use the storage controller's neon_local notification + # mechanism might fail (neon_local's stopping and endpoint isn't atomic wrt the storage + # controller's attempts to notify the endpoint). + ".*reconciler.*neon_local notification hook failed.*", + ".*reconciler.*neon_local error.*", ] diff --git a/test_runner/fixtures/pageserver/http.py b/test_runner/fixtures/pageserver/http.py index d5441bd694..7949612714 100644 --- a/test_runner/fixtures/pageserver/http.py +++ b/test_runner/fixtures/pageserver/http.py @@ -1,6 +1,5 @@ from __future__ import annotations -import json import time from collections import defaultdict from dataclasses import dataclass @@ -253,39 +252,30 @@ class PageserverHttpClient(requests.Session, MetricsGetter): self, tenant_id: Union[TenantId, TenantShardId], config: None | Dict[str, Any] = None, - config_null: bool = False, generation: Optional[int] = None, ): - if config_null: - assert config is None - body: Any = None - else: - # null-config is prohibited by the API - config = config or {} - body = {"config": config} - if generation is not None: - body.update({"generation": generation}) + config = config or {} - res = self.post( - f"http://localhost:{self.port}/v1/tenant/{tenant_id}/attach", - data=json.dumps(body), - headers={"Content-Type": "application/json"}, + return self.tenant_location_conf( + tenant_id, + location_conf={ + "mode": "AttachedSingle", + "secondary_conf": None, + "tenant_conf": config, + "generation": generation, + }, ) - self.verbose_error(res) - def tenant_detach(self, tenant_id: TenantId, detach_ignored=False, timeout_secs=None): - params = {} - if detach_ignored: - params["detach_ignored"] = "true" - - kwargs = {} - if timeout_secs is not None: - kwargs["timeout"] = timeout_secs - - res = self.post( - f"http://localhost:{self.port}/v1/tenant/{tenant_id}/detach", params=params, **kwargs + def tenant_detach(self, tenant_id: TenantId): + return self.tenant_location_conf( + tenant_id, + location_conf={ + "mode": "Detached", + "secondary_conf": None, + "tenant_conf": {}, + "generation": None, + }, ) - self.verbose_error(res) def tenant_reset(self, tenant_id: Union[TenantId, TenantShardId], drop_cache: bool): params = {} @@ -340,17 +330,6 @@ class PageserverHttpClient(requests.Session, MetricsGetter): self.verbose_error(res) return res - def tenant_load(self, tenant_id: TenantId, generation=None): - body = None - if generation is not None: - body = {"generation": generation} - res = self.post(f"http://localhost:{self.port}/v1/tenant/{tenant_id}/load", json=body) - self.verbose_error(res) - - def tenant_ignore(self, tenant_id: TenantId): - res = self.post(f"http://localhost:{self.port}/v1/tenant/{tenant_id}/ignore") - self.verbose_error(res) - def tenant_status( self, tenant_id: Union[TenantId, TenantShardId], activate: bool = False ) -> Dict[Any, Any]: @@ -594,6 +573,7 @@ class PageserverHttpClient(requests.Session, MetricsGetter): force_repartition=False, force_image_layer_creation=False, wait_until_uploaded=False, + enhanced_gc_bottom_most_compaction=False, ): self.is_testing_enabled_or_skip() query = {} @@ -603,6 +583,8 @@ class PageserverHttpClient(requests.Session, MetricsGetter): query["force_image_layer_creation"] = "true" if wait_until_uploaded: query["wait_until_uploaded"] = "true" + if enhanced_gc_bottom_most_compaction: + query["enhanced_gc_bottom_most_compaction"] = "true" log.info(f"Requesting compact: tenant {tenant_id}, timeline {timeline_id}") res = self.put( @@ -630,13 +612,15 @@ class PageserverHttpClient(requests.Session, MetricsGetter): tenant_id: Union[TenantId, TenantShardId], timeline_id: TimelineId, timestamp: datetime, + with_lease: bool = False, **kwargs, ): log.info( - f"Requesting lsn by timestamp {timestamp}, tenant {tenant_id}, timeline {timeline_id}" + f"Requesting lsn by timestamp {timestamp}, tenant {tenant_id}, timeline {timeline_id}, {with_lease=}" ) + with_lease_query = f"{with_lease=}".lower() res = self.get( - f"http://localhost:{self.port}/v1/tenant/{tenant_id}/timeline/{timeline_id}/get_lsn_by_timestamp?timestamp={timestamp.isoformat()}Z", + f"http://localhost:{self.port}/v1/tenant/{tenant_id}/timeline/{timeline_id}/get_lsn_by_timestamp?timestamp={timestamp.isoformat()}Z&{with_lease_query}", **kwargs, ) self.verbose_error(res) diff --git a/test_runner/fixtures/pageserver/utils.py b/test_runner/fixtures/pageserver/utils.py index 72384c138b..60535b7592 100644 --- a/test_runner/fixtures/pageserver/utils.py +++ b/test_runner/fixtures/pageserver/utils.py @@ -430,52 +430,6 @@ def enable_remote_storage_versioning( return response -def wait_tenant_status_404( - pageserver_http: PageserverHttpClient, - tenant_id: TenantId, - iterations: int, - interval: float = 0.250, -): - def tenant_is_missing(): - data = {} - try: - data = pageserver_http.tenant_status(tenant_id) - log.info(f"tenant status {data}") - except PageserverApiException as e: - log.debug(e) - if e.status_code == 404: - return - - raise RuntimeError(f"Timeline exists state {data.get('state')}") - - wait_until(iterations, interval=interval, func=tenant_is_missing) - - -def tenant_delete_wait_completed( - pageserver_http: PageserverHttpClient, - tenant_id: TenantId, - iterations: int, - ignore_errors: bool = False, -): - if not ignore_errors: - pageserver_http.tenant_delete(tenant_id=tenant_id) - else: - interval = 0.5 - - def delete_request_sent(): - try: - pageserver_http.tenant_delete(tenant_id=tenant_id) - except PageserverApiException as e: - log.debug(e) - if e.status_code == 404: - return - except Exception as e: - log.debug(e) - - wait_until(iterations, interval=interval, func=delete_request_sent) - wait_tenant_status_404(pageserver_http, tenant_id=tenant_id, iterations=iterations) - - MANY_SMALL_LAYERS_TENANT_CONFIG = { "gc_period": "0s", "compaction_period": "0s", diff --git a/test_runner/performance/pageserver/pagebench/test_pageserver_max_throughput_getpage_at_latest_lsn.py b/test_runner/performance/pageserver/pagebench/test_pageserver_max_throughput_getpage_at_latest_lsn.py index 68f3d9dcbe..1d579214b0 100644 --- a/test_runner/performance/pageserver/pagebench/test_pageserver_max_throughput_getpage_at_latest_lsn.py +++ b/test_runner/performance/pageserver/pagebench/test_pageserver_max_throughput_getpage_at_latest_lsn.py @@ -85,6 +85,8 @@ def test_pageserver_max_throughput_getpage_at_latest_lsn( f"max_throughput_latest_lsn-{n_tenants}-{pgbench_scale}", n_tenants, setup_wrapper, + # https://github.com/neondatabase/neon/issues/8070 + timeout_in_seconds=60, ) env.pageserver.allowed_errors.append( diff --git a/test_runner/performance/pageserver/util.py b/test_runner/performance/pageserver/util.py index f31cd9a9f8..92e05663ce 100644 --- a/test_runner/performance/pageserver/util.py +++ b/test_runner/performance/pageserver/util.py @@ -2,7 +2,7 @@ Utilities used by all code in this sub-directory """ -from typing import Any, Callable, Dict, Tuple +from typing import Any, Callable, Dict, Optional, Tuple import fixtures.pageserver.many_tenants as many_tenants from fixtures.common_types import TenantId, TimelineId @@ -41,6 +41,7 @@ def setup_pageserver_with_tenants( name: str, n_tenants: int, setup: Callable[[NeonEnv], Tuple[TenantId, TimelineId, Dict[str, Any]]], + timeout_in_seconds: Optional[int] = None, ) -> NeonEnv: """ Utility function to set up a pageserver with a given number of identical tenants. @@ -50,6 +51,6 @@ def setup_pageserver_with_tenants( return many_tenants.single_timeline(neon_env_builder, setup, n_tenants) env = neon_env_builder.build_and_use_snapshot(name, doit) - env.start() + env.start(timeout_in_seconds=timeout_in_seconds) ensure_pageserver_ready_for_benchmarking(env, n_tenants) return env diff --git a/test_runner/performance/test_bulk_insert.py b/test_runner/performance/test_bulk_insert.py index 3f56da7c1d..3dad348976 100644 --- a/test_runner/performance/test_bulk_insert.py +++ b/test_runner/performance/test_bulk_insert.py @@ -4,7 +4,6 @@ import pytest from fixtures.benchmark_fixture import MetricReport from fixtures.common_types import Lsn from fixtures.compare_fixtures import NeonCompare, PgCompare -from fixtures.pageserver.utils import wait_tenant_status_404 from fixtures.pg_version import PgVersion @@ -68,7 +67,6 @@ def measure_recovery_time(env: NeonCompare): (attach_gen, _) = attach_status client.tenant_delete(env.tenant) - wait_tenant_status_404(client, env.tenant, iterations=60, interval=0.5) env.env.pageserver.tenant_create(tenant_id=env.tenant, generation=attach_gen) # Measure recovery time diff --git a/test_runner/performance/test_gc_feedback.py b/test_runner/performance/test_gc_feedback.py index 9a03994b29..4c326111c2 100644 --- a/test_runner/performance/test_gc_feedback.py +++ b/test_runner/performance/test_gc_feedback.py @@ -33,7 +33,7 @@ def test_gc_feedback(neon_env_builder: NeonEnvBuilder, zenbenchmark: NeonBenchma "checkpoint_distance": f"{1024 ** 2}", "compaction_target_size": f"{1024 ** 2}", # set PITR interval to be small, so we can do GC - "pitr_interval": "10 s", + "pitr_interval": "60 s", # "compaction_threshold": "3", # "image_creation_threshold": "2", } @@ -99,6 +99,52 @@ def test_gc_feedback(neon_env_builder: NeonEnvBuilder, zenbenchmark: NeonBenchma MetricReport.LOWER_IS_BETTER, ) + client.timeline_compact(tenant_id, timeline_id, enhanced_gc_bottom_most_compaction=True) + tline_detail = client.timeline_detail(tenant_id, timeline_id) + logical_size = tline_detail["current_logical_size"] + physical_size = tline_detail["current_physical_size"] + + max_num_of_deltas_above_image = 0 + max_total_num_of_deltas = 0 + for key_range in client.perf_info(tenant_id, timeline_id): + max_total_num_of_deltas = max(max_total_num_of_deltas, key_range["total_num_of_deltas"]) + max_num_of_deltas_above_image = max( + max_num_of_deltas_above_image, key_range["num_of_deltas_above_image"] + ) + zenbenchmark.record( + "logical_size_after_bottom_most_compaction", + logical_size // MB, + "Mb", + MetricReport.LOWER_IS_BETTER, + ) + zenbenchmark.record( + "physical_size_after_bottom_most_compaction", + physical_size // MB, + "Mb", + MetricReport.LOWER_IS_BETTER, + ) + zenbenchmark.record( + "physical/logical ratio after bottom_most_compaction", + physical_size / logical_size, + "", + MetricReport.LOWER_IS_BETTER, + ) + zenbenchmark.record( + "max_total_num_of_deltas_after_bottom_most_compaction", + max_total_num_of_deltas, + "", + MetricReport.LOWER_IS_BETTER, + ) + zenbenchmark.record( + "max_num_of_deltas_above_image_after_bottom_most_compaction", + max_num_of_deltas_above_image, + "", + MetricReport.LOWER_IS_BETTER, + ) + + with endpoint.cursor() as cur: + cur.execute("SELECT * FROM t") # ensure data is not corrupted + layer_map_path = env.repo_dir / "layer-map.json" log.info(f"Writing layer map to {layer_map_path}") with layer_map_path.open("w") as f: diff --git a/test_runner/performance/test_logical_replication.py b/test_runner/performance/test_logical_replication.py index b799f7248f..7d11facc29 100644 --- a/test_runner/performance/test_logical_replication.py +++ b/test_runner/performance/test_logical_replication.py @@ -2,9 +2,10 @@ import time import pytest from fixtures.log_helper import log -from fixtures.neon_fixtures import NeonEnv, PgBin, logical_replication_sync +from fixtures.neon_fixtures import AuxFileStore, NeonEnv, PgBin, logical_replication_sync +@pytest.mark.parametrize("pageserver_aux_file_policy", [AuxFileStore.V2]) @pytest.mark.timeout(1000) def test_logical_replication(neon_simple_env: NeonEnv, pg_bin: PgBin, vanilla_pg): env = neon_simple_env diff --git a/test_runner/performance/test_storage_controller_scale.py b/test_runner/performance/test_storage_controller_scale.py index cb013ae8c3..a4c8c8ac42 100644 --- a/test_runner/performance/test_storage_controller_scale.py +++ b/test_runner/performance/test_storage_controller_scale.py @@ -48,7 +48,16 @@ def test_storage_controller_many_tenants( # We will intentionally stress reconciler concurrrency, which triggers a warning when lots # of shards are hitting the delayed path. - env.storage_controller.allowed_errors.append(".*Many shards are waiting to reconcile") + env.storage_controller.allowed_errors.extend( + [ + # We will intentionally stress reconciler concurrrency, which triggers a warning when lots + # of shards are hitting the delayed path. + ".*Many shards are waiting to reconcile", + # We will create many timelines concurrently, so they might get slow enough to trip the warning + # that timeline creation is holding a lock too long. + ".*Shared lock by TimelineCreate.*was held.*", + ] + ) for ps in env.pageservers: # This can happen because when we do a loop over all pageservers and mark them offline/active, diff --git a/test_runner/regress/test_attach_tenant_config.py b/test_runner/regress/test_attach_tenant_config.py index f4667a82dc..e117c2140f 100644 --- a/test_runner/regress/test_attach_tenant_config.py +++ b/test_runner/regress/test_attach_tenant_config.py @@ -7,7 +7,7 @@ from fixtures.neon_fixtures import ( NeonEnv, NeonEnvBuilder, ) -from fixtures.pageserver.http import PageserverApiException, TenantConfig +from fixtures.pageserver.http import TenantConfig from fixtures.remote_storage import LocalFsStorage, RemoteStorageKind from fixtures.utils import wait_until @@ -82,8 +82,8 @@ def test_null_body(negative_env: NegativeTests): tenant_id = negative_env.tenant_id ps_http = env.pageserver.http_client() - res = ps_http.post( - f"{ps_http.base_url}/v1/tenant/{tenant_id}/attach", + res = ps_http.put( + f"{ps_http.base_url}/v1/tenant/{tenant_id}/location_config", data=b"null", headers={"Content-Type": "application/json"}, ) @@ -99,35 +99,16 @@ def test_null_config(negative_env: NegativeTests): tenant_id = negative_env.tenant_id ps_http = env.pageserver.http_client() - res = ps_http.post( - f"{ps_http.base_url}/v1/tenant/{tenant_id}/attach", - data=b'{"config": null}', + res = ps_http.put( + f"{ps_http.base_url}/v1/tenant/{tenant_id}/location_config", + json={"mode": "AttachedSingle", "generation": 1, "tenant_conf": None}, headers={"Content-Type": "application/json"}, ) assert res.status_code == 400 -def test_config_with_unknown_keys_is_bad_request(negative_env: NegativeTests): - """ - If we send a config with unknown keys, the request should be rejected with status 400. - """ - - env = negative_env.neon_env - tenant_id = negative_env.tenant_id - - config_with_unknown_keys = { - "compaction_period": "1h", - "this_key_does_not_exist": "some value", - } - - with pytest.raises(PageserverApiException) as e: - env.pageserver.tenant_attach(tenant_id, config=config_with_unknown_keys) - assert e.type == PageserverApiException - assert e.value.status_code == 400 - - @pytest.mark.parametrize("content_type", [None, "application/json"]) -def test_no_config(positive_env: NeonEnv, content_type: Optional[str]): +def test_empty_config(positive_env: NeonEnv, content_type: Optional[str]): """ When the 'config' body attribute is omitted, the request should be accepted and the tenant should use the default configuration @@ -141,11 +122,13 @@ def test_no_config(positive_env: NeonEnv, content_type: Optional[str]): ps_http.tenant_detach(tenant_id) assert tenant_id not in [TenantId(t["id"]) for t in ps_http.tenant_list()] - body = {"generation": env.storage_controller.attach_hook_issue(tenant_id, env.pageserver.id)} - - ps_http.post( - f"{ps_http.base_url}/v1/tenant/{tenant_id}/attach", - json=body, + ps_http.put( + f"{ps_http.base_url}/v1/tenant/{tenant_id}/location_config", + json={ + "mode": "AttachedSingle", + "generation": env.storage_controller.attach_hook_issue(tenant_id, env.pageserver.id), + "tenant_conf": {}, + }, headers=None if content_type else {"Content-Type": "application/json"}, ).raise_for_status() diff --git a/test_runner/regress/test_import.py b/test_runner/regress/test_import.py index ac27a4cf36..d97e882a70 100644 --- a/test_runner/regress/test_import.py +++ b/test_runner/regress/test_import.py @@ -76,7 +76,7 @@ def test_import_from_vanilla(test_output_dir, pg_bin, vanilla_pg, neon_env_build start_lsn = manifest["WAL-Ranges"][0]["Start-LSN"] end_lsn = manifest["WAL-Ranges"][0]["End-LSN"] - endpoint_id = "ep-import_from_vanilla" + branch_name = "import_from_vanilla" tenant = TenantId.generate() timeline = TimelineId.generate() @@ -106,8 +106,8 @@ def test_import_from_vanilla(test_output_dir, pg_bin, vanilla_pg, neon_env_build str(tenant), "--timeline-id", str(timeline), - "--node-name", - endpoint_id, + "--branch-name", + branch_name, "--base-lsn", start_lsn, "--base-tarfile", @@ -146,7 +146,7 @@ def test_import_from_vanilla(test_output_dir, pg_bin, vanilla_pg, neon_env_build wait_for_upload(client, tenant, timeline, Lsn(end_lsn)) # Check it worked - endpoint = env.endpoints.create_start(endpoint_id, tenant_id=tenant) + endpoint = env.endpoints.create_start(branch_name, tenant_id=tenant) assert endpoint.safe_psql("select count(*) from t") == [(300000,)] vanilla_pg.stop() @@ -265,7 +265,7 @@ def _import( tenant = TenantId.generate() # Import to pageserver - endpoint_id = "ep-import_from_pageserver" + branch_name = "import_from_pageserver" client = env.pageserver.http_client() env.pageserver.tenant_create(tenant) env.neon_cli.raw_cli( @@ -276,8 +276,8 @@ def _import( str(tenant), "--timeline-id", str(timeline), - "--node-name", - endpoint_id, + "--branch-name", + branch_name, "--base-lsn", str(lsn), "--base-tarfile", @@ -292,7 +292,7 @@ def _import( wait_for_upload(client, tenant, timeline, lsn) # Check it worked - endpoint = env.endpoints.create_start(endpoint_id, tenant_id=tenant, lsn=lsn) + endpoint = env.endpoints.create_start(branch_name, tenant_id=tenant, lsn=lsn) assert endpoint.safe_psql("select count(*) from tbl") == [(expected_num_rows,)] # Take another fullbackup diff --git a/test_runner/regress/test_local_file_cache.py b/test_runner/regress/test_local_file_cache.py index 76c6581448..3c404c3b23 100644 --- a/test_runner/regress/test_local_file_cache.py +++ b/test_runner/regress/test_local_file_cache.py @@ -1,4 +1,5 @@ import os +import queue import random import threading import time @@ -8,11 +9,7 @@ from fixtures.neon_fixtures import DEFAULT_BRANCH_NAME, NeonEnvBuilder from fixtures.utils import query_scalar -def test_local_file_cache_unlink(neon_env_builder: NeonEnvBuilder, build_type: str): - if build_type == "debug": - # Disable vectored read path cross validation since it makes the test time out. - neon_env_builder.pageserver_config_override = "validate_vectored_get=false" - +def test_local_file_cache_unlink(neon_env_builder: NeonEnvBuilder): env = neon_env_builder.init_start() cache_dir = os.path.join(env.repo_dir, "file_cache") @@ -33,11 +30,10 @@ def test_local_file_cache_unlink(neon_env_builder: NeonEnvBuilder, build_type: s cur = endpoint.connect().cursor() + stop = threading.Event() n_rows = 100000 n_threads = 20 - n_updates_per_thread = 10000 n_updates_per_connection = 1000 - n_total_updates = n_threads * n_updates_per_thread cur.execute("CREATE TABLE lfctest (id int4 PRIMARY KEY, n int) WITH (fillfactor=10)") cur.execute(f"INSERT INTO lfctest SELECT g, 1 FROM generate_series(1, {n_rows}) g") @@ -48,11 +44,11 @@ def test_local_file_cache_unlink(neon_env_builder: NeonEnvBuilder, build_type: s # performed (plus the initial 1 on each row). # # Furthermore, each thread will reconnect between every 1000 updates. - def run_updates(): + def run_updates(n_updates_performed_q: queue.Queue[int]): n_updates_performed = 0 conn = endpoint.connect() cur = conn.cursor() - for _ in range(n_updates_per_thread): + while not stop.is_set(): id = random.randint(1, n_rows) cur.execute(f"UPDATE lfctest SET n = n + 1 WHERE id = {id}") n_updates_performed += 1 @@ -61,19 +57,28 @@ def test_local_file_cache_unlink(neon_env_builder: NeonEnvBuilder, build_type: s conn.close() conn = endpoint.connect() cur = conn.cursor() + n_updates_performed_q.put(n_updates_performed) + n_updates_performed_q: queue.Queue[int] = queue.Queue() threads: List[threading.Thread] = [] for _i in range(n_threads): - thread = threading.Thread(target=run_updates, args=(), daemon=True) + thread = threading.Thread(target=run_updates, args=(n_updates_performed_q,), daemon=True) thread.start() threads.append(thread) time.sleep(5) + # unlink, this is what we're actually testing new_cache_dir = os.path.join(env.repo_dir, "file_cache_new") os.rename(cache_dir, new_cache_dir) + time.sleep(10) + + stop.set() + + n_updates_performed = 0 for thread in threads: thread.join() + n_updates_performed += n_updates_performed_q.get() - assert query_scalar(cur, "SELECT SUM(n) FROM lfctest") == n_total_updates + n_rows + assert query_scalar(cur, "SELECT SUM(n) FROM lfctest") == n_rows + n_updates_performed diff --git a/test_runner/regress/test_lsn_mapping.py b/test_runner/regress/test_lsn_mapping.py index 263730a823..67e82f8d30 100644 --- a/test_runner/regress/test_lsn_mapping.py +++ b/test_runner/regress/test_lsn_mapping.py @@ -12,10 +12,24 @@ from fixtures.utils import query_scalar, wait_until from requests.exceptions import ReadTimeout -# -# Test pageserver get_lsn_by_timestamp API -# -def test_lsn_mapping(neon_env_builder: NeonEnvBuilder): +def assert_lsn_lease_granted(result, with_lease: bool): + """ + Asserts an LSN lease is granted when `with_lease` flag is turned on. + Always asserts no LSN lease is granted when `with_lease` flag is off. + """ + if with_lease: + assert result.get("valid_until") + else: + assert result.get("valid_until") is None + + +@pytest.mark.parametrize("with_lease", [True, False]) +def test_lsn_mapping(neon_env_builder: NeonEnvBuilder, with_lease: bool): + """ + Test pageserver get_lsn_by_timestamp API. + + :param with_lease: Whether to get a lease associated with returned LSN. + """ env = neon_env_builder.init_start() tenant_id, _ = env.neon_cli.create_tenant( @@ -67,23 +81,33 @@ def test_lsn_mapping(neon_env_builder: NeonEnvBuilder): # Check edge cases # Timestamp is in the future probe_timestamp = tbl[-1][1] + timedelta(hours=1) - result = client.timeline_get_lsn_by_timestamp(tenant_id, timeline_id, probe_timestamp) + result = client.timeline_get_lsn_by_timestamp( + tenant_id, timeline_id, probe_timestamp, with_lease=with_lease + ) assert result["kind"] == "future" + assert_lsn_lease_granted(result, with_lease) # make sure that we return a well advanced lsn here assert Lsn(result["lsn"]) > start_lsn # Timestamp is in the unreachable past probe_timestamp = tbl[0][1] - timedelta(hours=10) - result = client.timeline_get_lsn_by_timestamp(tenant_id, timeline_id, probe_timestamp) + result = client.timeline_get_lsn_by_timestamp( + tenant_id, timeline_id, probe_timestamp, with_lease=with_lease + ) assert result["kind"] == "past" + assert_lsn_lease_granted(result, with_lease) + # make sure that we return the minimum lsn here at the start of the range assert Lsn(result["lsn"]) < start_lsn # Probe a bunch of timestamps in the valid range for i in range(1, len(tbl), 100): probe_timestamp = tbl[i][1] - result = client.timeline_get_lsn_by_timestamp(tenant_id, timeline_id, probe_timestamp) + result = client.timeline_get_lsn_by_timestamp( + tenant_id, timeline_id, probe_timestamp, with_lease=with_lease + ) assert result["kind"] not in ["past", "nodata"] + assert_lsn_lease_granted(result, with_lease) lsn = result["lsn"] # Call get_lsn_by_timestamp to get the LSN # Launch a new read-only node at that LSN, and check that only the rows @@ -105,8 +129,11 @@ def test_lsn_mapping(neon_env_builder: NeonEnvBuilder): # Timestamp is in the unreachable past probe_timestamp = tbl[0][1] - timedelta(hours=10) - result = client.timeline_get_lsn_by_timestamp(tenant_id, timeline_id_child, probe_timestamp) + result = client.timeline_get_lsn_by_timestamp( + tenant_id, timeline_id_child, probe_timestamp, with_lease=with_lease + ) assert result["kind"] == "past" + assert_lsn_lease_granted(result, with_lease) # make sure that we return the minimum lsn here at the start of the range assert Lsn(result["lsn"]) >= last_flush_lsn diff --git a/test_runner/regress/test_next_xid.py b/test_runner/regress/test_next_xid.py index 98fb06a0d6..b9e7e642b5 100644 --- a/test_runner/regress/test_next_xid.py +++ b/test_runner/regress/test_next_xid.py @@ -1,13 +1,14 @@ -import json import os import time from pathlib import Path -from fixtures.common_types import Lsn, TenantId, TimelineId +from fixtures.common_types import TenantId, TimelineId from fixtures.log_helper import log -from fixtures.neon_fixtures import NeonEnvBuilder, PgBin, wait_for_wal_insert_lsn -from fixtures.pageserver.utils import ( - wait_for_last_record_lsn, +from fixtures.neon_fixtures import ( + NeonEnvBuilder, + PgBin, + import_timeline_from_vanilla_postgres, + wait_for_wal_insert_lsn, ) from fixtures.remote_storage import RemoteStorageKind from fixtures.utils import query_scalar @@ -76,7 +77,6 @@ def test_import_at_2bil( ): neon_env_builder.enable_pageserver_remote_storage(RemoteStorageKind.LOCAL_FS) env = neon_env_builder.init_start() - ps_http = env.pageserver.http_client() # Reset the vanilla Postgres instance to somewhat before 2 billion transactions. pg_resetwal_path = os.path.join(pg_bin.pg_bin_path, "pg_resetwal") @@ -92,68 +92,28 @@ def test_import_at_2bil( assert vanilla_pg.safe_psql("select count(*) from tt") == [(300000,)] vanilla_pg.safe_psql("CREATE TABLE t (t text);") vanilla_pg.safe_psql("INSERT INTO t VALUES ('inserted in vanilla')") - - endpoint_id = "ep-import_from_vanilla" - tenant = TenantId.generate() - timeline = TimelineId.generate() - - env.pageserver.tenant_create(tenant) - - # Take basebackup - basebackup_dir = os.path.join(test_output_dir, "basebackup") - base_tar = os.path.join(basebackup_dir, "base.tar") - wal_tar = os.path.join(basebackup_dir, "pg_wal.tar") - os.mkdir(basebackup_dir) vanilla_pg.safe_psql("CHECKPOINT") - pg_bin.run( - [ - "pg_basebackup", - "-F", - "tar", - "-d", - vanilla_pg.connstr(), - "-D", - basebackup_dir, - ] + + tenant_id = TenantId.generate() + env.pageserver.tenant_create(tenant_id) + timeline_id = TimelineId.generate() + + # Import the cluster to Neon + import_timeline_from_vanilla_postgres( + test_output_dir, + env, + pg_bin, + tenant_id, + timeline_id, + "imported_2bil_xids", + vanilla_pg.connstr(), ) + vanilla_pg.stop() # don't need the original server anymore - # Get start_lsn and end_lsn - with open(os.path.join(basebackup_dir, "backup_manifest")) as f: - manifest = json.load(f) - start_lsn = manifest["WAL-Ranges"][0]["Start-LSN"] - end_lsn = manifest["WAL-Ranges"][0]["End-LSN"] - - def import_tar(base, wal): - env.neon_cli.raw_cli( - [ - "timeline", - "import", - "--tenant-id", - str(tenant), - "--timeline-id", - str(timeline), - "--node-name", - endpoint_id, - "--base-lsn", - start_lsn, - "--base-tarfile", - base, - "--end-lsn", - end_lsn, - "--wal-tarfile", - wal, - "--pg-version", - env.pg_version, - ] - ) - - # Importing correct backup works - import_tar(base_tar, wal_tar) - wait_for_last_record_lsn(ps_http, tenant, timeline, Lsn(end_lsn)) - + # Check that it works endpoint = env.endpoints.create_start( - endpoint_id, - tenant_id=tenant, + "imported_2bil_xids", + tenant_id=tenant_id, config_lines=[ "log_autovacuum_min_duration = 0", "autovacuum_naptime='5 s'", @@ -161,7 +121,6 @@ def test_import_at_2bil( ) assert endpoint.safe_psql("select count(*) from t") == [(1,)] - # Ok, consume conn = endpoint.connect() cur = conn.cursor() @@ -213,7 +172,7 @@ def test_import_at_2bil( cur.execute("checkpoint") # wait until pageserver receives that data - wait_for_wal_insert_lsn(env, endpoint, tenant, timeline) + wait_for_wal_insert_lsn(env, endpoint, tenant_id, timeline_id) # Restart endpoint endpoint.stop() diff --git a/test_runner/regress/test_pageserver_secondary.py b/test_runner/regress/test_pageserver_secondary.py index 2782d33e15..8431840dc0 100644 --- a/test_runner/regress/test_pageserver_secondary.py +++ b/test_runner/regress/test_pageserver_secondary.py @@ -11,8 +11,6 @@ from fixtures.neon_fixtures import NeonEnvBuilder, NeonPageserver, StorageScrubb from fixtures.pageserver.common_types import parse_layer_file_name from fixtures.pageserver.utils import ( assert_prefix_empty, - poll_for_remote_storage_iterations, - tenant_delete_wait_completed, wait_for_upload_queue_empty, ) from fixtures.remote_storage import LocalFsStorage, RemoteStorageKind, S3Storage, s3_storage @@ -363,8 +361,7 @@ def test_live_migration(neon_env_builder: NeonEnvBuilder): # Check that deletion works properly on a tenant that was live-migrated # (reproduce https://github.com/neondatabase/neon/issues/6802) - iterations = poll_for_remote_storage_iterations(remote_storage_kind) - tenant_delete_wait_completed(pageserver_b.http_client(), tenant_id, iterations) + pageserver_b.http_client().tenant_delete(tenant_id) def test_heatmap_uploads(neon_env_builder: NeonEnvBuilder): @@ -552,7 +549,7 @@ def test_secondary_downloads(neon_env_builder: NeonEnvBuilder): ) log.info("Deleting tenant...") - tenant_delete_wait_completed(ps_attached.http_client(), tenant_id, 10) + ps_attached.http_client().tenant_delete(tenant_id) assert_prefix_empty( neon_env_builder.pageserver_remote_storage, diff --git a/test_runner/regress/test_pg_regress.py b/test_runner/regress/test_pg_regress.py index 885a94a557..756a2c17c9 100644 --- a/test_runner/regress/test_pg_regress.py +++ b/test_runner/regress/test_pg_regress.py @@ -23,11 +23,11 @@ if TYPE_CHECKING: # Run the main PostgreSQL regression tests, in src/test/regress. # +@pytest.mark.timeout(600) @pytest.mark.parametrize("shard_count", [None, 4]) def test_pg_regress( neon_env_builder: NeonEnvBuilder, test_output_dir: Path, - build_type: str, pg_bin: PgBin, capsys: CaptureFixture[str], base_dir: Path, @@ -43,10 +43,6 @@ def test_pg_regress( if shard_count is not None: neon_env_builder.num_pageservers = shard_count - if build_type == "debug": - # Disable vectored read path cross validation since it makes the test time out. - neon_env_builder.pageserver_config_override = "validate_vectored_get=false" - neon_env_builder.enable_pageserver_remote_storage(s3_storage()) neon_env_builder.enable_scrub_on_exit() env = neon_env_builder.init_start(initial_tenant_shard_count=shard_count) diff --git a/test_runner/regress/test_physical_replication.py b/test_runner/regress/test_physical_replication.py index 034f2b669d..a1bff32eed 100644 --- a/test_runner/regress/test_physical_replication.py +++ b/test_runner/regress/test_physical_replication.py @@ -6,7 +6,6 @@ from fixtures.neon_fixtures import NeonEnv def test_physical_replication(neon_simple_env: NeonEnv): env = neon_simple_env - n_records = 100000 with env.endpoints.create_start( branch_name="main", endpoint_id="primary", @@ -22,8 +21,20 @@ def test_physical_replication(neon_simple_env: NeonEnv): with p_con.cursor() as p_cur: with secondary.connect() as s_con: with s_con.cursor() as s_cur: - for pk in range(n_records): + runtime_secs = 30 + started_at = time.time() + pk = 0 + while True: + pk += 1 + now = time.time() + if now - started_at > runtime_secs: + break p_cur.execute("insert into t (pk) values (%s)", (pk,)) + # an earlier version of this test was based on a fixed number of loop iterations + # and selected for pk=(random.randrange(1, fixed number of loop iterations)). + # => the probability of selection for a value that was never inserted changed from 99.9999% to 0% over the course of the test. + # + # We changed the test to where=(random.randrange(1, 2*pk)), which means the probability is now fixed to 50%. s_cur.execute( - "select * from t where pk=%s", (random.randrange(1, n_records),) + "select * from t where pk=%s", (random.randrange(1, 2 * pk),) ) diff --git a/test_runner/regress/test_remote_storage.py b/test_runner/regress/test_remote_storage.py index 7f79bf5d5c..b26bd3422f 100644 --- a/test_runner/regress/test_remote_storage.py +++ b/test_runner/regress/test_remote_storage.py @@ -164,13 +164,14 @@ def test_remote_storage_backup_and_restore( "data": {"reason": "storage-sync-list-remote-timelines"}, } + # Even though the tenant is broken, subsequent calls to location_conf API will succeed, but + # the tenant will always end up in a broken state as a result of the failpoint. # Ensure that even though the tenant is broken, retrying the attachment fails - with pytest.raises(Exception, match="Tenant state is Broken"): - # Use same generation as in previous attempt - gen_state = env.storage_controller.inspect(tenant_id) - assert gen_state is not None - generation = gen_state[0] - env.pageserver.tenant_attach(tenant_id, generation=generation) + tenant_info = wait_until_tenant_state(pageserver_http, tenant_id, "Broken", 15) + gen_state = env.storage_controller.inspect(tenant_id) + assert gen_state is not None + generation = gen_state[0] + env.pageserver.tenant_attach(tenant_id, generation=generation) # Restart again, this implicitly clears the failpoint. # test_remote_failures=1 remains active, though, as it's in the pageserver config. diff --git a/test_runner/regress/test_s3_restore.py b/test_runner/regress/test_s3_restore.py index 6383d24c57..9992647e56 100644 --- a/test_runner/regress/test_s3_restore.py +++ b/test_runner/regress/test_s3_restore.py @@ -11,8 +11,6 @@ from fixtures.pageserver.utils import ( MANY_SMALL_LAYERS_TENANT_CONFIG, assert_prefix_empty, enable_remote_storage_versioning, - poll_for_remote_storage_iterations, - tenant_delete_wait_completed, wait_for_upload, ) from fixtures.remote_storage import RemoteStorageKind, s3_storage @@ -83,8 +81,7 @@ def test_tenant_s3_restore( assert ( ps_http.get_metric_value("pageserver_tenant_manager_slots", {"mode": "attached"}) == 1 ), "tenant removed before we deletion was issued" - iterations = poll_for_remote_storage_iterations(remote_storage_kind) - tenant_delete_wait_completed(ps_http, tenant_id, iterations) + ps_http.tenant_delete(tenant_id) ps_http.deletion_queue_flush(execute=True) assert ( ps_http.get_metric_value("pageserver_tenant_manager_slots", {"mode": "attached"}) == 0 diff --git a/test_runner/regress/test_sharding.py b/test_runner/regress/test_sharding.py index 56075c5975..62a9f422ee 100644 --- a/test_runner/regress/test_sharding.py +++ b/test_runner/regress/test_sharding.py @@ -190,19 +190,20 @@ def test_sharding_split_compaction(neon_env_builder: NeonEnvBuilder, failpoint: """ Test that after a split, we clean up parent layer data in the child shards via compaction. """ + TENANT_CONF = { # small checkpointing and compaction targets to ensure we generate many upload operations - "checkpoint_distance": f"{128 * 1024}", - "compaction_threshold": "1", - "compaction_target_size": f"{128 * 1024}", + "checkpoint_distance": 128 * 1024, + "compaction_threshold": 1, + "compaction_target_size": 128 * 1024, # no PITR horizon, we specify the horizon when we request on-demand GC "pitr_interval": "3600s", # disable background compaction and GC. We invoke it manually when we want it to happen. "gc_period": "0s", "compaction_period": "0s", # create image layers eagerly, so that GC can remove some layers - "image_creation_threshold": "1", - "image_layer_creation_check_threshold": "0", + "image_creation_threshold": 1, + "image_layer_creation_check_threshold": 0, } neon_env_builder.storage_controller_config = { @@ -261,7 +262,9 @@ def test_sharding_split_compaction(neon_env_builder: NeonEnvBuilder, failpoint: env.pageserver.start() # Cleanup part 2: once layers are outside the PITR window, they will be rewritten if they are partially redundant - env.storage_controller.pageserver_api().set_tenant_config(tenant_id, {"pitr_interval": "0s"}) + updated_conf = TENANT_CONF.copy() + updated_conf["pitr_interval"] = "0s" + env.storage_controller.pageserver_api().set_tenant_config(tenant_id, updated_conf) env.storage_controller.reconcile_until_idle() for shard in shards: diff --git a/test_runner/regress/test_storage_controller.py b/test_runner/regress/test_storage_controller.py index 30f96ceee8..9cc13ecfdb 100644 --- a/test_runner/regress/test_storage_controller.py +++ b/test_runner/regress/test_storage_controller.py @@ -24,7 +24,6 @@ from fixtures.pageserver.utils import ( enable_remote_storage_versioning, list_prefix, remote_storage_delete_key, - tenant_delete_wait_completed, timeline_delete_wait_completed, ) from fixtures.pg_version import PgVersion @@ -158,7 +157,7 @@ def test_storage_controller_smoke( # Delete all the tenants for tid in tenant_ids: - tenant_delete_wait_completed(env.storage_controller.pageserver_api(), tid, 10) + env.storage_controller.pageserver_api().tenant_delete(tid) env.storage_controller.consistency_check() @@ -1384,7 +1383,8 @@ def test_lock_time_tracing(neon_env_builder: NeonEnvBuilder): tenant_id = env.initial_tenant env.storage_controller.allowed_errors.extend( [ - ".*Lock on.*", + ".*Exclusive lock by.*", + ".*Shared lock by.*", ".*Scheduling is disabled by policy.*", f".*Operation TimelineCreate on key {tenant_id} has waited.*", ] @@ -1416,11 +1416,25 @@ def test_lock_time_tracing(neon_env_builder: NeonEnvBuilder): ) thread_update_tenant_policy.join() - env.storage_controller.assert_log_contains("Lock on UpdatePolicy was held for") - env.storage_controller.assert_log_contains( + env.storage_controller.assert_log_contains("Exclusive lock by UpdatePolicy was held for") + _, last_log_cursor = env.storage_controller.assert_log_contains( f"Operation TimelineCreate on key {tenant_id} has waited" ) + # Test out shared lock + env.storage_controller.configure_failpoints( + ("tenant-create-timeline-shared-lock", "return(31000)") + ) + + timeline_id = TimelineId.generate() + # This will hold the shared lock for enough time to cause an warning + env.storage_controller.pageserver_api().timeline_create( + pg_version=PgVersion.NOT_SET, tenant_id=tenant_id, new_timeline_id=timeline_id + ) + env.storage_controller.assert_log_contains( + "Shared lock by TimelineCreate was held for", offset=last_log_cursor + ) + @pytest.mark.parametrize("remote_storage", [RemoteStorageKind.LOCAL_FS, s3_storage()]) @pytest.mark.parametrize("shard_count", [None, 4]) @@ -1504,6 +1518,49 @@ def test_tenant_import(neon_env_builder: NeonEnvBuilder, shard_count, remote_sto workload.validate() +def retryable_node_operation(op, ps_id, max_attempts, backoff): + while max_attempts > 0: + try: + op(ps_id) + return + except StorageControllerApiException as e: + max_attempts -= 1 + log.info(f"Operation failed ({max_attempts} attempts left): {e}") + + if max_attempts == 0: + raise e + + time.sleep(backoff) + + +def poll_node_status(env, node_id, desired_scheduling_policy, max_attempts, backoff): + log.info(f"Polling {node_id} for {desired_scheduling_policy} scheduling policy") + while max_attempts > 0: + try: + status = env.storage_controller.node_status(node_id) + policy = status["scheduling"] + if policy == desired_scheduling_policy: + return + else: + max_attempts -= 1 + log.info(f"Status call returned {policy=} ({max_attempts} attempts left)") + + if max_attempts == 0: + raise AssertionError( + f"Status for {node_id=} did not reach {desired_scheduling_policy=}" + ) + + time.sleep(backoff) + except StorageControllerApiException as e: + max_attempts -= 1 + log.info(f"Status call failed ({max_attempts} retries left): {e}") + + if max_attempts == 0: + raise e + + time.sleep(backoff) + + def test_graceful_cluster_restart(neon_env_builder: NeonEnvBuilder): """ Graceful reststart of storage controller clusters use the drain and @@ -1527,58 +1584,11 @@ def test_graceful_cluster_restart(neon_env_builder: NeonEnvBuilder): ) # Give things a chance to settle. - # A call to `reconcile_until_idle` could be used here instead, - # however since all attachments are placed on the same node, - # we'd have to wait for a long time (2 minutes-ish) for optimizations - # to quiesce. - # TODO: once the initial attachment selection is fixed, update this - # to use `reconcile_until_idle`. - time.sleep(2) + env.storage_controller.reconcile_until_idle(timeout_secs=30) nodes = env.storage_controller.node_list() assert len(nodes) == 2 - def retryable_node_operation(op, ps_id, max_attempts, backoff): - while max_attempts > 0: - try: - op(ps_id) - return - except StorageControllerApiException as e: - max_attempts -= 1 - log.info(f"Operation failed ({max_attempts} attempts left): {e}") - - if max_attempts == 0: - raise e - - time.sleep(backoff) - - def poll_node_status(node_id, desired_scheduling_policy, max_attempts, backoff): - log.info(f"Polling {node_id} for {desired_scheduling_policy} scheduling policy") - while max_attempts > 0: - try: - status = env.storage_controller.node_status(node_id) - policy = status["scheduling"] - if policy == desired_scheduling_policy: - return - else: - max_attempts -= 1 - log.info(f"Status call returned {policy=} ({max_attempts} attempts left)") - - if max_attempts == 0: - raise AssertionError( - f"Status for {node_id=} did not reach {desired_scheduling_policy=}" - ) - - time.sleep(backoff) - except StorageControllerApiException as e: - max_attempts -= 1 - log.info(f"Status call failed ({max_attempts} retries left): {e}") - - if max_attempts == 0: - raise e - - time.sleep(backoff) - def assert_shard_counts_balanced(env: NeonEnv, shard_counts, total_shards): # Assert that all nodes have some attached shards assert len(shard_counts) == len(env.pageservers) @@ -1594,7 +1604,7 @@ def test_graceful_cluster_restart(neon_env_builder: NeonEnvBuilder): retryable_node_operation( lambda ps_id: env.storage_controller.node_drain(ps_id), ps.id, max_attempts=3, backoff=2 ) - poll_node_status(ps.id, "PauseForRestart", max_attempts=6, backoff=5) + poll_node_status(env, ps.id, "PauseForRestart", max_attempts=6, backoff=5) shard_counts = get_node_shard_counts(env, tenant_ids) log.info(f"Shard counts after draining node {ps.id}: {shard_counts}") @@ -1604,12 +1614,12 @@ def test_graceful_cluster_restart(neon_env_builder: NeonEnvBuilder): assert sum(shard_counts.values()) == total_shards ps.restart() - poll_node_status(ps.id, "Active", max_attempts=10, backoff=1) + poll_node_status(env, ps.id, "Active", max_attempts=10, backoff=1) retryable_node_operation( lambda ps_id: env.storage_controller.node_fill(ps_id), ps.id, max_attempts=3, backoff=2 ) - poll_node_status(ps.id, "Active", max_attempts=6, backoff=5) + poll_node_status(env, ps.id, "Active", max_attempts=6, backoff=5) shard_counts = get_node_shard_counts(env, tenant_ids) log.info(f"Shard counts after filling node {ps.id}: {shard_counts}") @@ -1619,3 +1629,43 @@ def test_graceful_cluster_restart(neon_env_builder: NeonEnvBuilder): shard_counts = get_node_shard_counts(env, tenant_ids) log.info(f"Shard counts after rolling restart: {shard_counts}") assert_shard_counts_balanced(env, shard_counts, total_shards) + + +def test_background_operation_cancellation(neon_env_builder: NeonEnvBuilder): + neon_env_builder.num_pageservers = 2 + env = neon_env_builder.init_configs() + env.start() + + tenant_count = 5 + shard_count_per_tenant = 8 + tenant_ids = [] + + for _ in range(0, tenant_count): + tid = TenantId.generate() + tenant_ids.append(tid) + env.neon_cli.create_tenant( + tid, placement_policy='{"Attached":1}', shard_count=shard_count_per_tenant + ) + + # See sleep comment in the test above. + time.sleep(2) + + nodes = env.storage_controller.node_list() + assert len(nodes) == 2 + + env.storage_controller.configure_failpoints(("sleepy-drain-loop", "return(2000)")) + + ps_id_to_drain = env.pageservers[0].id + + retryable_node_operation( + lambda ps_id: env.storage_controller.node_drain(ps_id), + ps_id_to_drain, + max_attempts=3, + backoff=2, + ) + + poll_node_status(env, ps_id_to_drain, "Draining", max_attempts=6, backoff=2) + + env.storage_controller.cancel_node_drain(ps_id_to_drain) + + poll_node_status(env, ps_id_to_drain, "Active", max_attempts=6, backoff=2) diff --git a/test_runner/regress/test_tenant_delete.py b/test_runner/regress/test_tenant_delete.py index fd3cc45c3f..d3fba32a19 100644 --- a/test_runner/regress/test_tenant_delete.py +++ b/test_runner/regress/test_tenant_delete.py @@ -1,17 +1,11 @@ -import concurrent.futures -import enum -import os -import shutil from threading import Thread import pytest from fixtures.common_types import Lsn, TenantId, TimelineId -from fixtures.log_helper import log from fixtures.neon_fixtures import ( NeonEnvBuilder, PgBin, StorageScrubber, - last_flush_lsn_upload, wait_for_last_flush_lsn, ) from fixtures.pageserver.http import PageserverApiException @@ -19,18 +13,37 @@ from fixtures.pageserver.utils import ( MANY_SMALL_LAYERS_TENANT_CONFIG, assert_prefix_empty, assert_prefix_not_empty, - poll_for_remote_storage_iterations, - tenant_delete_wait_completed, wait_for_upload, - wait_tenant_status_404, - wait_until_tenant_active, - wait_until_tenant_state, ) -from fixtures.remote_storage import RemoteStorageKind, available_s3_storages, s3_storage +from fixtures.remote_storage import RemoteStorageKind, s3_storage from fixtures.utils import run_pg_bench_small, wait_until from requests.exceptions import ReadTimeout +def error_tolerant_delete(ps_http, tenant_id): + """ + For tests that inject 500 errors, we must retry repeatedly when issuing deletions + """ + while True: + try: + ps_http.tenant_delete(tenant_id=tenant_id) + except PageserverApiException as e: + if e.status_code == 500: + # This test uses failure injection, which can produce 500s as the pageserver expects + # the object store to always be available, and the ListObjects during deletion is generally + # an infallible operation. This can show up as a clear simulated error, or as a general + # error during delete_objects() + assert ( + "simulated failure of remote operation" in e.message + or "failed to delete" in e.message + ) + else: + raise + else: + # Success, drop out + break + + def test_tenant_delete_smoke( neon_env_builder: NeonEnvBuilder, pg_bin: PgBin, @@ -59,21 +72,7 @@ def test_tenant_delete_smoke( # Check that deleting a non-existent tenant gives the expected result: this is a loop because we # may need to retry on some remote storage errors injected by the test harness - while True: - try: - ps_http.tenant_delete(tenant_id=tenant_id) - except PageserverApiException as e: - if e.status_code == 500: - # This test uses failure injection, which can produce 500s as the pageserver expects - # the object store to always be available, and the ListObjects during deletion is generally - # an infallible operation - assert "simulated failure of remote operation" in e.message - elif e.status_code == 404: - # This is our expected result: trying to erase a non-existent tenant gives us 404 - assert "NotFound" in e.message - break - else: - raise + error_tolerant_delete(ps_http, tenant_id) env.neon_cli.create_tenant( tenant_id=tenant_id, @@ -108,10 +107,8 @@ def test_tenant_delete_smoke( # Upload a heatmap so that we exercise deletion of that too ps_http.tenant_heatmap_upload(tenant_id) - iterations = poll_for_remote_storage_iterations(remote_storage_kind) - assert ps_http.get_metric_value("pageserver_tenant_manager_slots", {"mode": "attached"}) == 2 - tenant_delete_wait_completed(ps_http, tenant_id, iterations) + error_tolerant_delete(ps_http, tenant_id) assert ps_http.get_metric_value("pageserver_tenant_manager_slots", {"mode": "attached"}) == 1 tenant_path = env.pageserver.tenant_dir(tenant_id) @@ -129,286 +126,7 @@ def test_tenant_delete_smoke( # Deletion updates the tenant count: the one default tenant remains assert ps_http.get_metric_value("pageserver_tenant_manager_slots", {"mode": "attached"}) == 1 - - -class Check(enum.Enum): - RETRY_WITHOUT_RESTART = enum.auto() - RETRY_WITH_RESTART = enum.auto() - - -FAILPOINTS = [ - "tenant-delete-before-shutdown", - "tenant-delete-before-create-remote-mark", - "tenant-delete-before-create-local-mark", - "tenant-delete-before-background", - "tenant-delete-before-polling-ongoing-deletions", - "tenant-delete-before-cleanup-remaining-fs-traces", - "tenant-delete-before-remove-timelines-dir", - "tenant-delete-before-remove-deleted-mark", - "tenant-delete-before-remove-tenant-dir", - # Some failpoints from timeline deletion - "timeline-delete-before-index-deleted-at", - "timeline-delete-before-rm", - "timeline-delete-before-index-delete", -] - -FAILPOINTS_BEFORE_BACKGROUND = [ - "timeline-delete-before-schedule", - "tenant-delete-before-shutdown", - "tenant-delete-before-create-remote-mark", - "tenant-delete-before-create-local-mark", - "tenant-delete-before-background", -] - - -def combinations(): - result = [] - - remotes = available_s3_storages() - - for remote_storage_kind in remotes: - for delete_failpoint in FAILPOINTS: - # Simulate failures for only one type of remote storage - # to avoid log pollution and make tests run faster - if remote_storage_kind is RemoteStorageKind.MOCK_S3: - simulate_failures = True - else: - simulate_failures = False - result.append((remote_storage_kind, delete_failpoint, simulate_failures)) - return result - - -@pytest.mark.parametrize("check", list(Check)) -@pytest.mark.parametrize("remote_storage_kind, failpoint, simulate_failures", combinations()) -def test_delete_tenant_exercise_crash_safety_failpoints( - neon_env_builder: NeonEnvBuilder, - remote_storage_kind: RemoteStorageKind, - failpoint: str, - simulate_failures: bool, - check: Check, - pg_bin: PgBin, -): - if simulate_failures: - neon_env_builder.pageserver_config_override = "test_remote_failures=1" - - neon_env_builder.enable_pageserver_remote_storage(remote_storage_kind) - - env = neon_env_builder.init_start(initial_tenant_conf=MANY_SMALL_LAYERS_TENANT_CONFIG) - - tenant_id = env.initial_tenant - - env.pageserver.allowed_errors.extend( - [ - # From deletion polling - f".*NotFound: tenant {env.initial_tenant}.*", - # allow errors caused by failpoints - f".*failpoint: {failpoint}", - # It appears when we stopped flush loop during deletion (attempt) and then pageserver is stopped - ".*shutdown.*tenant_id.*shutdown.*timeline_id.*: failed to freeze and flush: cannot flush frozen layers when flush_loop is not running, state is Exited", - # We may leave some upload tasks in the queue. They're likely deletes. - # For uploads we explicitly wait with `last_flush_lsn_upload` below. - # So by ignoring these instead of waiting for empty upload queue - # we execute more distinct code paths. - '.*stopping left-over name="remote upload".*', - # an on-demand is cancelled by shutdown - ".*initial size calculation failed: downloading failed, possibly for shutdown", - ] - ) - - if simulate_failures: - env.pageserver.allowed_errors.append( - # The deletion queue will complain when it encounters simulated S3 errors - ".*deletion executor: DeleteObjects request failed.*", - ) - - ps_http = env.pageserver.http_client() - - timeline_id = env.neon_cli.create_timeline("delete", tenant_id=tenant_id) - with env.endpoints.create_start("delete", tenant_id=tenant_id) as endpoint: - # generate enough layers - run_pg_bench_small(pg_bin, endpoint.connstr()) - last_flush_lsn_upload(env, endpoint, tenant_id, timeline_id) - - assert_prefix_not_empty( - neon_env_builder.pageserver_remote_storage, - prefix="/".join( - ( - "tenants", - str(tenant_id), - ) - ), - ) - - ps_http.configure_failpoints((failpoint, "return")) - - iterations = poll_for_remote_storage_iterations(remote_storage_kind) - - # These failpoints are earlier than background task is spawned. - # so they result in api request failure. - if failpoint in FAILPOINTS_BEFORE_BACKGROUND: - with pytest.raises(PageserverApiException, match=failpoint): - ps_http.tenant_delete(tenant_id) - - else: - ps_http.tenant_delete(tenant_id) - tenant_info = wait_until_tenant_state( - pageserver_http=ps_http, - tenant_id=tenant_id, - expected_state="Broken", - iterations=iterations, - ) - - reason = tenant_info["state"]["data"]["reason"] - log.info(f"tenant broken: {reason}") - - # failpoint may not be the only error in the stack - assert reason.endswith(f"failpoint: {failpoint}"), reason - - if check is Check.RETRY_WITH_RESTART: - env.pageserver.restart() - - if failpoint in ( - "tenant-delete-before-shutdown", - "tenant-delete-before-create-remote-mark", - ): - wait_until_tenant_active( - ps_http, tenant_id=tenant_id, iterations=iterations, period=0.25 - ) - tenant_delete_wait_completed(ps_http, tenant_id, iterations=iterations) - else: - # Pageserver should've resumed deletion after restart. - wait_tenant_status_404(ps_http, tenant_id, iterations=iterations + 10) - elif check is Check.RETRY_WITHOUT_RESTART: - # this should succeed - # this also checks that delete can be retried even when tenant is in Broken state - ps_http.configure_failpoints((failpoint, "off")) - - tenant_delete_wait_completed(ps_http, tenant_id, iterations=iterations) - - tenant_dir = env.pageserver.tenant_dir(tenant_id) - # Check local is empty - assert not tenant_dir.exists() - - # Check remote is empty - assert_prefix_empty( - neon_env_builder.pageserver_remote_storage, - prefix="/".join( - ( - "tenants", - str(tenant_id), - ) - ), - allowed_postfix="initdb.tar.zst", - ) - - -def test_tenant_delete_is_resumed_on_attach( - neon_env_builder: NeonEnvBuilder, - pg_bin: PgBin, -): - remote_storage_kind = s3_storage() - neon_env_builder.enable_pageserver_remote_storage(remote_storage_kind) - - env = neon_env_builder.init_start(initial_tenant_conf=MANY_SMALL_LAYERS_TENANT_CONFIG) - env.pageserver.allowed_errors.append( - # lucky race with stopping from flushing a layer we fail to schedule any uploads - ".*layer flush task.+: could not flush frozen layer: update_metadata_file" - ) - - tenant_id = env.initial_tenant - - ps_http = env.pageserver.http_client() - # create two timelines - for timeline in ["first", "second"]: - timeline_id = env.neon_cli.create_timeline(timeline, tenant_id=tenant_id) - with env.endpoints.create_start(timeline, tenant_id=tenant_id) as endpoint: - run_pg_bench_small(pg_bin, endpoint.connstr()) - wait_for_last_flush_lsn(env, endpoint, tenant=tenant_id, timeline=timeline_id) - - # sanity check, data should be there - assert_prefix_not_empty( - neon_env_builder.pageserver_remote_storage, - prefix="/".join( - ( - "tenants", - str(tenant_id), - ) - ), - ) - - # failpoint before we remove index_part from s3 - failpoint = "timeline-delete-before-index-delete" - ps_http.configure_failpoints((failpoint, "return")) - - env.pageserver.allowed_errors.extend( - ( - # allow errors caused by failpoints - f".*failpoint: {failpoint}", - # From deletion polling - f".*NotFound: tenant {env.initial_tenant}.*", - # It appears when we stopped flush loop during deletion (attempt) and then pageserver is stopped - ".*shutdown.*tenant_id.*shutdown.*timeline_id.*: failed to freeze and flush: cannot flush frozen layers when flush_loop is not running, state is Exited", - # error from http response is also logged - ".*InternalServerError\\(Tenant is marked as deleted on remote storage.*", - '.*shutdown_pageserver{exit_code=0}: stopping left-over name="remote upload".*', - ) - ) - - iterations = poll_for_remote_storage_iterations(remote_storage_kind) - - ps_http.tenant_delete(tenant_id) - - tenant_info = wait_until_tenant_state( - pageserver_http=ps_http, - tenant_id=tenant_id, - expected_state="Broken", - iterations=iterations, - ) - - assert_prefix_not_empty( - neon_env_builder.pageserver_remote_storage, - prefix="/".join( - ( - "tenants", - str(tenant_id), - ) - ), - ) - - reason = tenant_info["state"]["data"]["reason"] - # failpoint may not be the only error in the stack - assert reason.endswith(f"failpoint: {failpoint}"), reason - - # now we stop pageserver and remove local tenant state - env.endpoints.stop_all() - env.pageserver.stop() - - dir_to_clear = env.pageserver.tenant_dir() - shutil.rmtree(dir_to_clear) - os.mkdir(dir_to_clear) - - env.pageserver.start() - - # now we call attach - env.pageserver.tenant_attach(tenant_id=tenant_id) - - # delete should be resumed - wait_tenant_status_404(ps_http, tenant_id, iterations) - - # we shouldn've created tenant dir on disk - tenant_path = env.pageserver.tenant_dir(tenant_id) - assert not tenant_path.exists() - - ps_http.deletion_queue_flush(execute=True) - assert_prefix_empty( - neon_env_builder.pageserver_remote_storage, - prefix="/".join( - ( - "tenants", - str(tenant_id), - ) - ), - ) + assert ps_http.get_metric_value("pageserver_tenant_manager_slots", {"mode": "inprogress"}) == 0 def test_long_timeline_create_cancelled_by_tenant_delete(neon_env_builder: NeonEnvBuilder): @@ -483,105 +201,6 @@ def test_long_timeline_create_cancelled_by_tenant_delete(neon_env_builder: NeonE deletion.join() -def test_tenant_delete_concurrent( - neon_env_builder: NeonEnvBuilder, - pg_bin: PgBin, -): - """ - Validate that concurrent delete requests to the same tenant behave correctly: - exactly one should execute: the rest should give 202 responses but not start - another deletion. - - This is a reproducer for https://github.com/neondatabase/neon/issues/5936 - """ - neon_env_builder.enable_pageserver_remote_storage(RemoteStorageKind.MOCK_S3) - env = neon_env_builder.init_start(initial_tenant_conf=MANY_SMALL_LAYERS_TENANT_CONFIG) - ps_http = env.pageserver.http_client() - tenant_id = env.initial_tenant - timeline_id = env.initial_timeline - - # Populate some data - with env.endpoints.create_start("main", tenant_id=tenant_id) as endpoint: - run_pg_bench_small(pg_bin, endpoint.connstr()) - last_flush_lsn_upload(env, endpoint, tenant_id, timeline_id) - - env.pageserver.allowed_errors.extend( - [ - # lucky race with stopping from flushing a layer we fail to schedule any uploads - ".*layer flush task.+: could not flush frozen layer: update_metadata_file", - ] - ) - - BEFORE_REMOVE_FAILPOINT = "tenant-delete-before-map-remove" - BEFORE_RUN_FAILPOINT = "tenant-delete-before-run" - - # We will let the initial delete run until right before it would remove - # the tenant's TenantSlot. This pauses it in a state where the tenant - # is visible in Stopping state, and concurrent requests should fail with 4xx. - ps_http.configure_failpoints((BEFORE_REMOVE_FAILPOINT, "pause")) - - def delete_tenant(): - return ps_http.tenant_delete(tenant_id) - - def hit_remove_failpoint(): - return env.pageserver.assert_log_contains(f"at failpoint {BEFORE_REMOVE_FAILPOINT}")[1] - - def hit_run_failpoint(): - env.pageserver.assert_log_contains(f"at failpoint {BEFORE_RUN_FAILPOINT}") - - with concurrent.futures.ThreadPoolExecutor() as executor: - background_200_req = executor.submit(delete_tenant) - assert background_200_req.result(timeout=10).status_code == 202 - - # Wait until the first request completes its work and is blocked on removing - # the TenantSlot from tenant manager. - log_cursor = wait_until(100, 0.1, hit_remove_failpoint) - assert log_cursor is not None - - # Start another request: this should succeed without actually entering the deletion code - ps_http.tenant_delete(tenant_id) - assert not env.pageserver.log_contains( - f"at failpoint {BEFORE_RUN_FAILPOINT}", offset=log_cursor - ) - - # Start another background request, which will pause after acquiring a TenantSlotGuard - # but before completing. - ps_http.configure_failpoints((BEFORE_RUN_FAILPOINT, "pause")) - background_4xx_req = executor.submit(delete_tenant) - wait_until(100, 0.1, hit_run_failpoint) - - # The TenantSlot is still present while the original request is hung before - # final removal - assert ( - ps_http.get_metric_value("pageserver_tenant_manager_slots", {"mode": "attached"}) == 1 - ) - - # Permit the original request to run to success - ps_http.configure_failpoints((BEFORE_REMOVE_FAILPOINT, "off")) - - # Permit the duplicate background request to run to completion and fail. - ps_http.configure_failpoints((BEFORE_RUN_FAILPOINT, "off")) - background_4xx_req.result(timeout=10) - assert not env.pageserver.log_contains( - f"at failpoint {BEFORE_RUN_FAILPOINT}", offset=log_cursor - ) - - # Physical deletion should have happened - assert_prefix_empty( - neon_env_builder.pageserver_remote_storage, - prefix="/".join( - ( - "tenants", - str(tenant_id), - ) - ), - ) - - # Zero tenants remain (we deleted the default tenant) - assert ps_http.get_metric_value("pageserver_tenant_manager_slots", {"mode": "attached"}) == 0 - assert ps_http.get_metric_value("pageserver_tenant_manager_slots", {"mode": "inprogress"}) == 0 - - def test_tenant_delete_races_timeline_creation( neon_env_builder: NeonEnvBuilder, pg_bin: PgBin, @@ -674,9 +293,7 @@ def test_tenant_delete_races_timeline_creation( # Disable the failpoint and wait for deletion to finish ps_http.configure_failpoints((BEFORE_INITDB_UPLOAD_FAILPOINT, "off")) - iterations = poll_for_remote_storage_iterations(remote_storage_kind) - - tenant_delete_wait_completed(ps_http, tenant_id, iterations, ignore_errors=True) + ps_http.tenant_delete(tenant_id) # Physical deletion should have happened assert_prefix_empty( @@ -727,8 +344,7 @@ def test_tenant_delete_scrubber(pg_bin: PgBin, neon_env_builder: NeonEnvBuilder) env.start() ps_http = env.pageserver.http_client() - iterations = poll_for_remote_storage_iterations(remote_storage_kind) - tenant_delete_wait_completed(ps_http, tenant_id, iterations) + ps_http.tenant_delete(tenant_id) env.stop() scrubber.scan_metadata() diff --git a/test_runner/regress/test_tenant_detach.py b/test_runner/regress/test_tenant_detach.py index 871351b2d5..2056840558 100644 --- a/test_runner/regress/test_tenant_detach.py +++ b/test_runner/regress/test_tenant_detach.py @@ -275,16 +275,6 @@ def test_tenant_detach_smoke(neon_env_builder: NeonEnvBuilder): env.pageserver.allowed_errors.extend(PERMIT_PAGE_SERVICE_ERRORS) - # first check for non existing tenant - tenant_id = TenantId.generate() - with pytest.raises( - expected_exception=PageserverApiException, - match=f"NotFound: tenant {tenant_id}", - ) as excinfo: - pageserver_http.tenant_detach(tenant_id) - - assert excinfo.value.status_code == 404 - # create new nenant tenant_id, timeline_id = env.neon_cli.create_tenant() @@ -344,94 +334,6 @@ def test_tenant_detach_smoke(neon_env_builder: NeonEnvBuilder): pageserver_http.timeline_gc(tenant_id, timeline_id, 0) -# Creates and ignores a tenant, then detaches it: first, with no parameters (should fail), -# then with parameters to force ignored tenant detach (should not fail). -def test_tenant_detach_ignored_tenant(neon_simple_env: NeonEnv): - env = neon_simple_env - client = env.pageserver.http_client() - - # create a new tenant - tenant_id, _ = env.neon_cli.create_tenant() - - env.pageserver.allowed_errors.extend(PERMIT_PAGE_SERVICE_ERRORS) - - # assert tenant exists on disk - assert env.pageserver.tenant_dir(tenant_id).exists() - - endpoint = env.endpoints.create_start("main", tenant_id=tenant_id) - # we rely upon autocommit after each statement - endpoint.safe_psql_many( - queries=[ - "CREATE TABLE t(key int primary key, value text)", - "INSERT INTO t SELECT generate_series(1,100000), 'payload'", - ] - ) - - # ignore tenant - client.tenant_ignore(tenant_id) - env.pageserver.allowed_errors.append(".*NotFound: tenant .*") - # ensure tenant couldn't be detached without the special flag for ignored tenant - log.info("detaching ignored tenant WITHOUT required flag") - with pytest.raises( - expected_exception=PageserverApiException, match=f"NotFound: tenant {tenant_id}" - ): - client.tenant_detach(tenant_id) - - log.info("tenant detached failed as expected") - - # ensure tenant is detached with ignore state - log.info("detaching ignored tenant with required flag") - client.tenant_detach(tenant_id, True) - log.info("ignored tenant detached without error") - - # check that nothing is left on disk for deleted tenant - assert not env.pageserver.tenant_dir(tenant_id).exists() - - # assert the tenant does not exists in the Pageserver - tenants_after_detach = [tenant["id"] for tenant in client.tenant_list()] - assert ( - tenant_id not in tenants_after_detach - ), f"Ignored and then detached tenant {tenant_id} should not be present in pageserver's memory" - - -# Creates a tenant, and detaches it with extra paremeter that forces ignored tenant detach. -# Tenant should be detached without issues. -def test_tenant_detach_regular_tenant(neon_simple_env: NeonEnv): - env = neon_simple_env - client = env.pageserver.http_client() - - # create a new tenant - tenant_id, _ = env.neon_cli.create_tenant() - - env.pageserver.allowed_errors.extend(PERMIT_PAGE_SERVICE_ERRORS) - - # assert tenant exists on disk - assert env.pageserver.tenant_dir(tenant_id).exists() - - endpoint = env.endpoints.create_start("main", tenant_id=tenant_id) - # we rely upon autocommit after each statement - endpoint.safe_psql_many( - queries=[ - "CREATE TABLE t(key int primary key, value text)", - "INSERT INTO t SELECT generate_series(1,100000), 'payload'", - ] - ) - - log.info("detaching regular tenant with detach ignored flag") - client.tenant_detach(tenant_id, True) - - log.info("regular tenant detached without error") - - # check that nothing is left on disk for deleted tenant - assert not env.pageserver.tenant_dir(tenant_id).exists() - - # assert the tenant does not exists in the Pageserver - tenants_after_detach = [tenant["id"] for tenant in client.tenant_list()] - assert ( - tenant_id not in tenants_after_detach - ), f"Ignored and then detached tenant {tenant_id} should not be present in pageserver's memory" - - def test_detach_while_attaching( neon_env_builder: NeonEnvBuilder, ): @@ -500,153 +402,6 @@ def test_detach_while_attaching( cur.execute("SELECT COUNT(*) FROM foo") -# Tests that `ignore` and `get` operations' combination is able to remove and restore the tenant in pageserver's memory. -# * writes some data into tenant's timeline -# * ensures it's synced with the remote storage -# * `ignore` the tenant -# * verify that ignored tenant files are generally unchanged, only an ignored mark had appeared -# * verify the ignored tenant is gone from pageserver's memory -# * restart the pageserver and verify that ignored tenant is still not loaded -# * `load` the same tenant -# * ensure that it's status is `Active` and it's present in pageserver's memory with all timelines -def test_ignored_tenant_reattach(neon_env_builder: NeonEnvBuilder): - neon_env_builder.enable_pageserver_remote_storage(RemoteStorageKind.MOCK_S3) - env = neon_env_builder.init_start() - pageserver_http = env.pageserver.http_client() - - ignored_tenant_id, _ = env.neon_cli.create_tenant() - tenant_dir = env.pageserver.tenant_dir(ignored_tenant_id) - tenants_before_ignore = [tenant["id"] for tenant in pageserver_http.tenant_list()] - tenants_before_ignore.sort() - timelines_before_ignore = [ - timeline["timeline_id"] - for timeline in pageserver_http.timeline_list(tenant_id=ignored_tenant_id) - ] - files_before_ignore = [tenant_path for tenant_path in tenant_dir.glob("**/*")] - - # ignore the tenant and veirfy it's not present in pageserver replies, with its files still on disk - pageserver_http.tenant_ignore(ignored_tenant_id) - - files_after_ignore_with_retain = [tenant_path for tenant_path in tenant_dir.glob("**/*")] - new_files = set(files_after_ignore_with_retain) - set(files_before_ignore) - disappeared_files = set(files_before_ignore) - set(files_after_ignore_with_retain) - assert ( - len(disappeared_files) == 0 - ), f"Tenant ignore should not remove files from disk, missing: {disappeared_files}" - assert ( - len(new_files) == 1 - ), f"Only tenant ignore file should appear on disk but got: {new_files}" - - tenants_after_ignore = [tenant["id"] for tenant in pageserver_http.tenant_list()] - assert ignored_tenant_id not in tenants_after_ignore, "Ignored tenant should be missing" - assert len(tenants_after_ignore) + 1 == len( - tenants_before_ignore - ), "Only ignored tenant should be missing" - - # restart the pageserver to ensure we don't load the ignore timeline - env.pageserver.stop() - env.pageserver.start() - tenants_after_restart = [tenant["id"] for tenant in pageserver_http.tenant_list()] - tenants_after_restart.sort() - assert ( - tenants_after_restart == tenants_after_ignore - ), "Ignored tenant should not be reloaded after pageserver restart" - - # now, load it from the local files and expect it works - env.pageserver.tenant_load(tenant_id=ignored_tenant_id) - wait_until_tenant_state(pageserver_http, ignored_tenant_id, "Active", 5) - - tenants_after_attach = [tenant["id"] for tenant in pageserver_http.tenant_list()] - tenants_after_attach.sort() - assert tenants_after_attach == tenants_before_ignore, "Should have all tenants back" - - timelines_after_ignore = [ - timeline["timeline_id"] - for timeline in pageserver_http.timeline_list(tenant_id=ignored_tenant_id) - ] - assert timelines_before_ignore == timelines_after_ignore, "Should have all timelines back" - - -# Tests that it's possible to `load` tenants with missing layers and get them restored: -# * writes some data into tenant's timeline -# * ensures it's synced with the remote storage -# * `ignore` the tenant -# * removes all timeline's local layers -# * `load` the same tenant -# * ensure that it's status is `Active` -# * check that timeline data is restored -def test_ignored_tenant_download_missing_layers(neon_env_builder: NeonEnvBuilder): - neon_env_builder.enable_pageserver_remote_storage(RemoteStorageKind.LOCAL_FS) - env = neon_env_builder.init_start() - pageserver_http = env.pageserver.http_client() - endpoint = env.endpoints.create_start("main") - - tenant_id = env.initial_tenant - timeline_id = env.initial_timeline - - env.pageserver.allowed_errors.extend(PERMIT_PAGE_SERVICE_ERRORS) - - data_id = 1 - data_secret = "very secret secret" - insert_test_data(pageserver_http, tenant_id, timeline_id, data_id, data_secret, endpoint) - - tenants_before_ignore = [tenant["id"] for tenant in pageserver_http.tenant_list()] - tenants_before_ignore.sort() - timelines_before_ignore = [ - timeline["timeline_id"] for timeline in pageserver_http.timeline_list(tenant_id=tenant_id) - ] - - # ignore the tenant and remove its layers - pageserver_http.tenant_ignore(tenant_id) - timeline_dir = env.pageserver.timeline_dir(tenant_id, timeline_id) - layers_removed = False - for dir_entry in timeline_dir.iterdir(): - if dir_entry.name.startswith("00000"): - # Looks like a layer file. Remove it - dir_entry.unlink() - layers_removed = True - assert layers_removed, f"Found no layers for tenant {timeline_dir}" - - # now, load it from the local files and expect it to work due to remote storage restoration - env.pageserver.tenant_load(tenant_id=tenant_id) - wait_until_tenant_state(pageserver_http, tenant_id, "Active", 5) - - tenants_after_attach = [tenant["id"] for tenant in pageserver_http.tenant_list()] - tenants_after_attach.sort() - assert tenants_after_attach == tenants_before_ignore, "Should have all tenants back" - - timelines_after_ignore = [ - timeline["timeline_id"] for timeline in pageserver_http.timeline_list(tenant_id=tenant_id) - ] - assert timelines_before_ignore == timelines_after_ignore, "Should have all timelines back" - - endpoint.stop() - endpoint.start() - ensure_test_data(data_id, data_secret, endpoint) - - -# Tests that attach is never working on a tenant, ignored or not, as long as it's not absent locally -# Similarly, tests that it's not possible to schedule a `load` for tenat that's not ignored. -def test_load_negatives(neon_env_builder: NeonEnvBuilder): - neon_env_builder.enable_pageserver_remote_storage(RemoteStorageKind.LOCAL_FS) - env = neon_env_builder.init_start() - pageserver_http = env.pageserver.http_client() - env.endpoints.create_start("main") - - tenant_id = env.initial_tenant - - env.pageserver.allowed_errors.extend(PERMIT_PAGE_SERVICE_ERRORS) - - env.pageserver.allowed_errors.append(".*tenant .*? already exists, state:.*") - with pytest.raises( - expected_exception=PageserverApiException, - match=f"tenant {tenant_id} already exists, state: Active", - ): - env.pageserver.tenant_load(tenant_id) - - pageserver_http.tenant_ignore(tenant_id) - - def test_detach_while_activating( neon_env_builder: NeonEnvBuilder, ): @@ -770,7 +525,7 @@ def test_metrics_while_ignoring_broken_tenant_and_reloading( wait_until(10, 0.5, found_broken) - client.tenant_ignore(env.initial_tenant) + client.tenant_detach(env.initial_tenant) def found_cleaned_up(): m = client.get_metrics() @@ -782,7 +537,7 @@ def test_metrics_while_ignoring_broken_tenant_and_reloading( wait_until(10, 0.5, found_cleaned_up) - env.pageserver.tenant_load(env.initial_tenant) + env.pageserver.tenant_attach(env.initial_tenant) def found_active(): m = client.get_metrics() diff --git a/test_runner/regress/test_tenant_relocation.py b/test_runner/regress/test_tenant_relocation.py index be289e03d6..9fe732e288 100644 --- a/test_runner/regress/test_tenant_relocation.py +++ b/test_runner/regress/test_tenant_relocation.py @@ -15,7 +15,6 @@ from fixtures.pageserver.utils import ( assert_tenant_state, wait_for_last_record_lsn, wait_for_upload, - wait_tenant_status_404, ) from fixtures.remote_storage import ( LocalFsStorage, @@ -348,9 +347,6 @@ def test_tenant_relocation( # is no longer involved, and if it is, we will see the error origin_http.tenant_detach(tenant_id) - # Wait a little, so that the detach operation has time to finish. - wait_tenant_status_404(origin_http, tenant_id, iterations=100, interval=1) - post_migration_check(ep_main, 500500, old_local_path_main) post_migration_check(ep_second, 1001000, old_local_path_second) diff --git a/test_runner/regress/test_tenant_size.py b/test_runner/regress/test_tenant_size.py index a3dd422903..6c85ddebbc 100644 --- a/test_runner/regress/test_tenant_size.py +++ b/test_runner/regress/test_tenant_size.py @@ -15,7 +15,6 @@ from fixtures.neon_fixtures import ( ) from fixtures.pageserver.http import PageserverApiException, PageserverHttpClient from fixtures.pageserver.utils import ( - tenant_delete_wait_completed, timeline_delete_wait_completed, wait_until_tenant_active, ) @@ -669,7 +668,7 @@ def test_synthetic_size_while_deleting(neon_env_builder: NeonEnvBuilder): ), ) - tenant_delete_wait_completed(client, env.initial_tenant, 10) + client.tenant_delete(env.initial_tenant) client.configure_failpoints((failpoint, "off")) diff --git a/test_runner/regress/test_timeline_detach_ancestor.py b/test_runner/regress/test_timeline_detach_ancestor.py index f0b2f7d733..606ce203cd 100644 --- a/test_runner/regress/test_timeline_detach_ancestor.py +++ b/test_runner/regress/test_timeline_detach_ancestor.py @@ -14,7 +14,7 @@ from fixtures.neon_fixtures import ( wait_for_last_flush_lsn, ) from fixtures.pageserver.http import HistoricLayerInfo, PageserverApiException -from fixtures.pageserver.utils import wait_tenant_status_404, wait_timeline_detail_404 +from fixtures.pageserver.utils import wait_timeline_detail_404 from fixtures.remote_storage import LocalFsStorage from fixtures.utils import assert_pageserver_backups_equal @@ -578,7 +578,6 @@ def test_timeline_ancestor_errors(neon_env_builder: NeonEnvBuilder): assert info.value.status_code == 400 client.tenant_delete(env.initial_tenant) - wait_tenant_status_404(client, env.initial_tenant, 10, 1) with pytest.raises(PageserverApiException) as e: client.detach_ancestor(env.initial_tenant, first_branch) diff --git a/test_runner/regress/test_timeline_size.py b/test_runner/regress/test_timeline_size.py index db5297870e..f47356839c 100644 --- a/test_runner/regress/test_timeline_size.py +++ b/test_runner/regress/test_timeline_size.py @@ -26,7 +26,6 @@ from fixtures.pageserver.utils import ( assert_tenant_state, timeline_delete_wait_completed, wait_for_upload_queue_empty, - wait_tenant_status_404, wait_until_tenant_active, ) from fixtures.pg_version import PgVersion @@ -841,7 +840,7 @@ def test_ondemand_activation(neon_env_builder: NeonEnvBuilder): # Detaching a stuck tenant should proceed promptly # (reproducer for https://github.com/neondatabase/neon/pull/6430) - env.pageserver.http_client().tenant_detach(detach_tenant_id, timeout_secs=10) + env.pageserver.http_client().tenant_detach(detach_tenant_id) tenant_ids.remove(detach_tenant_id) # FIXME: currently the mechanism for cancelling attach is to set state to broken, which is reported spuriously at error level env.pageserver.allowed_errors.append( @@ -864,39 +863,33 @@ def delete_lazy_activating( ): pageserver_http = pageserver.http_client() - # Deletion itself won't complete due to our failpoint: Tenant::shutdown can't complete while calculating - # logical size is paused in a failpoint. So instead we will use a log observation to check that - # on-demand activation was triggered by the tenant deletion - log_match = f".*attach{{tenant_id={delete_tenant_id} shard_id=0000 gen=[0-9a-f]+}}: Activating tenant \\(on-demand\\).*" - if expect_attaching: assert pageserver_http.tenant_status(delete_tenant_id)["state"]["slug"] == "Attaching" with concurrent.futures.ThreadPoolExecutor() as executor: log.info("Starting background delete") - def activated_on_demand(): - assert pageserver.log_contains(log_match) is not None + def shutting_down(): + assert pageserver.log_contains(".*Waiting for timelines.*") is not None def delete_tenant(): pageserver_http.tenant_delete(delete_tenant_id) background_delete = executor.submit(delete_tenant) - log.info(f"Waiting for activation message '{log_match}'") + # We expect deletion to enter shutdown of the tenant even though it's in the attaching state try: - wait_until(10, 1, activated_on_demand) + # Deletion will get to the point in shutdown where it's waiting for timeline shutdown, then + # hang because of our failpoint blocking activation. + wait_until(10, 1, shutting_down) finally: log.info("Clearing failpoint") pageserver_http.configure_failpoints(("timeline-calculate-logical-size-pause", "off")) - # Deletion should complete successfully now that failpoint is unblocked + # Deletion should complete successfully now that failpoint is unblocked and shutdown can complete log.info("Joining background delete") background_delete.result(timeout=10) - # Poll for deletion to complete - wait_tenant_status_404(pageserver_http, tenant_id=delete_tenant_id, iterations=40) - def test_timeline_logical_size_task_priority(neon_env_builder: NeonEnvBuilder): """ diff --git a/test_runner/regress/test_wal_acceptor.py b/test_runner/regress/test_wal_acceptor.py index 7bf208db54..ac1a3bef67 100644 --- a/test_runner/regress/test_wal_acceptor.py +++ b/test_runner/regress/test_wal_acceptor.py @@ -1,4 +1,5 @@ import filecmp +import logging import os import random import shutil @@ -2178,3 +2179,102 @@ def test_broker_discovery(neon_env_builder: NeonEnvBuilder): do_something() do_something() + + +# Test creates 5 endpoints and tries to wake them up randomly. All timeouts are +# configured to be very short, so that we expect that: +# - pageserver will update remote_consistent_lsn very often +# - safekeepers will upload partial WAL segments very often +# - safekeeper will try to evict and unevict timelines +# +# Test checks that there are no critical errors while doing this. Also it checks +# that every safekeeper has at least one successful eviction. +@pytest.mark.parametrize("delete_offloaded_wal", [False, True]) +@pytest.mark.parametrize("restart_chance", [0.0, 0.2]) +def test_s3_eviction( + neon_env_builder: NeonEnvBuilder, delete_offloaded_wal: bool, restart_chance: float +): + neon_env_builder.num_safekeepers = 3 + neon_env_builder.enable_safekeeper_remote_storage(RemoteStorageKind.LOCAL_FS) + env = neon_env_builder.init_start( + initial_tenant_conf={ + "checkpoint_timeout": "100ms", + } + ) + + extra_opts = [ + "--enable-offload", + "--partial-backup-timeout", + "50ms", + "--control-file-save-interval", + "1s", + ] + if delete_offloaded_wal: + extra_opts.append("--delete-offloaded-wal") + + for sk in env.safekeepers: + sk.stop().start(extra_opts=extra_opts) + + n_timelines = 5 + + branch_names = [f"branch{tlin}" for tlin in range(n_timelines)] + timelines = [] + ps_client = env.pageservers[0].http_client() + + # start postgres on each timeline + endpoints: list[Endpoint] = [] + for branch_name in branch_names: + timeline_id = env.neon_cli.create_branch(branch_name) + timelines.append(timeline_id) + + endpoints.append(env.endpoints.create_start(branch_name)) + endpoints[-1].safe_psql("CREATE TABLE t(i int)") + endpoints[-1].safe_psql("INSERT INTO t VALUES (0)") + + lsn = endpoints[-1].safe_psql("SELECT pg_current_wal_flush_lsn()")[0][0] + log.info(f"{branch_name}: LSN={lsn}") + + endpoints[-1].stop() + + # update remote_consistent_lsn on pageserver + ps_client.timeline_checkpoint(env.initial_tenant, timelines[-1], wait_until_uploaded=True) + + check_values = [0] * n_timelines + + n_iters = 20 + for _ in range(n_iters): + if log.isEnabledFor(logging.DEBUG): + for j in range(n_timelines): + detail = ps_client.timeline_detail(env.initial_tenant, timelines[j]) + log.debug( + f'{branch_names[j]}: RCL={detail["remote_consistent_lsn"]}, LRL={detail["last_record_lsn"]}' + ) + + i = random.randint(0, n_timelines - 1) + log.info(f"Starting endpoint {i}") + endpoints[i].start() + check_values[i] += 1 + res = endpoints[i].safe_psql("UPDATE t SET i = i + 1 RETURNING i") + assert res[0][0] == check_values[i] + + lsn = endpoints[i].safe_psql("SELECT pg_current_wal_flush_lsn()")[0][0] + log.info(f"{branch_names[i]}: LSN={lsn}") + + endpoints[i].stop() + + # update remote_consistent_lsn on pageserver + ps_client.timeline_checkpoint(env.initial_tenant, timelines[i], wait_until_uploaded=True) + + # restarting random safekeepers + for sk in env.safekeepers: + if random.random() < restart_chance: + sk.stop().start(extra_opts=extra_opts) + time.sleep(0.5) + + # require at least one successful eviction in at least one safekeeper + # TODO: require eviction in each safekeeper after https://github.com/neondatabase/neon/issues/8148 is fixed + assert any( + sk.log_contains("successfully evicted timeline") + and sk.log_contains("successfully restored evicted timeline") + for sk in env.safekeepers + ) diff --git a/test_runner/regress/test_wal_acceptor_async.py b/test_runner/regress/test_wal_acceptor_async.py index 971fad787a..3f0a4a2ff8 100644 --- a/test_runner/regress/test_wal_acceptor_async.py +++ b/test_runner/regress/test_wal_acceptor_async.py @@ -200,9 +200,8 @@ async def run_restarts_under_load( # assert that at least one transaction has completed in every worker stats.check_progress() - # testing #6530, temporary here - # TODO: remove afer partial backup is enabled by default - victim.start(extra_opts=["--partial-backup-enabled", "--partial-backup-timeout=2s"]) + # testing #6530 + victim.start(extra_opts=["--partial-backup-timeout=2s"]) log.info("Iterations are finished, exiting coroutines...") stats.running = False diff --git a/test_runner/regress/test_walredo_not_left_behind_on_detach.py b/test_runner/regress/test_walredo_not_left_behind_on_detach.py index ad37807dba..375cfcb4fe 100644 --- a/test_runner/regress/test_walredo_not_left_behind_on_detach.py +++ b/test_runner/regress/test_walredo_not_left_behind_on_detach.py @@ -37,7 +37,7 @@ def test_walredo_not_left_behind_on_detach(neon_env_builder: NeonEnvBuilder): expected_exception=PageserverApiException, match=f"NotFound: tenant {tenant_id}", ): - pageserver_http.tenant_detach(tenant_id) + pageserver_http.tenant_status(tenant_id) # create new nenant tenant_id, _ = env.neon_cli.create_tenant() diff --git a/trace/src/main.rs b/trace/src/main.rs index 049f922b6f..79e1df988d 100644 --- a/trace/src/main.rs +++ b/trace/src/main.rs @@ -38,12 +38,6 @@ enum Command { /// Print stats and anomalies about the traces Analyze, - - /// Draw the traces in svg format - Draw, - - /// Send the read requests to a pageserver - Replay, } // HACK This function will change and improve as we see what kind of analysis is useful. @@ -167,8 +161,6 @@ fn main() -> anyhow::Result<()> { analyze_trace(reader); } } - Command::Draw => todo!(), - Command::Replay => todo!(), } Ok(()) diff --git a/vendor/postgres-v14 b/vendor/postgres-v14 index 4c51945a61..7845c122d5 160000 --- a/vendor/postgres-v14 +++ b/vendor/postgres-v14 @@ -1 +1 @@ -Subproject commit 4c51945a6167ca06c0169e7a4ca5a8e7ffa3faba +Subproject commit 7845c122d51d3ebb547a984a640ac0310a2fadce diff --git a/vendor/postgres-v15 b/vendor/postgres-v15 index e22098d86d..2ff5ecc67c 160000 --- a/vendor/postgres-v15 +++ b/vendor/postgres-v15 @@ -1 +1 @@ -Subproject commit e22098d86d6c40276b6bd75c29133a33fb283ab6 +Subproject commit 2ff5ecc67c64e5fe44b7dde598e64e4538e0c373 diff --git a/vendor/postgres-v16 b/vendor/postgres-v16 index 9837db1578..d55e0aca10 160000 --- a/vendor/postgres-v16 +++ b/vendor/postgres-v16 @@ -1 +1 @@ -Subproject commit 9837db157837fcf43ef7348be0017d3a2238cd27 +Subproject commit d55e0aca104af0b611cf5565f1033b2acd2dcc1c diff --git a/vendor/revisions.json b/vendor/revisions.json index f945ea6d73..e755cf2e9d 100644 --- a/vendor/revisions.json +++ b/vendor/revisions.json @@ -1,5 +1,5 @@ { - "v16": ["16.3", "9837db157837fcf43ef7348be0017d3a2238cd27"], - "v15": ["15.7", "e22098d86d6c40276b6bd75c29133a33fb283ab6"], - "v14": ["14.12", "4c51945a6167ca06c0169e7a4ca5a8e7ffa3faba"] + "v16": ["16.3", "d55e0aca104af0b611cf5565f1033b2acd2dcc1c"], + "v15": ["15.7", "2ff5ecc67c64e5fe44b7dde598e64e4538e0c373"], + "v14": ["14.12", "7845c122d51d3ebb547a984a640ac0310a2fadce"] } diff --git a/workspace_hack/Cargo.toml b/workspace_hack/Cargo.toml index df16c71789..f43076171f 100644 --- a/workspace_hack/Cargo.toml +++ b/workspace_hack/Cargo.toml @@ -25,6 +25,7 @@ axum = { version = "0.6", features = ["ws"] } base64 = { version = "0.21", features = ["alloc"] } base64ct = { version = "1", default-features = false, features = ["std"] } bytes = { version = "1", features = ["serde"] } +camino = { version = "1", default-features = false, features = ["serde1"] } chrono = { version = "0.4", default-features = false, features = ["clock", "serde", "wasmbind"] } clap = { version = "4", features = ["derive", "string"] } clap_builder = { version = "4", default-features = false, features = ["color", "help", "std", "string", "suggestions", "usage"] } @@ -114,7 +115,6 @@ syn-dff4ba8e3ae991db = { package = "syn", version = "1", features = ["extra-trai syn-f595c2ba2a3f28df = { package = "syn", version = "2", features = ["extra-traits", "fold", "full", "visit", "visit-mut"] } time-macros = { version = "0.2", default-features = false, features = ["formatting", "parsing", "serde"] } toml_datetime = { version = "0.6", default-features = false, features = ["serde"] } -toml_edit = { version = "0.19", features = ["serde"] } zstd = { version = "0.13" } zstd-safe = { version = "7", default-features = false, features = ["arrays", "legacy", "std", "zdict_builder"] } zstd-sys = { version = "2", default-features = false, features = ["legacy", "std", "zdict_builder"] }