From 4aa9786c6bffe8094cfc82947504d2044e284d7b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?JC=20Gr=C3=BCnhage?= Date: Tue, 7 Jan 2025 14:45:18 +0100 Subject: [PATCH 001/111] Fix promote-images-prod after splitting it out (#10292) ## Problem `promote-images` was split into `promote-images-dev` and `promote-images-prod` in https://github.com/neondatabase/neon/pull/10267. `dev` credentials were loaded in `promote-images-dev` and `prod` credentials were loaded in `promote-images-prod`, but `promote-images-prod` needs `dev` credentials as well to access the `dev` images to replicate them from `dev` to `prod`. ## Summary of changes Load `dev` credentials in `promote-images-prod` as well. --- .github/workflows/build_and_test.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index 5c2b397c82..01f5c3ede9 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -979,6 +979,16 @@ jobs: VERSIONS: v14 v15 v16 v17 steps: + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + aws-region: eu-central-1 + role-to-assume: ${{ vars.DEV_AWS_OIDC_ROLE_ARN }} + role-duration-seconds: 3600 + + - name: Login to Amazon Dev ECR + uses: aws-actions/amazon-ecr-login@v2 + - uses: docker/login-action@v3 with: username: ${{ secrets.NEON_DOCKERHUB_USERNAME }} From 0a117fb1f1a9c2a062c8a60d53f2eb00637392e8 Mon Sep 17 00:00:00 2001 From: Folke Behrens Date: Tue, 7 Jan 2025 16:24:54 +0100 Subject: [PATCH 002/111] proxy: Parse Notification twice only for unknown topic (#10296) ## Problem We currently parse Notification twice even in the happy path. ## Summary of changes Use `#[serde(other)]` to catch unknown topics and defer the second parsing. --- proxy/src/redis/notifications.rs | 55 ++++++++++++-------------------- 1 file changed, 21 insertions(+), 34 deletions(-) diff --git a/proxy/src/redis/notifications.rs b/proxy/src/redis/notifications.rs index 4383d6be2c..bf9d61ded3 100644 --- a/proxy/src/redis/notifications.rs +++ b/proxy/src/redis/notifications.rs @@ -37,7 +37,6 @@ struct NotificationHeader<'a> { #[derive(Clone, Debug, Serialize, Deserialize, Eq, PartialEq)] #[serde(tag = "topic", content = "data")] -// Message to contributors: Make sure to align these topic names with the list below. pub(crate) enum Notification { #[serde( rename = "/allowed_ips_updated", @@ -74,21 +73,9 @@ pub(crate) enum Notification { PasswordUpdate { password_update: PasswordUpdate }, #[serde(rename = "/cancel_session")] Cancel(CancelSession), -} -/// Returns true if the topic name given is a known topic that we can deserialize and action on. -/// Returns false otherwise. -fn known_topic(s: &str) -> bool { - // Message to contributors: Make sure to align these topic names with the enum above. - matches!( - s, - "/allowed_ips_updated" - | "/block_public_or_vpc_access_updated" - | "/allowed_vpc_endpoints_updated_for_org" - | "/allowed_vpc_endpoints_updated_for_projects" - | "/password_updated" - | "/cancel_session" - ) + #[serde(other, skip_serializing)] + UnknownTopic, } #[derive(Clone, Debug, Serialize, Deserialize, Eq, PartialEq)] @@ -178,32 +165,29 @@ impl MessageHandler { let payload: String = msg.get_payload()?; tracing::debug!(?payload, "received a message payload"); - // For better error handling, we first parse the payload to extract the topic. - // If there's a topic we don't support, we can handle that error more gracefully. - let header: NotificationHeader = match serde_json::from_str(&payload) { - Ok(msg) => msg, - Err(e) => { - Metrics::get().proxy.redis_errors_total.inc(RedisErrors { - channel: msg.get_channel_name(), - }); - tracing::error!("broken message: {e}"); + let msg: Notification = match serde_json::from_str(&payload) { + Ok(Notification::UnknownTopic) => { + match serde_json::from_str::(&payload) { + // don't update the metric for redis errors if it's just a topic we don't know about. + Ok(header) => tracing::warn!(topic = header.topic, "unknown topic"), + Err(e) => { + Metrics::get().proxy.redis_errors_total.inc(RedisErrors { + channel: msg.get_channel_name(), + }); + tracing::error!("broken message: {e}"); + } + }; return Ok(()); } - }; - - if !known_topic(header.topic) { - // don't update the metric for redis errors if it's just a topic we don't know about. - tracing::warn!(topic = header.topic, "unknown topic"); - return Ok(()); - } - - let msg: Notification = match serde_json::from_str(&payload) { Ok(msg) => msg, Err(e) => { Metrics::get().proxy.redis_errors_total.inc(RedisErrors { channel: msg.get_channel_name(), }); - tracing::error!(topic = header.topic, "broken message: {e}"); + match serde_json::from_str::(&payload) { + Ok(header) => tracing::error!(topic = header.topic, "broken message: {e}"), + Err(_) => tracing::error!("broken message: {e}"), + }; return Ok(()); } }; @@ -278,6 +262,8 @@ impl MessageHandler { invalidate_cache(cache, msg); }); } + + Notification::UnknownTopic => unreachable!(), } Ok(()) @@ -304,6 +290,7 @@ fn invalidate_cache(cache: Arc, msg: Notification) { Notification::AllowedVpcEndpointsUpdatedForProjects { .. } => { // https://github.com/neondatabase/neon/pull/10073 } + Notification::UnknownTopic => unreachable!(), } } From 43a5e575d611c546c9bf5d538a57d8984560589d Mon Sep 17 00:00:00 2001 From: Fedor Dikarev Date: Tue, 7 Jan 2025 21:00:56 +0100 Subject: [PATCH 003/111] ci: use reusable workflow for MacOs build (#9889) Closes: https://github.com/neondatabase/cloud/issues/17784 ## Problem Currently, we run the whole CI pipeline for any changes. It's slow and expensive. ## Suggestion Starting with MacOs builds: - check what files were changed - rebuild only needed parts - reuse results from previous builds when available - run builds in parallel when possible --------- Co-authored-by: Alexander Bayandin --- .github/file-filters.yaml | 12 ++ .github/workflows/build-macos.yml | 241 ++++++++++++++++++++++++ .github/workflows/neon_extra_builds.yml | 139 ++++---------- 3 files changed, 290 insertions(+), 102 deletions(-) create mode 100644 .github/file-filters.yaml create mode 100644 .github/workflows/build-macos.yml diff --git a/.github/file-filters.yaml b/.github/file-filters.yaml new file mode 100644 index 0000000000..886cd3919a --- /dev/null +++ b/.github/file-filters.yaml @@ -0,0 +1,12 @@ +rust_code: ['**/*.rs', '**/Cargo.toml', '**/Cargo.lock'] + +v14: ['vendor/postgres-v14/**', 'Makefile', 'pgxn/**'] +v15: ['vendor/postgres-v15/**', 'Makefile', 'pgxn/**'] +v16: ['vendor/postgres-v16/**', 'Makefile', 'pgxn/**'] +v17: ['vendor/postgres-v17/**', 'Makefile', 'pgxn/**'] + +rebuild_neon_extra: + - .github/workflows/neon_extra_builds.yml + +rebuild_macos: + - .github/workflows/build-macos.yml diff --git a/.github/workflows/build-macos.yml b/.github/workflows/build-macos.yml new file mode 100644 index 0000000000..01d82a1ed2 --- /dev/null +++ b/.github/workflows/build-macos.yml @@ -0,0 +1,241 @@ +name: Check neon with MacOS builds + +on: + workflow_call: + inputs: + pg_versions: + description: "Array of the pg versions to build for, for example: ['v14', 'v17']" + type: string + default: '[]' + required: false + rebuild_rust_code: + description: "Rebuild Rust code" + type: boolean + default: false + required: false + rebuild_everything: + description: "If true, rebuild for all versions" + type: boolean + default: false + required: false + +env: + RUST_BACKTRACE: 1 + COPT: '-Werror' + +# TODO: move `check-*` and `files-changed` jobs to the "Caller" Workflow +# We should care about that as Github has limitations: +# - You can connect up to four levels of workflows +# - You can call a maximum of 20 unique reusable workflows from a single workflow file. +# https://docs.github.com/en/actions/sharing-automations/reusing-workflows#limitations +jobs: + build-pgxn: + if: | + (inputs.pg_versions != '[]' || inputs.rebuild_everything) && ( + contains(github.event.pull_request.labels.*.name, 'run-extra-build-macos') || + contains(github.event.pull_request.labels.*.name, 'run-extra-build-*') || + github.ref_name == 'main' + ) + timeout-minutes: 30 + runs-on: macos-15 + strategy: + matrix: + postgres-version: ${{ inputs.rebuild_everything && fromJson('["v14", "v15", "v16", "v17"]') || fromJSON(inputs.pg_versions) }} + env: + # Use release build only, to have less debug info around + # Hence keeping target/ (and general cache size) smaller + BUILD_TYPE: release + steps: + - name: Checkout main repo + uses: actions/checkout@v4 + + - name: Set pg ${{ matrix.postgres-version }} for caching + id: pg_rev + run: echo pg_rev=$(git rev-parse HEAD:vendor/postgres-${{ matrix.postgres-version }}) | tee -a "${GITHUB_OUTPUT}" + + - name: Cache postgres ${{ matrix.postgres-version }} build + id: cache_pg + uses: actions/cache@v4 + with: + path: pg_install/${{ matrix.postgres-version }} + key: v1-${{ runner.os }}-${{ runner.arch }}-${{ env.BUILD_TYPE }}-pg-${{ matrix.postgres-version }}-${{ steps.pg_rev.outputs.pg_rev }}-${{ hashFiles('Makefile') }} + + - name: Checkout submodule vendor/postgres-${{ matrix.postgres-version }} + if: steps.cache_pg.outputs.cache-hit != 'true' + run: | + git submodule init vendor/postgres-${{ matrix.postgres-version }} + git submodule update --depth 1 --recursive + + - name: Install build dependencies + if: steps.cache_pg.outputs.cache-hit != 'true' + run: | + brew install flex bison openssl protobuf icu4c + + - name: Set extra env for macOS + if: steps.cache_pg.outputs.cache-hit != 'true' + run: | + echo 'LDFLAGS=-L/usr/local/opt/openssl@3/lib' >> $GITHUB_ENV + echo 'CPPFLAGS=-I/usr/local/opt/openssl@3/include' >> $GITHUB_ENV + + - name: Build Postgres ${{ matrix.postgres-version }} + if: steps.cache_pg.outputs.cache-hit != 'true' + run: | + make postgres-${{ matrix.postgres-version }} -j$(sysctl -n hw.ncpu) + + - name: Build Neon Pg Ext ${{ matrix.postgres-version }} + if: steps.cache_pg.outputs.cache-hit != 'true' + run: | + make "neon-pg-ext-${{ matrix.postgres-version }}" -j$(sysctl -n hw.ncpu) + + - name: Get postgres headers ${{ matrix.postgres-version }} + if: steps.cache_pg.outputs.cache-hit != 'true' + run: | + make postgres-headers-${{ matrix.postgres-version }} -j$(sysctl -n hw.ncpu) + + build-walproposer-lib: + if: | + (inputs.pg_versions != '[]' || inputs.rebuild_everything) && ( + contains(github.event.pull_request.labels.*.name, 'run-extra-build-macos') || + contains(github.event.pull_request.labels.*.name, 'run-extra-build-*') || + github.ref_name == 'main' + ) + timeout-minutes: 30 + runs-on: macos-15 + needs: [build-pgxn] + env: + # Use release build only, to have less debug info around + # Hence keeping target/ (and general cache size) smaller + BUILD_TYPE: release + steps: + - name: Checkout main repo + uses: actions/checkout@v4 + + - name: Set pg v17 for caching + id: pg_rev + run: echo pg_rev=$(git rev-parse HEAD:vendor/postgres-v17) | tee -a "${GITHUB_OUTPUT}" + + - name: Cache postgres v17 build + id: cache_pg + uses: actions/cache@v4 + with: + path: pg_install/v17 + key: v1-${{ runner.os }}-${{ runner.arch }}-${{ env.BUILD_TYPE }}-pg-v17-${{ steps.pg_rev.outputs.pg_rev }}-${{ hashFiles('Makefile') }} + + - name: Cache walproposer-lib + id: cache_walproposer_lib + uses: actions/cache@v4 + with: + path: pg_install/build/walproposer-lib + key: v1-${{ runner.os }}-${{ runner.arch }}-${{ env.BUILD_TYPE }}-walproposer_lib-v17-${{ steps.pg_rev.outputs.pg_rev }}-${{ hashFiles('Makefile') }} + + - name: Checkout submodule vendor/postgres-v17 + if: steps.cache_walproposer_lib.outputs.cache-hit != 'true' + run: | + git submodule init vendor/postgres-v17 + git submodule update --depth 1 --recursive + + - name: Install build dependencies + if: steps.cache_walproposer_lib.outputs.cache-hit != 'true' + run: | + brew install flex bison openssl protobuf icu4c + + - name: Set extra env for macOS + if: steps.cache_walproposer_lib.outputs.cache-hit != 'true' + run: | + echo 'LDFLAGS=-L/usr/local/opt/openssl@3/lib' >> $GITHUB_ENV + echo 'CPPFLAGS=-I/usr/local/opt/openssl@3/include' >> $GITHUB_ENV + + - name: Build walproposer-lib (only for v17) + if: steps.cache_walproposer_lib.outputs.cache-hit != 'true' + run: + make walproposer-lib -j$(sysctl -n hw.ncpu) + + cargo-build: + if: | + (inputs.pg_versions != '[]' || inputs.rebuild_rust_code || inputs.rebuild_everything) && ( + contains(github.event.pull_request.labels.*.name, 'run-extra-build-macos') || + contains(github.event.pull_request.labels.*.name, 'run-extra-build-*') || + github.ref_name == 'main' + ) + timeout-minutes: 30 + runs-on: macos-15 + needs: [build-pgxn, build-walproposer-lib] + env: + # Use release build only, to have less debug info around + # Hence keeping target/ (and general cache size) smaller + BUILD_TYPE: release + steps: + - name: Checkout main repo + uses: actions/checkout@v4 + with: + submodules: true + + - name: Set pg v14 for caching + id: pg_rev_v14 + run: echo pg_rev=$(git rev-parse HEAD:vendor/postgres-v14) | tee -a "${GITHUB_OUTPUT}" + - name: Set pg v15 for caching + id: pg_rev_v15 + run: echo pg_rev=$(git rev-parse HEAD:vendor/postgres-v15) | tee -a "${GITHUB_OUTPUT}" + - name: Set pg v16 for caching + id: pg_rev_v16 + run: echo pg_rev=$(git rev-parse HEAD:vendor/postgres-v16) | tee -a "${GITHUB_OUTPUT}" + - name: Set pg v17 for caching + id: pg_rev_v17 + run: echo pg_rev=$(git rev-parse HEAD:vendor/postgres-v17) | tee -a "${GITHUB_OUTPUT}" + + - name: Cache postgres v14 build + id: cache_pg + uses: actions/cache@v4 + with: + path: pg_install/v14 + key: v1-${{ runner.os }}-${{ runner.arch }}-${{ env.BUILD_TYPE }}-pg-v14-${{ steps.pg_rev_v14.outputs.pg_rev }}-${{ hashFiles('Makefile') }} + - name: Cache postgres v15 build + id: cache_pg_v15 + uses: actions/cache@v4 + with: + path: pg_install/v15 + key: v1-${{ runner.os }}-${{ runner.arch }}-${{ env.BUILD_TYPE }}-pg-v15-${{ steps.pg_rev_v15.outputs.pg_rev }}-${{ hashFiles('Makefile') }} + - name: Cache postgres v16 build + id: cache_pg_v16 + uses: actions/cache@v4 + with: + path: pg_install/v16 + key: v1-${{ runner.os }}-${{ runner.arch }}-${{ env.BUILD_TYPE }}-pg-v16-${{ steps.pg_rev_v16.outputs.pg_rev }}-${{ hashFiles('Makefile') }} + - name: Cache postgres v17 build + id: cache_pg_v17 + uses: actions/cache@v4 + with: + path: pg_install/v17 + key: v1-${{ runner.os }}-${{ runner.arch }}-${{ env.BUILD_TYPE }}-pg-v17-${{ steps.pg_rev_v17.outputs.pg_rev }}-${{ hashFiles('Makefile') }} + + - name: Cache cargo deps (only for v17) + uses: actions/cache@v4 + with: + path: | + ~/.cargo/registry + !~/.cargo/registry/src + ~/.cargo/git + target + key: v1-${{ runner.os }}-${{ runner.arch }}-cargo-${{ hashFiles('./Cargo.lock') }}-${{ hashFiles('./rust-toolchain.toml') }}-rust + + - name: Cache walproposer-lib + id: cache_walproposer_lib + uses: actions/cache@v4 + with: + path: pg_install/build/walproposer-lib + key: v1-${{ runner.os }}-${{ runner.arch }}-${{ env.BUILD_TYPE }}-walproposer_lib-v17-${{ steps.pg_rev_v17.outputs.pg_rev }}-${{ hashFiles('Makefile') }} + + - name: Install build dependencies + run: | + brew install flex bison openssl protobuf icu4c + + - name: Set extra env for macOS + run: | + echo 'LDFLAGS=-L/usr/local/opt/openssl@3/lib' >> $GITHUB_ENV + echo 'CPPFLAGS=-I/usr/local/opt/openssl@3/include' >> $GITHUB_ENV + + - name: Run cargo build (only for v17) + run: PQ_LIB_DIR=$(pwd)/pg_install/v17/lib cargo build --all --release -j$(sysctl -n hw.ncpu) + + - name: Check that no warnings are produced (only for v17) + run: ./run_clippy.sh diff --git a/.github/workflows/neon_extra_builds.yml b/.github/workflows/neon_extra_builds.yml index 1f85c2e102..5b5910badf 100644 --- a/.github/workflows/neon_extra_builds.yml +++ b/.github/workflows/neon_extra_builds.yml @@ -31,19 +31,15 @@ jobs: uses: ./.github/workflows/build-build-tools-image.yml secrets: inherit - check-macos-build: - needs: [ check-permissions ] - if: | - contains(github.event.pull_request.labels.*.name, 'run-extra-build-macos') || - contains(github.event.pull_request.labels.*.name, 'run-extra-build-*') || - github.ref_name == 'main' - timeout-minutes: 90 - runs-on: macos-15 - - env: - # Use release build only, to have less debug info around - # Hence keeping target/ (and general cache size) smaller - BUILD_TYPE: release + files-changed: + name: Detect what files changed + runs-on: ubuntu-22.04 + timeout-minutes: 3 + outputs: + v17: ${{ steps.files_changed.outputs.v17 }} + postgres_changes: ${{ steps.postgres_changes.outputs.changes }} + rebuild_rust_code: ${{ steps.files_changed.outputs.rust_code }} + rebuild_everything: ${{ steps.files_changed.outputs.rebuild_neon_extra || steps.files_changed.outputs.rebuild_macos }} steps: - name: Checkout @@ -51,106 +47,45 @@ jobs: with: submodules: true - - name: Install macOS postgres dependencies - run: brew install flex bison openssl protobuf icu4c - - - name: Set pg 14 revision for caching - id: pg_v14_rev - run: echo pg_rev=$(git rev-parse HEAD:vendor/postgres-v14) >> $GITHUB_OUTPUT - - - name: Set pg 15 revision for caching - id: pg_v15_rev - run: echo pg_rev=$(git rev-parse HEAD:vendor/postgres-v15) >> $GITHUB_OUTPUT - - - name: Set pg 16 revision for caching - id: pg_v16_rev - run: echo pg_rev=$(git rev-parse HEAD:vendor/postgres-v16) >> $GITHUB_OUTPUT - - - name: Set pg 17 revision for caching - id: pg_v17_rev - run: echo pg_rev=$(git rev-parse HEAD:vendor/postgres-v17) >> $GITHUB_OUTPUT - - - name: Cache postgres v14 build - id: cache_pg_14 - uses: actions/cache@v4 + - name: Check for Postgres changes + uses: dorny/paths-filter@1441771bbfdd59dcd748680ee64ebd8faab1a242 #v3 + id: files_changed with: - path: pg_install/v14 - key: v1-${{ runner.os }}-${{ runner.arch }}-${{ env.BUILD_TYPE }}-pg-${{ steps.pg_v14_rev.outputs.pg_rev }}-${{ hashFiles('Makefile') }} + token: ${{ github.token }} + filters: .github/file-filters.yaml + base: ${{ github.event_name != 'pull_request' && (github.event.merge_group.base_ref || github.ref_name) || '' }} + ref: ${{ github.event_name != 'pull_request' && (github.event.merge_group.head_ref || github.ref) || '' }} - - name: Cache postgres v15 build - id: cache_pg_15 - uses: actions/cache@v4 - with: - path: pg_install/v15 - key: v1-${{ runner.os }}-${{ runner.arch }}-${{ env.BUILD_TYPE }}-pg-${{ steps.pg_v15_rev.outputs.pg_rev }}-${{ hashFiles('Makefile') }} - - - name: Cache postgres v16 build - id: cache_pg_16 - uses: actions/cache@v4 - with: - path: pg_install/v16 - key: v1-${{ runner.os }}-${{ runner.arch }}-${{ env.BUILD_TYPE }}-pg-${{ steps.pg_v16_rev.outputs.pg_rev }}-${{ hashFiles('Makefile') }} - - - name: Cache postgres v17 build - id: cache_pg_17 - uses: actions/cache@v4 - with: - path: pg_install/v17 - key: v1-${{ runner.os }}-${{ runner.arch }}-${{ env.BUILD_TYPE }}-pg-${{ steps.pg_v17_rev.outputs.pg_rev }}-${{ hashFiles('Makefile') }} - - - name: Set extra env for macOS + - name: Filter out only v-string for build matrix + id: postgres_changes run: | - echo 'LDFLAGS=-L/usr/local/opt/openssl@3/lib' >> $GITHUB_ENV - echo 'CPPFLAGS=-I/usr/local/opt/openssl@3/include' >> $GITHUB_ENV + v_strings_only_as_json_array=$(echo ${{ steps.files_changed.outputs.chnages }} | jq '.[]|select(test("v\\d+"))' | jq --slurp -c) + echo "changes=${v_strings_only_as_json_array}" | tee -a "${GITHUB_OUTPUT}" - - name: Cache cargo deps - uses: actions/cache@v4 - with: - path: | - ~/.cargo/registry - !~/.cargo/registry/src - ~/.cargo/git - target - key: v1-${{ runner.os }}-${{ runner.arch }}-cargo-${{ hashFiles('./Cargo.lock') }}-${{ hashFiles('./rust-toolchain.toml') }}-rust - - - name: Build postgres v14 - if: steps.cache_pg_14.outputs.cache-hit != 'true' - run: make postgres-v14 -j$(sysctl -n hw.ncpu) - - - name: Build postgres v15 - if: steps.cache_pg_15.outputs.cache-hit != 'true' - run: make postgres-v15 -j$(sysctl -n hw.ncpu) - - - name: Build postgres v16 - if: steps.cache_pg_16.outputs.cache-hit != 'true' - run: make postgres-v16 -j$(sysctl -n hw.ncpu) - - - name: Build postgres v17 - if: steps.cache_pg_17.outputs.cache-hit != 'true' - run: make postgres-v17 -j$(sysctl -n hw.ncpu) - - - name: Build neon extensions - run: make neon-pg-ext -j$(sysctl -n hw.ncpu) - - - name: Build walproposer-lib - run: make walproposer-lib -j$(sysctl -n hw.ncpu) - - - name: Run cargo build - run: PQ_LIB_DIR=$(pwd)/pg_install/v16/lib cargo build --all --release - - - name: Check that no warnings are produced - run: ./run_clippy.sh + check-macos-build: + needs: [ check-permissions, files-changed ] + if: | + contains(github.event.pull_request.labels.*.name, 'run-extra-build-macos') || + contains(github.event.pull_request.labels.*.name, 'run-extra-build-*') || + github.ref_name == 'main' + uses: ./.github/workflows/build-macos.yml + with: + pg_versions: ${{ needs.files-changed.outputs.postgres_changes }} + rebuild_rust_code: ${{ needs.files-changed.outputs.rebuild_rust_code }} + rebuild_everything: ${{ fromJson(needs.files-changed.outputs.rebuild_everything) }} gather-rust-build-stats: - needs: [ check-permissions, build-build-tools-image ] + needs: [ check-permissions, build-build-tools-image, files-changed ] permissions: id-token: write # aws-actions/configure-aws-credentials statuses: write contents: write if: | - contains(github.event.pull_request.labels.*.name, 'run-extra-build-stats') || - contains(github.event.pull_request.labels.*.name, 'run-extra-build-*') || - github.ref_name == 'main' + (needs.files-changed.outputs.v17 == 'true' || needs.files-changed.outputs.rebuild_everything == 'true') && ( + contains(github.event.pull_request.labels.*.name, 'run-extra-build-stats') || + contains(github.event.pull_request.labels.*.name, 'run-extra-build-*') || + github.ref_name == 'main' + ) runs-on: [ self-hosted, large ] container: image: ${{ needs.build-build-tools-image.outputs.image }}-bookworm From 237dae71a1bd00d1611fb2298b2e3cb13883155b Mon Sep 17 00:00:00 2001 From: Erik Grinaker Date: Tue, 7 Jan 2025 23:49:00 +0100 Subject: [PATCH 004/111] Revert "pageserver,safekeeper: disable heap profiling (#10268)" (#10303) This reverts commit b33299dc37d9269fe55bd3256b7a4a72c129b81c. Heap profiles weren't the culprit after all. Touches #10225. --- pageserver/src/bin/pageserver.rs | 10 ++++------ safekeeper/src/bin/safekeeper.rs | 10 ++++------ 2 files changed, 8 insertions(+), 12 deletions(-) diff --git a/pageserver/src/bin/pageserver.rs b/pageserver/src/bin/pageserver.rs index b92ff4ebf9..567a69da3b 100644 --- a/pageserver/src/bin/pageserver.rs +++ b/pageserver/src/bin/pageserver.rs @@ -53,12 +53,10 @@ project_build_tag!(BUILD_TAG); #[global_allocator] static GLOBAL: tikv_jemallocator::Jemalloc = tikv_jemallocator::Jemalloc; -// Configure jemalloc to sample allocations for profiles every 1 MB (1 << 20). -// TODO: disabled because concurrent CPU profiles cause seg faults. See: -// https://github.com/neondatabase/neon/issues/10225. -//#[allow(non_upper_case_globals)] -//#[export_name = "malloc_conf"] -//pub static malloc_conf: &[u8] = b"prof:true,prof_active:true,lg_prof_sample:20\0"; +/// Configure jemalloc to sample allocations for profiles every 1 MB (1 << 20). +#[allow(non_upper_case_globals)] +#[export_name = "malloc_conf"] +pub static malloc_conf: &[u8] = b"prof:true,prof_active:true,lg_prof_sample:20\0"; const PID_FILE_NAME: &str = "pageserver.pid"; diff --git a/safekeeper/src/bin/safekeeper.rs b/safekeeper/src/bin/safekeeper.rs index e0ba38d638..13f6e34575 100644 --- a/safekeeper/src/bin/safekeeper.rs +++ b/safekeeper/src/bin/safekeeper.rs @@ -51,12 +51,10 @@ use utils::{ #[global_allocator] static GLOBAL: tikv_jemallocator::Jemalloc = tikv_jemallocator::Jemalloc; -// Configure jemalloc to sample allocations for profiles every 1 MB (1 << 20). -// TODO: disabled because concurrent CPU profiles cause seg faults. See: -// https://github.com/neondatabase/neon/issues/10225. -//#[allow(non_upper_case_globals)] -//#[export_name = "malloc_conf"] -//pub static malloc_conf: &[u8] = b"prof:true,prof_active:true,lg_prof_sample:20\0"; +/// Configure jemalloc to sample allocations for profiles every 1 MB (1 << 20). +#[allow(non_upper_case_globals)] +#[export_name = "malloc_conf"] +pub static malloc_conf: &[u8] = b"prof:true,prof_active:true,lg_prof_sample:20\0"; const PID_FILE_NAME: &str = "safekeeper.pid"; const ID_FILE_NAME: &str = "safekeeper.id"; From 5c76e2a983295f3123631b0178309a942f584596 Mon Sep 17 00:00:00 2001 From: "Alex Chi Z." <4198311+skyzh@users.noreply.github.com> Date: Tue, 7 Jan 2025 18:24:17 -0500 Subject: [PATCH 005/111] fix(storage-scrubber): ignore errors if index_part is not consistent (#10304) ## Problem Consider the pageserver is doing the following sequence of operations: * upload X files * update index_part to add X and remove Y * delete Y files When storage scrubber obtains the initial timeline snapshot before "update index_part" (that is the old version that contains Y but not X), and then obtains the index_part file after it gets updated, it will report all Y files are missing. ## Summary of changes Do not report layer file missing if index_part listed and downloaded are not the same (i.e. different last_modified times) Signed-off-by: Alex Chi Z --- storage_scrubber/src/checks.rs | 24 +++++++++++++------ storage_scrubber/src/lib.rs | 7 +++--- .../src/pageserver_physical_gc.rs | 6 ++++- .../src/scan_pageserver_metadata.rs | 10 ++++++-- storage_scrubber/src/tenant_snapshot.rs | 2 ++ 5 files changed, 36 insertions(+), 13 deletions(-) diff --git a/storage_scrubber/src/checks.rs b/storage_scrubber/src/checks.rs index 32c86052ef..b42709868b 100644 --- a/storage_scrubber/src/checks.rs +++ b/storage_scrubber/src/checks.rs @@ -1,4 +1,5 @@ use std::collections::{HashMap, HashSet}; +use std::time::SystemTime; use itertools::Itertools; use pageserver::tenant::checks::check_valid_layermap; @@ -88,9 +89,14 @@ pub(crate) async fn branch_cleanup_and_check_errors( match s3_data.blob_data { BlobDataParseResult::Parsed { index_part, - index_part_generation: _index_part_generation, - s3_layers: _s3_layers, + index_part_generation: _, + s3_layers: _, + index_part_last_modified_time, + index_part_snapshot_time, } => { + // Ignore missing file error if index_part downloaded is different from the one when listing the layer files. + let ignore_error = index_part_snapshot_time < index_part_last_modified_time + && !cfg!(debug_assertions); if !IndexPart::KNOWN_VERSIONS.contains(&index_part.version()) { result .errors @@ -171,7 +177,7 @@ pub(crate) async fn branch_cleanup_and_check_errors( is_l0, ); - if is_l0 { + if is_l0 || ignore_error { result.warnings.push(msg); } else { result.errors.push(msg); @@ -308,6 +314,8 @@ pub(crate) enum BlobDataParseResult { Parsed { index_part: Box, index_part_generation: Generation, + index_part_last_modified_time: SystemTime, + index_part_snapshot_time: SystemTime, s3_layers: HashSet<(LayerName, Generation)>, }, /// The remains of an uncleanly deleted Timeline or aborted timeline creation(e.g. an initdb archive only, or some layer without an index) @@ -484,9 +492,9 @@ async fn list_timeline_blobs_impl( } if let Some(index_part_object_key) = index_part_object.as_ref() { - let index_part_bytes = + let (index_part_bytes, index_part_last_modified_time) = match download_object_with_retries(remote_client, &index_part_object_key.key).await { - Ok(index_part_bytes) => index_part_bytes, + Ok(data) => data, Err(e) => { // It is possible that the branch gets deleted in-between we list the objects // and we download the index part file. @@ -500,7 +508,7 @@ async fn list_timeline_blobs_impl( )); } }; - + let index_part_snapshot_time = index_part_object_key.last_modified; match serde_json::from_slice(&index_part_bytes) { Ok(index_part) => { return Ok(ListTimelineBlobsResult::Ready(RemoteTimelineBlobData { @@ -508,6 +516,8 @@ async fn list_timeline_blobs_impl( index_part: Box::new(index_part), index_part_generation, s3_layers, + index_part_last_modified_time, + index_part_snapshot_time, }, unused_index_keys: index_part_keys, unknown_keys, @@ -625,7 +635,7 @@ pub(crate) async fn list_tenant_manifests( let manifest_bytes = match download_object_with_retries(remote_client, &latest_listing_object.key).await { - Ok(bytes) => bytes, + Ok((bytes, _)) => bytes, Err(e) => { // It is possible that the tenant gets deleted in-between we list the objects // and we download the manifest file. diff --git a/storage_scrubber/src/lib.rs b/storage_scrubber/src/lib.rs index be526daaf0..224235098c 100644 --- a/storage_scrubber/src/lib.rs +++ b/storage_scrubber/src/lib.rs @@ -13,7 +13,7 @@ pub mod tenant_snapshot; use std::env; use std::fmt::Display; use std::sync::Arc; -use std::time::Duration; +use std::time::{Duration, SystemTime}; use anyhow::Context; use aws_config::retry::{RetryConfigBuilder, RetryMode}; @@ -509,10 +509,11 @@ async fn list_objects_with_retries( panic!("MAX_RETRIES is not allowed to be 0"); } +/// Returns content, last modified time async fn download_object_with_retries( remote_client: &GenericRemoteStorage, key: &RemotePath, -) -> anyhow::Result> { +) -> anyhow::Result<(Vec, SystemTime)> { let cancel = CancellationToken::new(); for trial in 0..MAX_RETRIES { let mut buf = Vec::new(); @@ -535,7 +536,7 @@ async fn download_object_with_retries( { Ok(bytes_read) => { tracing::debug!("Downloaded {bytes_read} bytes for object {key}"); - return Ok(buf); + return Ok((buf, download.last_modified)); } Err(e) => { error!("Failed to stream object body for key {key}: {e}"); diff --git a/storage_scrubber/src/pageserver_physical_gc.rs b/storage_scrubber/src/pageserver_physical_gc.rs index d19b8a5f91..a997373375 100644 --- a/storage_scrubber/src/pageserver_physical_gc.rs +++ b/storage_scrubber/src/pageserver_physical_gc.rs @@ -450,6 +450,8 @@ async fn gc_ancestor( index_part: _, index_part_generation: _, s3_layers, + index_part_last_modified_time: _, + index_part_snapshot_time: _, } => s3_layers, BlobDataParseResult::Relic => { // Post-deletion tenant location: don't try and GC it. @@ -586,7 +588,9 @@ async fn gc_timeline( BlobDataParseResult::Parsed { index_part, index_part_generation, - s3_layers: _s3_layers, + s3_layers: _, + index_part_last_modified_time: _, + index_part_snapshot_time: _, } => (index_part, *index_part_generation, data.unused_index_keys), BlobDataParseResult::Relic => { // Post-deletion tenant location: don't try and GC it. diff --git a/storage_scrubber/src/scan_pageserver_metadata.rs b/storage_scrubber/src/scan_pageserver_metadata.rs index c8de6e46b3..a31fb5b242 100644 --- a/storage_scrubber/src/scan_pageserver_metadata.rs +++ b/storage_scrubber/src/scan_pageserver_metadata.rs @@ -47,6 +47,8 @@ impl MetadataSummary { index_part, index_part_generation: _, s3_layers: _, + index_part_last_modified_time: _, + index_part_snapshot_time: _, } = &data.blob_data { *self @@ -195,7 +197,9 @@ pub async fn scan_pageserver_metadata( if let BlobDataParseResult::Parsed { index_part, index_part_generation, - s3_layers: _s3_layers, + s3_layers: _, + index_part_last_modified_time: _, + index_part_snapshot_time: _, } = &data.blob_data { if index_part.deleted_at.is_some() { @@ -318,9 +322,11 @@ pub async fn scan_pageserver_metadata( match &data.blob_data { BlobDataParseResult::Parsed { - index_part: _index_part, + index_part: _, index_part_generation: _index_part_generation, s3_layers, + index_part_last_modified_time: _, + index_part_snapshot_time: _, } => { tenant_objects.push(ttid, s3_layers.clone()); } diff --git a/storage_scrubber/src/tenant_snapshot.rs b/storage_scrubber/src/tenant_snapshot.rs index 39e0b5c9b4..60e79fb859 100644 --- a/storage_scrubber/src/tenant_snapshot.rs +++ b/storage_scrubber/src/tenant_snapshot.rs @@ -268,6 +268,8 @@ impl SnapshotDownloader { index_part, index_part_generation, s3_layers: _, + index_part_last_modified_time: _, + index_part_snapshot_time: _, } => { self.download_timeline( ttid, From dc284247a5b0d4fd442868c9dc555dd0ab50c0c3 Mon Sep 17 00:00:00 2001 From: Vlad Lazar Date: Wed, 8 Jan 2025 10:26:53 +0000 Subject: [PATCH 006/111] storage_controller: fix node flap detach race (#10298) ## Problem The observed state removal may race with the inline updates of the observed state done from `Service::node_activate_reconcile`. This was intended to work as follows: 1. Detaches while the node is unavailable remove the entry from the observed state. 2. `Service::node_activate_reconcile` diffs the locations returned by the pageserver with the observed state and detaches in-line when required. ## Summary of changes This PR removes step (1) and lets background reconciliations deal with the mismatch between the intent and observed state. A follow up will attempt to remove `Service::node_activate_reconcile` altogether. Closes https://github.com/neondatabase/neon/issues/10253 --- storage_controller/src/reconciler.rs | 16 ++- storage_controller/src/service.rs | 4 + test_runner/fixtures/neon_fixtures.py | 7 +- .../regress/test_storage_controller.py | 122 +++++++++++++++++- 4 files changed, 139 insertions(+), 10 deletions(-) diff --git a/storage_controller/src/reconciler.rs b/storage_controller/src/reconciler.rs index 475f91eff4..e0a854fff7 100644 --- a/storage_controller/src/reconciler.rs +++ b/storage_controller/src/reconciler.rs @@ -14,7 +14,6 @@ use std::sync::Arc; use std::time::{Duration, Instant}; use tokio_util::sync::CancellationToken; use utils::backoff::exponential_backoff; -use utils::failpoint_support; use utils::generation::Generation; use utils::id::{NodeId, TimelineId}; use utils::lsn::Lsn; @@ -212,11 +211,12 @@ impl Reconciler { lazy: bool, ) -> Result<(), ReconcileError> { if !node.is_available() && config.mode == LocationConfigMode::Detached { - // Attempts to detach from offline nodes may be imitated without doing I/O: a node which is offline - // will get fully reconciled wrt the shard's intent state when it is reactivated, irrespective of - // what we put into `observed`, in [`crate::service::Service::node_activate_reconcile`] - tracing::info!("Node {node} is unavailable during detach: proceeding anyway, it will be detached on next activation"); - self.observed.locations.remove(&node.get_id()); + // [`crate::service::Service::node_activate_reconcile`] will update the observed state + // when the node comes back online. At that point, the intent and observed states will + // be mismatched and a background reconciliation will detach. + tracing::info!( + "Node {node} is unavailable during detach: proceeding anyway, it will be detached via background reconciliation" + ); return Ok(()); } @@ -749,6 +749,8 @@ impl Reconciler { }; if increment_generation { + pausable_failpoint!("reconciler-pre-increment-generation"); + let generation = self .persistence .increment_generation(self.tenant_shard_id, node.get_id()) @@ -824,7 +826,7 @@ impl Reconciler { .handle_detach(self.tenant_shard_id, self.shard.stripe_size); } - failpoint_support::sleep_millis_async!("sleep-on-reconcile-epilogue"); + pausable_failpoint!("reconciler-epilogue"); Ok(()) } diff --git a/storage_controller/src/service.rs b/storage_controller/src/service.rs index 222cb9fdd4..359fcb3288 100644 --- a/storage_controller/src/service.rs +++ b/storage_controller/src/service.rs @@ -83,6 +83,7 @@ use utils::{ generation::Generation, http::error::ApiError, id::{NodeId, TenantId, TimelineId}, + pausable_failpoint, sync::gate::Gate, }; @@ -1024,6 +1025,8 @@ impl Service { ) .await; + pausable_failpoint!("heartbeat-pre-node-state-configure"); + // This is the code path for geniune availability transitions (i.e node // goes unavailable and/or comes back online). let res = self @@ -2492,6 +2495,7 @@ impl Service { // Persist updates // Ordering: write to the database before applying changes in-memory, so that // we will not appear time-travel backwards on a restart. + let mut schedule_context = ScheduleContext::default(); for ShardUpdate { tenant_shard_id, diff --git a/test_runner/fixtures/neon_fixtures.py b/test_runner/fixtures/neon_fixtures.py index 8fd9eec8ce..00fdda2998 100644 --- a/test_runner/fixtures/neon_fixtures.py +++ b/test_runner/fixtures/neon_fixtures.py @@ -2521,6 +2521,7 @@ class NeonPageserver(PgProtocol, LogUtils): self, extra_env_vars: dict[str, str] | None = None, timeout_in_seconds: int | None = None, + await_active: bool = True, ) -> Self: """ Start the page server. @@ -2547,8 +2548,10 @@ class NeonPageserver(PgProtocol, LogUtils): ) self.running = True - if self.env.storage_controller.running and self.env.storage_controller.node_registered( - self.id + if ( + await_active + and self.env.storage_controller.running + and self.env.storage_controller.node_registered(self.id) ): self.env.storage_controller.poll_node_status( self.id, PageserverAvailability.ACTIVE, None, max_attempts=200, backoff=0.1 diff --git a/test_runner/regress/test_storage_controller.py b/test_runner/regress/test_storage_controller.py index 7062c35e05..973d0cdf82 100644 --- a/test_runner/regress/test_storage_controller.py +++ b/test_runner/regress/test_storage_controller.py @@ -17,6 +17,7 @@ from fixtures.compute_reconfigure import ComputeReconfigure from fixtures.log_helper import log from fixtures.neon_fixtures import ( DEFAULT_AZ_ID, + LogCursor, NeonEnv, NeonEnvBuilder, NeonPageserver, @@ -2406,7 +2407,14 @@ def test_storage_controller_step_down(neon_env_builder: NeonEnvBuilder): env.storage_controller.tenant_create(tid) env.storage_controller.reconcile_until_idle() - env.storage_controller.configure_failpoints(("sleep-on-reconcile-epilogue", "return(10000)")) + env.storage_controller.configure_failpoints(("reconciler-epilogue", "pause")) + + def unpause_failpoint(): + time.sleep(2) + env.storage_controller.configure_failpoints(("reconciler-epilogue", "off")) + + thread = threading.Thread(target=unpause_failpoint) + thread.start() # Make a change to the tenant config to trigger a slow reconcile virtual_ps_http = PageserverHttpClient(env.storage_controller_port, lambda: True) @@ -2421,6 +2429,8 @@ def test_storage_controller_step_down(neon_env_builder: NeonEnvBuilder): observed_state = env.storage_controller.step_down() log.info(f"Storage controller stepped down with {observed_state=}") + thread.join() + # Validate that we waited for the slow reconcile to complete # and updated the observed state in the storcon before stepping down. node_id = str(env.pageserver.id) @@ -3294,3 +3304,113 @@ def test_storage_controller_detached_stopped( # Confirm the detach happened assert env.pageserver.http_client().tenant_list_locations()["tenant_shards"] == [] + + +@run_only_on_default_postgres("Postgres version makes no difference here") +def test_storage_controller_node_flap_detach_race( + neon_env_builder: NeonEnvBuilder, +): + """ + Reproducer for https://github.com/neondatabase/neon/issues/10253. + + When a node's availability flaps, the reconciliations spawned by the node + going offline may race with the reconciliation done when then node comes + back online. + """ + neon_env_builder.num_pageservers = 4 + + env = neon_env_builder.init_configs() + env.start() + + tenant_id = TenantId.generate() + env.storage_controller.tenant_create( + tenant_id, + shard_count=2, + ) + env.storage_controller.reconcile_until_idle() + + stopped_nodes = [s["node_id"] for s in env.storage_controller.locate(tenant_id)] + + def has_hit_failpoint(failpoint: str, offset: LogCursor | None = None) -> LogCursor: + res = env.storage_controller.log_contains(f"at failpoint {failpoint}", offset=offset) + assert res + return res[1] + + # Stop the nodes which host attached shards. + # This will trigger reconciliations which pause before incrmenenting the generation, + # and, more importantly, updating the `generation_pageserver` of the shards. + env.storage_controller.configure_failpoints(("reconciler-pre-increment-generation", "pause")) + for node_id in stopped_nodes: + env.get_pageserver(node_id).stop(immediate=True) + + def failure_handled() -> LogCursor: + stop_offset = None + + for node_id in stopped_nodes: + res = env.storage_controller.log_contains(f"node {node_id} going offline") + assert res + stop_offset = res[1] + + assert stop_offset + return stop_offset + + offset = wait_until(failure_handled) + + # Now restart the nodes and make them pause before marking themselves as available + # or running the activation reconciliation. + env.storage_controller.configure_failpoints(("heartbeat-pre-node-state-configure", "pause")) + + for node_id in stopped_nodes: + env.get_pageserver(node_id).start(await_active=False) + + offset = wait_until( + lambda: has_hit_failpoint("heartbeat-pre-node-state-configure", offset=offset) + ) + + # The nodes have restarted and are waiting to perform activaction reconciliation. + # Unpause the initial reconciliation triggered by the nodes going offline. + # It will attempt to detach from the old location, but notice that the old location + # is not yet available, and then stop before processing the results of the reconciliation. + env.storage_controller.configure_failpoints(("reconciler-epilogue", "pause")) + env.storage_controller.configure_failpoints(("reconciler-pre-increment-generation", "off")) + + offset = wait_until(lambda: has_hit_failpoint("reconciler-epilogue", offset=offset)) + + # Let the nodes perform activation reconciliation while still holding up processing the result + # from the initial reconcile triggered by going offline. + env.storage_controller.configure_failpoints(("heartbeat-pre-node-state-configure", "off")) + + def activate_reconciliation_done(): + for node_id in stopped_nodes: + assert env.storage_controller.log_contains( + f"Node {node_id} transition to active", offset=offset + ) + + wait_until(activate_reconciliation_done) + + # Finally, allow the initial reconcile to finish up. + env.storage_controller.configure_failpoints(("reconciler-epilogue", "off")) + + # Give things a chance to settle and validate that no stale locations exist + env.storage_controller.reconcile_until_idle() + + def validate_locations(): + shard_locations = defaultdict(list) + for ps in env.pageservers: + locations = ps.http_client().tenant_list_locations()["tenant_shards"] + for loc in locations: + shard_locations[loc[0]].append( + {"generation": loc[1]["generation"], "mode": loc[1]["mode"], "node": ps.id} + ) + + log.info(f"Shard locations: {shard_locations}") + + attached_locations = { + k: list(filter(lambda loc: loc["mode"] == "AttachedSingle", v)) + for k, v in shard_locations.items() + } + + for shard, locs in attached_locations.items(): + assert len(locs) == 1, f"{shard} has {len(locs)} attached locations" + + wait_until(validate_locations, timeout=10) From 68d8acfd058b7b2d0deb041d14252f17d50ad05f Mon Sep 17 00:00:00 2001 From: John Spray Date: Wed, 8 Jan 2025 18:12:09 +0000 Subject: [PATCH 007/111] storage controller: don't hold detached tenants in memory (#10264) ## Problem Typical deployments of neon have some tenants that stay in use continuously, and a background churning population of tenants that are created and then fall idle, and are configured to Detached state. Currently, this churn of short lived tenants results in an ever-increasing memory footprint. Closes: https://github.com/neondatabase/neon/issues/9712 ## Summary of changes - At startup, filter to only load shards that don't have Detached policy - In process_result, check if a tenant's shards are all Detached and observed=={}, and if so drop them from memory - In tenant_location_conf and other tenant mutators, load the tenants' shards on-demand if they are not present --- storage_controller/src/id_lock_map.rs | 8 + storage_controller/src/persistence.rs | 34 ++- storage_controller/src/service.rs | 195 +++++++++++++++--- storage_controller/src/tenant_shard.rs | 4 + .../regress/test_storage_controller.py | 105 +++++++++- 5 files changed, 318 insertions(+), 28 deletions(-) diff --git a/storage_controller/src/id_lock_map.rs b/storage_controller/src/id_lock_map.rs index fcd3eb57e2..2d8b674f86 100644 --- a/storage_controller/src/id_lock_map.rs +++ b/storage_controller/src/id_lock_map.rs @@ -112,6 +112,14 @@ where } } + pub(crate) fn try_exclusive(&self, key: T, operation: I) -> Option> { + let mut locked = self.entities.lock().unwrap(); + let entry = locked.entry(key).or_default().clone(); + let mut guard = TracingExclusiveGuard::new(entry.try_write_owned().ok()?); + *guard.guard = Some(operation); + Some(guard) + } + /// Rather than building a lock guard that re-takes the [`Self::entities`] lock, we just do /// periodic housekeeping to avoid the map growing indefinitely pub(crate) fn housekeeping(&self) { diff --git a/storage_controller/src/persistence.rs b/storage_controller/src/persistence.rs index cc377e606e..c5eb106f24 100644 --- a/storage_controller/src/persistence.rs +++ b/storage_controller/src/persistence.rs @@ -97,6 +97,7 @@ pub(crate) enum DatabaseOperation { TenantGenerations, ShardGenerations, ListTenantShards, + LoadTenant, InsertTenantShards, UpdateTenantShard, DeleteTenant, @@ -330,11 +331,40 @@ impl Persistence { /// At startup, load the high level state for shards, such as their config + policy. This will /// be enriched at runtime with state discovered on pageservers. - pub(crate) async fn list_tenant_shards(&self) -> DatabaseResult> { + /// + /// We exclude shards configured to be detached. During startup, if we see any attached locations + /// for such shards, they will automatically be detached as 'orphans'. + pub(crate) async fn load_active_tenant_shards( + &self, + ) -> DatabaseResult> { + use crate::schema::tenant_shards::dsl::*; self.with_measured_conn( DatabaseOperation::ListTenantShards, move |conn| -> DatabaseResult<_> { - Ok(crate::schema::tenant_shards::table.load::(conn)?) + let query = tenant_shards.filter( + placement_policy.ne(serde_json::to_string(&PlacementPolicy::Detached).unwrap()), + ); + let result = query.load::(conn)?; + + Ok(result) + }, + ) + .await + } + + /// When restoring a previously detached tenant into memory, load it from the database + pub(crate) async fn load_tenant( + &self, + filter_tenant_id: TenantId, + ) -> DatabaseResult> { + use crate::schema::tenant_shards::dsl::*; + self.with_measured_conn( + DatabaseOperation::LoadTenant, + move |conn| -> DatabaseResult<_> { + let query = tenant_shards.filter(tenant_id.eq(filter_tenant_id.to_string())); + let result = query.load::(conn)?; + + Ok(result) }, ) .await diff --git a/storage_controller/src/service.rs b/storage_controller/src/service.rs index 359fcb3288..fd4ee7fd10 100644 --- a/storage_controller/src/service.rs +++ b/storage_controller/src/service.rs @@ -155,6 +155,7 @@ enum TenantOperations { TimelineArchivalConfig, TimelineDetachAncestor, TimelineGcBlockUnblock, + DropDetached, } #[derive(Clone, strum_macros::Display)] @@ -416,8 +417,8 @@ pub struct Service { /// Queue of tenants who are waiting for concurrency limits to permit them to reconcile /// Send into this queue to promptly attempt to reconcile this shard next time units are available. /// - /// Note that this state logically lives inside ServiceInner, but carrying Sender here makes the code simpler - /// by avoiding needing a &mut ref to something inside the ServiceInner. This could be optimized to + /// Note that this state logically lives inside ServiceState, but carrying Sender here makes the code simpler + /// by avoiding needing a &mut ref to something inside the ServiceState. This could be optimized to /// use a VecDeque instead of a channel to reduce synchronization overhead, at the cost of some code complexity. delayed_reconcile_tx: tokio::sync::mpsc::Sender, @@ -1165,6 +1166,20 @@ impl Service { } } + // If we just finished detaching all shards for a tenant, it might be time to drop it from memory. + if tenant.policy == PlacementPolicy::Detached { + // We may only drop a tenant from memory while holding the exclusive lock on the tenant ID: this protects us + // from concurrent execution wrt a request handler that might expect the tenant to remain in memory for the + // duration of the request. + let guard = self.tenant_op_locks.try_exclusive( + tenant.tenant_shard_id.tenant_id, + TenantOperations::DropDetached, + ); + if let Some(guard) = guard { + self.maybe_drop_tenant(tenant.tenant_shard_id.tenant_id, &mut locked, &guard); + } + } + // Maybe some other work can proceed now that this job finished. if self.reconciler_concurrency.available_permits() > 0 { while let Ok(tenant_shard_id) = locked.delayed_reconcile_rx.try_recv() { @@ -1294,7 +1309,7 @@ impl Service { .set(nodes.len() as i64); tracing::info!("Loading shards from database..."); - let mut tenant_shard_persistence = persistence.list_tenant_shards().await?; + let mut tenant_shard_persistence = persistence.load_active_tenant_shards().await?; tracing::info!( "Loaded {} shards from database.", tenant_shard_persistence.len() @@ -1546,8 +1561,14 @@ impl Service { // the pageserver API (not via this service), we will auto-create any missing tenant // shards with default state. let insert = { - let locked = self.inner.write().unwrap(); - !locked.tenants.contains_key(&attach_req.tenant_shard_id) + match self + .maybe_load_tenant(attach_req.tenant_shard_id.tenant_id, &_tenant_lock) + .await + { + Ok(_) => false, + Err(ApiError::NotFound(_)) => true, + Err(e) => return Err(e.into()), + } }; if insert { @@ -2439,6 +2460,99 @@ impl Service { } } + /// For APIs that might act on tenants with [`PlacementPolicy::Detached`], first check if + /// the tenant is present in memory. If not, load it from the database. If it is found + /// in neither location, return a NotFound error. + /// + /// Caller must demonstrate they hold a lock guard, as otherwise two callers might try and load + /// it at the same time, or we might race with [`Self::maybe_drop_tenant`] + async fn maybe_load_tenant( + &self, + tenant_id: TenantId, + _guard: &TracingExclusiveGuard, + ) -> Result<(), ApiError> { + let present_in_memory = { + let locked = self.inner.read().unwrap(); + locked + .tenants + .range(TenantShardId::tenant_range(tenant_id)) + .next() + .is_some() + }; + + if present_in_memory { + return Ok(()); + } + + let tenant_shards = self.persistence.load_tenant(tenant_id).await?; + if tenant_shards.is_empty() { + return Err(ApiError::NotFound( + anyhow::anyhow!("Tenant {} not found", tenant_id).into(), + )); + } + + // TODO: choose a fresh AZ to use for this tenant when un-detaching: there definitely isn't a running + // compute, so no benefit to making AZ sticky across detaches. + + let mut locked = self.inner.write().unwrap(); + tracing::info!( + "Loaded {} shards for tenant {}", + tenant_shards.len(), + tenant_id + ); + + locked.tenants.extend(tenant_shards.into_iter().map(|p| { + let intent = IntentState::new(); + let shard = + TenantShard::from_persistent(p, intent).expect("Corrupt shard row in database"); + + // Sanity check: when loading on-demand, we should always be loaded something Detached + debug_assert!(shard.policy == PlacementPolicy::Detached); + if shard.policy != PlacementPolicy::Detached { + tracing::error!( + "Tenant shard {} loaded on-demand, but has non-Detached policy {:?}", + shard.tenant_shard_id, + shard.policy + ); + } + + (shard.tenant_shard_id, shard) + })); + + Ok(()) + } + + /// If all shards for a tenant are detached, and in a fully quiescent state (no observed locations on pageservers), + /// and have no reconciler running, then we can drop the tenant from memory. It will be reloaded on-demand + /// if we are asked to attach it again (see [`Self::maybe_load_tenant`]). + /// + /// Caller must demonstrate they hold a lock guard, as otherwise it is unsafe to drop a tenant from + /// memory while some other function might assume it continues to exist while not holding the lock on Self::inner. + fn maybe_drop_tenant( + &self, + tenant_id: TenantId, + locked: &mut std::sync::RwLockWriteGuard, + _guard: &TracingExclusiveGuard, + ) { + let mut tenant_shards = locked.tenants.range(TenantShardId::tenant_range(tenant_id)); + if tenant_shards.all(|(_id, shard)| { + shard.policy == PlacementPolicy::Detached + && shard.reconciler.is_none() + && shard.observed.is_empty() + }) { + let keys = locked + .tenants + .range(TenantShardId::tenant_range(tenant_id)) + .map(|(id, _)| id) + .copied() + .collect::>(); + for key in keys { + tracing::info!("Dropping detached tenant shard {} from memory", key); + locked.tenants.remove(&key); + } + } + } + /// This API is used by the cloud control plane to migrate unsharded tenants that it created /// directly with pageservers into this service. /// @@ -2465,14 +2579,26 @@ impl Service { ) .await; - if !tenant_shard_id.is_unsharded() { + let tenant_id = if !tenant_shard_id.is_unsharded() { return Err(ApiError::BadRequest(anyhow::anyhow!( "This API is for importing single-sharded or unsharded tenants" ))); - } + } else { + tenant_shard_id.tenant_id + }; + + // In case we are waking up a Detached tenant + match self.maybe_load_tenant(tenant_id, &_tenant_lock).await { + Ok(()) | Err(ApiError::NotFound(_)) => { + // This is a creation or an update + } + Err(e) => { + return Err(e); + } + }; // First check if this is a creation or an update - let create_or_update = self.tenant_location_config_prepare(tenant_shard_id.tenant_id, req); + let create_or_update = self.tenant_location_config_prepare(tenant_id, req); let mut result = TenantLocationConfigResponse { shards: Vec::new(), @@ -2600,6 +2726,8 @@ impl Service { let tenant_id = req.tenant_id; let patch = req.config; + self.maybe_load_tenant(tenant_id, &_tenant_lock).await?; + let base = { let locked = self.inner.read().unwrap(); let shards = locked @@ -2644,19 +2772,7 @@ impl Service { ) .await; - let tenant_exists = { - let locked = self.inner.read().unwrap(); - let mut r = locked - .tenants - .range(TenantShardId::tenant_range(req.tenant_id)); - r.next().is_some() - }; - - if !tenant_exists { - return Err(ApiError::NotFound( - anyhow::anyhow!("Tenant {} not found", req.tenant_id).into(), - )); - } + self.maybe_load_tenant(req.tenant_id, &_tenant_lock).await?; self.set_tenant_config_and_reconcile(req.tenant_id, req.config) .await @@ -2949,6 +3065,8 @@ impl Service { let _tenant_lock = trace_exclusive_lock(&self.tenant_op_locks, tenant_id, TenantOperations::Delete).await; + self.maybe_load_tenant(tenant_id, &_tenant_lock).await?; + // Detach all shards. This also deletes local pageserver shard data. let (detach_waiters, node) = { let mut detach_waiters = Vec::new(); @@ -3068,6 +3186,8 @@ impl Service { ) .await; + self.maybe_load_tenant(tenant_id, &_tenant_lock).await?; + failpoint_support::sleep_millis_async!("tenant-update-policy-exclusive-lock"); let TenantPolicyRequest { @@ -5150,11 +5270,13 @@ impl Service { ))); } - let mut shards = self.persistence.list_tenant_shards().await?; - shards.sort_by_key(|tsp| (tsp.tenant_id.clone(), tsp.shard_number, tsp.shard_count)); + let mut persistent_shards = self.persistence.load_active_tenant_shards().await?; + persistent_shards + .sort_by_key(|tsp| (tsp.tenant_id.clone(), tsp.shard_number, tsp.shard_count)); + expect_shards.sort_by_key(|tsp| (tsp.tenant_id.clone(), tsp.shard_number, tsp.shard_count)); - if shards != expect_shards { + if persistent_shards != expect_shards { tracing::error!("Consistency check failed on shards."); tracing::error!( "Shards in memory: {}", @@ -5163,7 +5285,7 @@ impl Service { ); tracing::error!( "Shards in database: {}", - serde_json::to_string(&shards) + serde_json::to_string(&persistent_shards) .map_err(|e| ApiError::InternalServerError(e.into()))? ); return Err(ApiError::InternalServerError(anyhow::anyhow!( @@ -6119,6 +6241,10 @@ impl Service { let mut pending_reconciles = 0; let mut az_violations = 0; + // If we find any tenants to drop from memory, stash them to offload after + // we're done traversing the map of tenants. + let mut drop_detached_tenants = Vec::new(); + let mut reconciles_spawned = 0; for shard in tenants.values_mut() { // Accumulate scheduling statistics @@ -6152,6 +6278,25 @@ impl Service { // Shard wanted to reconcile but for some reason couldn't. pending_reconciles += 1; } + + // If this tenant is detached, try dropping it from memory. This is usually done + // proactively in [`Self::process_results`], but we do it here to handle the edge + // case where a reconcile completes while someone else is holding an op lock for the tenant. + if shard.tenant_shard_id.shard_number == ShardNumber(0) + && shard.policy == PlacementPolicy::Detached + { + if let Some(guard) = self.tenant_op_locks.try_exclusive( + shard.tenant_shard_id.tenant_id, + TenantOperations::DropDetached, + ) { + drop_detached_tenants.push((shard.tenant_shard_id.tenant_id, guard)); + } + } + } + + // Process any deferred tenant drops + for (tenant_id, guard) in drop_detached_tenants { + self.maybe_drop_tenant(tenant_id, &mut locked, &guard); } metrics::METRICS_REGISTRY diff --git a/storage_controller/src/tenant_shard.rs b/storage_controller/src/tenant_shard.rs index cba579e8a7..c17989a316 100644 --- a/storage_controller/src/tenant_shard.rs +++ b/storage_controller/src/tenant_shard.rs @@ -465,6 +465,10 @@ impl ObservedState { locations: HashMap::new(), } } + + pub(crate) fn is_empty(&self) -> bool { + self.locations.is_empty() + } } impl TenantShard { diff --git a/test_runner/regress/test_storage_controller.py b/test_runner/regress/test_storage_controller.py index 973d0cdf82..207f55a214 100644 --- a/test_runner/regress/test_storage_controller.py +++ b/test_runner/regress/test_storage_controller.py @@ -3299,13 +3299,116 @@ def test_storage_controller_detached_stopped( "generation": None, }, ) - + env.storage_controller.reconcile_until_idle() env.storage_controller.consistency_check() # Confirm the detach happened assert env.pageserver.http_client().tenant_list_locations()["tenant_shards"] == [] +@run_only_on_default_postgres("Postgres version makes no difference here") +def test_storage_controller_detach_lifecycle( + neon_env_builder: NeonEnvBuilder, +): + """ + Test that detached tenants are handled properly through their lifecycle: getting dropped + from memory when detached, then getting loaded back on-demand. + """ + + remote_storage_kind = s3_storage() + neon_env_builder.enable_pageserver_remote_storage(remote_storage_kind) + + neon_env_builder.num_pageservers = 1 + + env = neon_env_builder.init_configs() + env.start() + virtual_ps_http = PageserverHttpClient(env.storage_controller_port, lambda: True) + + tenant_id = TenantId.generate() + timeline_id = TimelineId.generate() + env.storage_controller.tenant_create( + tenant_id, + shard_count=1, + ) + virtual_ps_http.timeline_create(PgVersion.NOT_SET, tenant_id, timeline_id) + + remote_prefix = "/".join( + ( + "tenants", + str(tenant_id), + ) + ) + # We will later check data is gone after deletion, so as a control check that it is present to begin with + assert_prefix_not_empty( + neon_env_builder.pageserver_remote_storage, + prefix=remote_prefix, + ) + + assert len(env.pageserver.http_client().tenant_list_locations()["tenant_shards"]) == 1 + assert len(env.storage_controller.tenant_list()) == 1 + + # Detach the tenant + virtual_ps_http.tenant_location_conf( + tenant_id, + { + "mode": "Detached", + "secondary_conf": None, + "tenant_conf": {}, + "generation": None, + }, + ) + # Ensure reconciles are done (the one we do inline in location_conf is advisory and if it takes too long that API just succeeds anyway) + env.storage_controller.reconcile_until_idle() + env.storage_controller.consistency_check() + + # Confirm the detach happened on pageserver + assert env.pageserver.http_client().tenant_list_locations()["tenant_shards"] == [] + # Confirm the tenant is not in memory on the controller + assert env.storage_controller.tenant_list() == [] + + # The detached tenant does not get loaded into memory across a controller restart + env.storage_controller.stop() + env.storage_controller.start() + assert env.storage_controller.tenant_list() == [] + env.storage_controller.consistency_check() + + # The detached tenant can be re-attached + virtual_ps_http.tenant_location_conf( + tenant_id, + { + "mode": "AttachedSingle", + "secondary_conf": None, + "tenant_conf": {}, + "generation": None, + }, + ) + assert len(env.pageserver.http_client().tenant_list_locations()["tenant_shards"]) == 1 + assert len(env.storage_controller.tenant_list()) == 1 + env.storage_controller.consistency_check() + + # Detach it again before doing deletion + virtual_ps_http.tenant_location_conf( + tenant_id, + { + "mode": "Detached", + "secondary_conf": None, + "tenant_conf": {}, + "generation": None, + }, + ) + env.storage_controller.reconcile_until_idle() + env.storage_controller.consistency_check() + + # A detached tenant can be deleted + virtual_ps_http.tenant_delete(tenant_id) + + # Such deletions really work (empty remote storage) + assert_prefix_empty( + neon_env_builder.pageserver_remote_storage, + prefix=remote_prefix, + ) + + @run_only_on_default_postgres("Postgres version makes no difference here") def test_storage_controller_node_flap_detach_race( neon_env_builder: NeonEnvBuilder, From 0ad0db6ff8b5b491244d251fa09c8093f725a1d3 Mon Sep 17 00:00:00 2001 From: Anastasia Lubennikova Date: Wed, 8 Jan 2025 18:55:04 +0000 Subject: [PATCH 008/111] compute: dropdb DROP SUBSCRIPTION fix (#10066) ## Problem Project gets stuck if database with subscriptions was deleted via API / UI. https://github.com/neondatabase/cloud/issues/18646 ## Summary of changes Before dropping the database, drop all the subscriptions in it. Do not drop slot on publisher, because we have no guarantee that the slot still exists or that the publisher is reachable. Add `DropSubscriptionsForDeletedDatabases` phase to run these operations in all databases, we're about to delete. Ignore the error if the database does not exist. --- compute_tools/src/compute.rs | 95 ++++++++++++-- compute_tools/src/spec_apply.rs | 32 ++++- .../sql/drop_subscription_for_drop_dbs.sql | 11 ++ test_runner/fixtures/neon_fixtures.py | 27 +++- test_runner/regress/test_compute_catalog.py | 116 +++++++++++++++++- 5 files changed, 262 insertions(+), 19 deletions(-) create mode 100644 compute_tools/src/sql/drop_subscription_for_drop_dbs.sql diff --git a/compute_tools/src/compute.rs b/compute_tools/src/compute.rs index 78f6033429..1ac97a378b 100644 --- a/compute_tools/src/compute.rs +++ b/compute_tools/src/compute.rs @@ -15,7 +15,7 @@ use std::time::Instant; use anyhow::{Context, Result}; use chrono::{DateTime, Utc}; -use compute_api::spec::{PgIdent, Role}; +use compute_api::spec::{Database, PgIdent, Role}; use futures::future::join_all; use futures::stream::FuturesUnordered; use futures::StreamExt; @@ -45,8 +45,10 @@ use crate::spec_apply::ApplySpecPhase::{ DropInvalidDatabases, DropRoles, HandleNeonExtension, HandleOtherExtensions, RenameAndDeleteDatabases, RenameRoles, RunInEachDatabase, }; +use crate::spec_apply::PerDatabasePhase; use crate::spec_apply::PerDatabasePhase::{ - ChangeSchemaPerms, DeleteDBRoleReferences, HandleAnonExtension, + ChangeSchemaPerms, DeleteDBRoleReferences, DropSubscriptionsForDeletedDatabases, + HandleAnonExtension, }; use crate::spec_apply::{apply_operations, MutableApplyContext, DB}; use crate::sync_sk::{check_if_synced, ping_safekeeper}; @@ -834,7 +836,7 @@ impl ComputeNode { conf } - async fn get_maintenance_client( + pub async fn get_maintenance_client( conf: &tokio_postgres::Config, ) -> Result { let mut conf = conf.clone(); @@ -943,6 +945,78 @@ impl ComputeNode { dbs: databases, })); + // Apply special pre drop database phase. + // NOTE: we use the code of RunInEachDatabase phase for parallelism + // and connection management, but we don't really run it in *each* database, + // only in databases, we're about to drop. + info!("Applying PerDatabase (pre-dropdb) phase"); + let concurrency_token = Arc::new(tokio::sync::Semaphore::new(concurrency)); + + // Run the phase for each database that we're about to drop. + let db_processes = spec + .delta_operations + .iter() + .flatten() + .filter_map(move |op| { + if op.action.as_str() == "delete_db" { + Some(op.name.clone()) + } else { + None + } + }) + .map(|dbname| { + let spec = spec.clone(); + let ctx = ctx.clone(); + let jwks_roles = jwks_roles.clone(); + let mut conf = conf.as_ref().clone(); + let concurrency_token = concurrency_token.clone(); + // We only need dbname field for this phase, so set other fields to dummy values + let db = DB::UserDB(Database { + name: dbname.clone(), + owner: "cloud_admin".to_string(), + options: None, + restrict_conn: false, + invalid: false, + }); + + debug!("Applying per-database phases for Database {:?}", &db); + + match &db { + DB::SystemDB => {} + DB::UserDB(db) => { + conf.dbname(db.name.as_str()); + } + } + + let conf = Arc::new(conf); + let fut = Self::apply_spec_sql_db( + spec.clone(), + conf, + ctx.clone(), + jwks_roles.clone(), + concurrency_token.clone(), + db, + [DropSubscriptionsForDeletedDatabases].to_vec(), + ); + + Ok(spawn(fut)) + }) + .collect::>>(); + + for process in db_processes.into_iter() { + let handle = process?; + if let Err(e) = handle.await? { + // Handle the error case where the database does not exist + // We do not check whether the DB exists or not in the deletion phase, + // so we shouldn't be strict about it in pre-deletion cleanup as well. + if e.to_string().contains("does not exist") { + warn!("Error dropping subscription: {}", e); + } else { + return Err(e); + } + }; + } + for phase in [ CreateSuperUser, DropInvalidDatabases, @@ -962,7 +1036,7 @@ impl ComputeNode { .await?; } - info!("Applying RunInEachDatabase phase"); + info!("Applying RunInEachDatabase2 phase"); let concurrency_token = Arc::new(tokio::sync::Semaphore::new(concurrency)); let db_processes = spec @@ -997,6 +1071,12 @@ impl ComputeNode { jwks_roles.clone(), concurrency_token.clone(), db, + [ + DeleteDBRoleReferences, + ChangeSchemaPerms, + HandleAnonExtension, + ] + .to_vec(), ); Ok(spawn(fut)) @@ -1043,16 +1123,13 @@ impl ComputeNode { jwks_roles: Arc>, concurrency_token: Arc, db: DB, + subphases: Vec, ) -> Result<()> { let _permit = concurrency_token.acquire().await?; let mut client_conn = None; - for subphase in [ - DeleteDBRoleReferences, - ChangeSchemaPerms, - HandleAnonExtension, - ] { + for subphase in subphases { apply_operations( spec.clone(), ctx.clone(), diff --git a/compute_tools/src/spec_apply.rs b/compute_tools/src/spec_apply.rs index 7308d5d36e..695a722d6d 100644 --- a/compute_tools/src/spec_apply.rs +++ b/compute_tools/src/spec_apply.rs @@ -47,6 +47,7 @@ pub enum PerDatabasePhase { DeleteDBRoleReferences, ChangeSchemaPerms, HandleAnonExtension, + DropSubscriptionsForDeletedDatabases, } #[derive(Clone, Debug)] @@ -326,13 +327,12 @@ async fn get_operations<'a>( // Use FORCE to drop database even if there are active connections. // We run this from `cloud_admin`, so it should have enough privileges. + // // NB: there could be other db states, which prevent us from dropping // the database. For example, if db is used by any active subscription // or replication slot. - // TODO: deal with it once we allow logical replication. Proper fix should - // involve returning an error code to the control plane, so it could - // figure out that this is a non-retryable error, return it to the user - // and fail operation permanently. + // Such cases are handled in the DropSubscriptionsForDeletedDatabases + // phase. We do all the cleanup before actually dropping the database. let drop_db_query: String = format!( "DROP DATABASE IF EXISTS {} WITH (FORCE)", &op.name.pg_quote() @@ -444,6 +444,30 @@ async fn get_operations<'a>( } ApplySpecPhase::RunInEachDatabase { db, subphase } => { match subphase { + PerDatabasePhase::DropSubscriptionsForDeletedDatabases => { + match &db { + DB::UserDB(db) => { + let drop_subscription_query: String = format!( + include_str!("sql/drop_subscription_for_drop_dbs.sql"), + datname_str = escape_literal(&db.name), + ); + + let operations = vec![Operation { + query: drop_subscription_query, + comment: Some(format!( + "optionally dropping subscriptions for DB {}", + db.name, + )), + }] + .into_iter(); + + Ok(Box::new(operations)) + } + // skip this cleanup for the system databases + // because users can't drop them + DB::SystemDB => Ok(Box::new(empty())), + } + } PerDatabasePhase::DeleteDBRoleReferences => { let ctx = ctx.read().await; diff --git a/compute_tools/src/sql/drop_subscription_for_drop_dbs.sql b/compute_tools/src/sql/drop_subscription_for_drop_dbs.sql new file mode 100644 index 0000000000..dfb925e48e --- /dev/null +++ b/compute_tools/src/sql/drop_subscription_for_drop_dbs.sql @@ -0,0 +1,11 @@ +DO $$ +DECLARE + subname TEXT; +BEGIN + FOR subname IN SELECT pg_subscription.subname FROM pg_subscription WHERE subdbid = (SELECT oid FROM pg_database WHERE datname = {datname_str}) LOOP + EXECUTE format('ALTER SUBSCRIPTION %I DISABLE;', subname); + EXECUTE format('ALTER SUBSCRIPTION %I SET (slot_name = NONE);', subname); + EXECUTE format('DROP SUBSCRIPTION %I;', subname); + END LOOP; +END; +$$; diff --git a/test_runner/fixtures/neon_fixtures.py b/test_runner/fixtures/neon_fixtures.py index 00fdda2998..e22e452a52 100644 --- a/test_runner/fixtures/neon_fixtures.py +++ b/test_runner/fixtures/neon_fixtures.py @@ -4933,13 +4933,30 @@ def check_restored_datadir_content( assert (mismatch, error) == ([], []) -def logical_replication_sync(subscriber: PgProtocol, publisher: PgProtocol) -> Lsn: +def logical_replication_sync( + subscriber: PgProtocol, + publisher: PgProtocol, + sub_dbname: str | None = None, + pub_dbname: str | None = None, +) -> Lsn: """Wait logical replication subscriber to sync with publisher.""" - publisher_lsn = Lsn(publisher.safe_psql("SELECT pg_current_wal_flush_lsn()")[0][0]) + if pub_dbname is not None: + publisher_lsn = Lsn( + publisher.safe_psql("SELECT pg_current_wal_flush_lsn()", dbname=pub_dbname)[0][0] + ) + else: + publisher_lsn = Lsn(publisher.safe_psql("SELECT pg_current_wal_flush_lsn()")[0][0]) + while True: - res = subscriber.safe_psql("select latest_end_lsn from pg_catalog.pg_stat_subscription")[0][ - 0 - ] + if sub_dbname is not None: + res = subscriber.safe_psql( + "select latest_end_lsn from pg_catalog.pg_stat_subscription", dbname=sub_dbname + )[0][0] + else: + res = subscriber.safe_psql( + "select latest_end_lsn from pg_catalog.pg_stat_subscription" + )[0][0] + if res: log.info(f"subscriber_lsn={res}") subscriber_lsn = Lsn(res) diff --git a/test_runner/regress/test_compute_catalog.py b/test_runner/regress/test_compute_catalog.py index b3719a45ed..e411aad97d 100644 --- a/test_runner/regress/test_compute_catalog.py +++ b/test_runner/regress/test_compute_catalog.py @@ -1,7 +1,9 @@ from __future__ import annotations +import logging + import requests -from fixtures.neon_fixtures import NeonEnv +from fixtures.neon_fixtures import NeonEnv, logical_replication_sync TEST_DB_NAMES = [ { @@ -136,3 +138,115 @@ def test_compute_create_databases(neon_simple_env: NeonEnv): assert curr_db is not None assert len(curr_db) == 1 assert curr_db[0] == db["name"] + + +def test_dropdb_with_subscription(neon_simple_env: NeonEnv): + """ + Test that compute_ctl can drop a database that has a logical replication subscription. + """ + env = neon_simple_env + + # Create and start endpoint so that neon_local put all the generated + # stuff into the spec.json file. + endpoint = env.endpoints.create_start("main") + + TEST_DB_NAMES = [ + { + "name": "neondb", + "owner": "cloud_admin", + }, + { + "name": "subscriber_db", + "owner": "cloud_admin", + }, + { + "name": "publisher_db", + "owner": "cloud_admin", + }, + ] + + # Update the spec.json file to create the databases + # and reconfigure the endpoint to apply the changes. + endpoint.respec_deep( + **{ + "skip_pg_catalog_updates": False, + "cluster": { + "databases": TEST_DB_NAMES, + }, + } + ) + endpoint.reconfigure() + + # connect to the publisher_db and create a publication + with endpoint.cursor(dbname="publisher_db") as cursor: + cursor.execute("CREATE PUBLICATION mypub FOR ALL TABLES") + cursor.execute("select pg_catalog.pg_create_logical_replication_slot('mysub', 'pgoutput');") + cursor.execute("CREATE TABLE t(a int)") + cursor.execute("INSERT INTO t VALUES (1)") + + # connect to the subscriber_db and create a subscription + # Note that we need to create subscription with + connstr = endpoint.connstr(dbname="publisher_db").replace("'", "''") + with endpoint.cursor(dbname="subscriber_db") as cursor: + cursor.execute("CREATE TABLE t(a int)") + cursor.execute( + f"CREATE SUBSCRIPTION mysub CONNECTION '{connstr}' PUBLICATION mypub WITH (create_slot = false) " + ) + + # wait for the subscription to be active + logical_replication_sync( + endpoint, endpoint, sub_dbname="subscriber_db", pub_dbname="publisher_db" + ) + + # Check that replication is working + with endpoint.cursor(dbname="subscriber_db") as cursor: + cursor.execute("SELECT * FROM t") + rows = cursor.fetchall() + assert len(rows) == 1 + assert rows[0][0] == 1 + + # drop the subscriber_db from the list + TEST_DB_NAMES_NEW = [ + { + "name": "neondb", + "owner": "cloud_admin", + }, + { + "name": "publisher_db", + "owner": "cloud_admin", + }, + ] + # Update the spec.json file to drop the database + # and reconfigure the endpoint to apply the changes. + endpoint.respec_deep( + **{ + "skip_pg_catalog_updates": False, + "cluster": { + "databases": TEST_DB_NAMES_NEW, + }, + "delta_operations": [ + {"action": "delete_db", "name": "subscriber_db"}, + # also test the case when we try to delete a non-existent database + # shouldn't happen in normal operation, + # but can occur when failed operations are retried + {"action": "delete_db", "name": "nonexistent_db"}, + ], + } + ) + + logging.info("Reconfiguring the endpoint to drop the subscriber_db") + endpoint.reconfigure() + + # Check that the subscriber_db is dropped + with endpoint.cursor() as cursor: + cursor.execute("SELECT datname FROM pg_database WHERE datname = %s", ("subscriber_db",)) + catalog_db = cursor.fetchone() + assert catalog_db is None + + # Check that we can still connect to the publisher_db + with endpoint.cursor(dbname="publisher_db") as cursor: + cursor.execute("SELECT * FROM current_database()") + curr_db = cursor.fetchone() + assert curr_db is not None + assert len(curr_db) == 1 + assert curr_db[0] == "publisher_db" From fcfff724547cda260e884d2d680f199fdbc9471c Mon Sep 17 00:00:00 2001 From: Ivan Efremov Date: Wed, 8 Jan 2025 21:34:53 +0200 Subject: [PATCH 009/111] impr(proxy): Decouple ip_allowlist from the CancelClosure (#10199) This PR removes the direct dependency of the IP allowlist from CancelClosure, allowing for more scalable and flexible IP restrictions and enabling the future use of Redis-based CancelMap storage. Changes: - Introduce a new BackendAuth async trait that retrieves the IP allowlist through existing authentication methods; - Improve cancellation error handling by instrument() async cancel_sesion() rather than dropping it. - Set and store IP allowlist for SCRAM Proxy to consistently perform IP allowance check Relates to #9660 --- proxy/src/auth/backend/console_redirect.rs | 32 ++++- proxy/src/auth/backend/mod.rs | 47 ++++++-- proxy/src/bin/proxy.rs | 54 ++++++++- proxy/src/cancellation.rs | 111 +++++++++++++++++- proxy/src/compute.rs | 28 ++++- proxy/src/console_redirect_proxy.rs | 28 ++--- .../control_plane/client/cplane_proxy_v1.rs | 36 ++++-- proxy/src/proxy/mod.rs | 31 +++-- 8 files changed, 307 insertions(+), 60 deletions(-) diff --git a/proxy/src/auth/backend/console_redirect.rs b/proxy/src/auth/backend/console_redirect.rs index c3de77b352..dbfda588cc 100644 --- a/proxy/src/auth/backend/console_redirect.rs +++ b/proxy/src/auth/backend/console_redirect.rs @@ -1,16 +1,18 @@ use async_trait::async_trait; use postgres_client::config::SslMode; use pq_proto::BeMessage as Be; +use std::fmt; use thiserror::Error; use tokio::io::{AsyncRead, AsyncWrite}; use tracing::{info, info_span}; -use super::ComputeCredentialKeys; +use super::{ComputeCredentialKeys, ControlPlaneApi}; +use crate::auth::backend::{BackendIpAllowlist, ComputeUserInfo}; use crate::auth::IpPattern; use crate::cache::Cached; use crate::config::AuthenticationConfig; use crate::context::RequestContext; -use crate::control_plane::{self, CachedNodeInfo, NodeInfo}; +use crate::control_plane::{self, client::cplane_proxy_v1, CachedNodeInfo, NodeInfo}; use crate::error::{ReportableError, UserFacingError}; use crate::proxy::connect_compute::ComputeConnectBackend; use crate::stream::PqStream; @@ -31,6 +33,13 @@ pub(crate) enum ConsoleRedirectError { #[derive(Debug)] pub struct ConsoleRedirectBackend { console_uri: reqwest::Url, + api: cplane_proxy_v1::NeonControlPlaneClient, +} + +impl fmt::Debug for cplane_proxy_v1::NeonControlPlaneClient { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "NeonControlPlaneClient") + } } impl UserFacingError for ConsoleRedirectError { @@ -71,9 +80,24 @@ pub(crate) fn new_psql_session_id() -> String { hex::encode(rand::random::<[u8; 8]>()) } +#[async_trait] +impl BackendIpAllowlist for ConsoleRedirectBackend { + async fn get_allowed_ips( + &self, + ctx: &RequestContext, + user_info: &ComputeUserInfo, + ) -> auth::Result> { + self.api + .get_allowed_ips_and_secret(ctx, user_info) + .await + .map(|(ips, _)| ips.as_ref().clone()) + .map_err(|e| e.into()) + } +} + impl ConsoleRedirectBackend { - pub fn new(console_uri: reqwest::Url) -> Self { - Self { console_uri } + pub fn new(console_uri: reqwest::Url, api: cplane_proxy_v1::NeonControlPlaneClient) -> Self { + Self { console_uri, api } } pub(crate) async fn authenticate( diff --git a/proxy/src/auth/backend/mod.rs b/proxy/src/auth/backend/mod.rs index 0c9a7f7825..de48be2952 100644 --- a/proxy/src/auth/backend/mod.rs +++ b/proxy/src/auth/backend/mod.rs @@ -16,7 +16,9 @@ use tokio::io::{AsyncRead, AsyncWrite}; use tracing::{debug, info, warn}; use crate::auth::credentials::check_peer_addr_is_in_list; -use crate::auth::{self, validate_password_and_exchange, AuthError, ComputeUserInfoMaybeEndpoint}; +use crate::auth::{ + self, validate_password_and_exchange, AuthError, ComputeUserInfoMaybeEndpoint, IpPattern, +}; use crate::cache::Cached; use crate::config::AuthenticationConfig; use crate::context::RequestContext; @@ -131,7 +133,7 @@ pub(crate) struct ComputeUserInfoNoEndpoint { pub(crate) options: NeonOptions, } -#[derive(Debug, Clone)] +#[derive(Debug, Clone, Default)] pub(crate) struct ComputeUserInfo { pub(crate) endpoint: EndpointId, pub(crate) user: RoleName, @@ -244,6 +246,15 @@ impl AuthenticationConfig { } } +#[async_trait::async_trait] +pub(crate) trait BackendIpAllowlist { + async fn get_allowed_ips( + &self, + ctx: &RequestContext, + user_info: &ComputeUserInfo, + ) -> auth::Result>; +} + /// True to its name, this function encapsulates our current auth trade-offs. /// Here, we choose the appropriate auth flow based on circumstances. /// @@ -256,7 +267,7 @@ async fn auth_quirks( allow_cleartext: bool, config: &'static AuthenticationConfig, endpoint_rate_limiter: Arc, -) -> auth::Result { +) -> auth::Result<(ComputeCredentials, Option>)> { // If there's no project so far, that entails that client doesn't // support SNI or other means of passing the endpoint (project) name. // We now expect to see a very specific payload in the place of password. @@ -315,7 +326,7 @@ async fn auth_quirks( ) .await { - Ok(keys) => Ok(keys), + Ok(keys) => Ok((keys, Some(allowed_ips.as_ref().clone()))), Err(e) => { if e.is_password_failed() { // The password could have been changed, so we invalidate the cache. @@ -385,7 +396,7 @@ impl<'a> Backend<'a, ComputeUserInfoMaybeEndpoint> { allow_cleartext: bool, config: &'static AuthenticationConfig, endpoint_rate_limiter: Arc, - ) -> auth::Result> { + ) -> auth::Result<(Backend<'a, ComputeCredentials>, Option>)> { let res = match self { Self::ControlPlane(api, user_info) => { debug!( @@ -394,7 +405,7 @@ impl<'a> Backend<'a, ComputeUserInfoMaybeEndpoint> { "performing authentication using the console" ); - let credentials = auth_quirks( + let (credentials, ip_allowlist) = auth_quirks( ctx, &*api, user_info, @@ -404,7 +415,7 @@ impl<'a> Backend<'a, ComputeUserInfoMaybeEndpoint> { endpoint_rate_limiter, ) .await?; - Backend::ControlPlane(api, credentials) + Ok((Backend::ControlPlane(api, credentials), ip_allowlist)) } Self::Local(_) => { return Err(auth::AuthError::bad_auth_method("invalid for local proxy")) @@ -413,7 +424,7 @@ impl<'a> Backend<'a, ComputeUserInfoMaybeEndpoint> { // TODO: replace with some metric info!("user successfully authenticated"); - Ok(res) + res } } @@ -441,6 +452,24 @@ impl Backend<'_, ComputeUserInfo> { } } +#[async_trait::async_trait] +impl BackendIpAllowlist for Backend<'_, ()> { + async fn get_allowed_ips( + &self, + ctx: &RequestContext, + user_info: &ComputeUserInfo, + ) -> auth::Result> { + let auth_data = match self { + Self::ControlPlane(api, ()) => api.get_allowed_ips_and_secret(ctx, user_info).await, + Self::Local(_) => Ok((Cached::new_uncached(Arc::new(vec![])), None)), + }; + + auth_data + .map(|(ips, _)| ips.as_ref().clone()) + .map_err(|e| e.into()) + } +} + #[async_trait::async_trait] impl ComputeConnectBackend for Backend<'_, ComputeCredentials> { async fn wake_compute( @@ -786,7 +815,7 @@ mod tests { .await .unwrap(); - assert_eq!(creds.info.endpoint, "my-endpoint"); + assert_eq!(creds.0.info.endpoint, "my-endpoint"); handle.await.unwrap(); } diff --git a/proxy/src/bin/proxy.rs b/proxy/src/bin/proxy.rs index 3b122d771c..70b50436bf 100644 --- a/proxy/src/bin/proxy.rs +++ b/proxy/src/bin/proxy.rs @@ -744,9 +744,59 @@ fn build_auth_backend( } AuthBackendType::ConsoleRedirect => { - let url = args.uri.parse()?; - let backend = ConsoleRedirectBackend::new(url); + let wake_compute_cache_config: CacheOptions = args.wake_compute_cache.parse()?; + let project_info_cache_config: ProjectInfoCacheOptions = + args.project_info_cache.parse()?; + let endpoint_cache_config: config::EndpointCacheConfig = + args.endpoint_cache_config.parse()?; + info!("Using NodeInfoCache (wake_compute) with options={wake_compute_cache_config:?}"); + info!( + "Using AllowedIpsCache (wake_compute) with options={project_info_cache_config:?}" + ); + info!("Using EndpointCacheConfig with options={endpoint_cache_config:?}"); + let caches = Box::leak(Box::new(control_plane::caches::ApiCaches::new( + wake_compute_cache_config, + project_info_cache_config, + endpoint_cache_config, + ))); + + let config::ConcurrencyLockOptions { + shards, + limiter, + epoch, + timeout, + } = args.wake_compute_lock.parse()?; + info!(?limiter, shards, ?epoch, "Using NodeLocks (wake_compute)"); + let locks = Box::leak(Box::new(control_plane::locks::ApiLocks::new( + "wake_compute_lock", + limiter, + shards, + timeout, + epoch, + &Metrics::get().wake_compute_lock, + )?)); + + let url = args.uri.clone().parse()?; + let ep_url: proxy::url::ApiUrl = args.auth_endpoint.parse()?; + let endpoint = http::Endpoint::new(ep_url, http::new_client()); + let mut wake_compute_rps_limit = args.wake_compute_limit.clone(); + RateBucketInfo::validate(&mut wake_compute_rps_limit)?; + let wake_compute_endpoint_rate_limiter = + Arc::new(WakeComputeRateLimiter::new(wake_compute_rps_limit)); + + // Since we use only get_allowed_ips_and_secret() wake_compute_endpoint_rate_limiter + // and locks are not used in ConsoleRedirectBackend, + // but they are required by the NeonControlPlaneClient + let api = control_plane::client::cplane_proxy_v1::NeonControlPlaneClient::new( + endpoint, + args.control_plane_token.clone(), + caches, + locks, + wake_compute_endpoint_rate_limiter, + ); + + let backend = ConsoleRedirectBackend::new(url, api); let config = Box::leak(Box::new(backend)); Ok(Either::Right(config)) diff --git a/proxy/src/cancellation.rs b/proxy/src/cancellation.rs index df618cf242..a96c43f2ce 100644 --- a/proxy/src/cancellation.rs +++ b/proxy/src/cancellation.rs @@ -12,8 +12,10 @@ use tokio::sync::Mutex; use tracing::{debug, info}; use uuid::Uuid; -use crate::auth::{check_peer_addr_is_in_list, IpPattern}; +use crate::auth::backend::{BackendIpAllowlist, ComputeUserInfo}; +use crate::auth::{check_peer_addr_is_in_list, AuthError, IpPattern}; use crate::config::ComputeConfig; +use crate::context::RequestContext; use crate::error::ReportableError; use crate::ext::LockExt; use crate::metrics::{CancellationRequest, CancellationSource, Metrics}; @@ -56,6 +58,9 @@ pub(crate) enum CancelError { #[error("IP is not allowed")] IpNotAllowed, + + #[error("Authentication backend error")] + AuthError(#[from] AuthError), } impl ReportableError for CancelError { @@ -68,6 +73,7 @@ impl ReportableError for CancelError { CancelError::Postgres(_) => crate::error::ErrorKind::Compute, CancelError::RateLimit => crate::error::ErrorKind::RateLimit, CancelError::IpNotAllowed => crate::error::ErrorKind::User, + CancelError::AuthError(_) => crate::error::ErrorKind::ControlPlane, } } } @@ -102,10 +108,7 @@ impl CancellationHandler

{ } } - /// Try to cancel a running query for the corresponding connection. - /// If the cancellation key is not found, it will be published to Redis. - /// check_allowed - if true, check if the IP is allowed to cancel the query - /// return Result primarily for tests + /// Cancelling only in notification, will be removed pub(crate) async fn cancel_session( &self, key: CancelKeyData, @@ -134,7 +137,8 @@ impl CancellationHandler

{ } // NB: we should immediately release the lock after cloning the token. - let Some(cancel_closure) = self.map.get(&key).and_then(|x| x.clone()) else { + let cancel_state = self.map.get(&key).and_then(|x| x.clone()); + let Some(cancel_closure) = cancel_state else { tracing::warn!("query cancellation key not found: {key}"); Metrics::get() .proxy @@ -185,6 +189,96 @@ impl CancellationHandler

{ cancel_closure.try_cancel_query(self.compute_config).await } + /// Try to cancel a running query for the corresponding connection. + /// If the cancellation key is not found, it will be published to Redis. + /// check_allowed - if true, check if the IP is allowed to cancel the query. + /// Will fetch IP allowlist internally. + /// + /// return Result primarily for tests + pub(crate) async fn cancel_session_auth( + &self, + key: CancelKeyData, + ctx: RequestContext, + check_allowed: bool, + auth_backend: &T, + ) -> Result<(), CancelError> { + // TODO: check for unspecified address is only for backward compatibility, should be removed + if !ctx.peer_addr().is_unspecified() { + let subnet_key = match ctx.peer_addr() { + IpAddr::V4(ip) => IpNet::V4(Ipv4Net::new_assert(ip, 24).trunc()), // use defaut mask here + IpAddr::V6(ip) => IpNet::V6(Ipv6Net::new_assert(ip, 64).trunc()), + }; + if !self.limiter.lock_propagate_poison().check(subnet_key, 1) { + // log only the subnet part of the IP address to know which subnet is rate limited + tracing::warn!("Rate limit exceeded. Skipping cancellation message, {subnet_key}"); + Metrics::get() + .proxy + .cancellation_requests_total + .inc(CancellationRequest { + source: self.from, + kind: crate::metrics::CancellationOutcome::RateLimitExceeded, + }); + return Err(CancelError::RateLimit); + } + } + + // NB: we should immediately release the lock after cloning the token. + let cancel_state = self.map.get(&key).and_then(|x| x.clone()); + let Some(cancel_closure) = cancel_state else { + tracing::warn!("query cancellation key not found: {key}"); + Metrics::get() + .proxy + .cancellation_requests_total + .inc(CancellationRequest { + source: self.from, + kind: crate::metrics::CancellationOutcome::NotFound, + }); + + if ctx.session_id() == Uuid::nil() { + // was already published, do not publish it again + return Ok(()); + } + + match self + .client + .try_publish(key, ctx.session_id(), ctx.peer_addr()) + .await + { + Ok(()) => {} // do nothing + Err(e) => { + // log it here since cancel_session could be spawned in a task + tracing::error!("failed to publish cancellation key: {key}, error: {e}"); + return Err(CancelError::IO(std::io::Error::new( + std::io::ErrorKind::Other, + e.to_string(), + ))); + } + } + return Ok(()); + }; + + let ip_allowlist = auth_backend + .get_allowed_ips(&ctx, &cancel_closure.user_info) + .await + .map_err(CancelError::AuthError)?; + + if check_allowed && !check_peer_addr_is_in_list(&ctx.peer_addr(), &ip_allowlist) { + // log it here since cancel_session could be spawned in a task + tracing::warn!("IP is not allowed to cancel the query: {key}"); + return Err(CancelError::IpNotAllowed); + } + + Metrics::get() + .proxy + .cancellation_requests_total + .inc(CancellationRequest { + source: self.from, + kind: crate::metrics::CancellationOutcome::Found, + }); + info!("cancelling query per user's request using key {key}"); + cancel_closure.try_cancel_query(self.compute_config).await + } + #[cfg(test)] fn contains(&self, session: &Session

) -> bool { self.map.contains_key(&session.key) @@ -248,6 +342,7 @@ pub struct CancelClosure { cancel_token: CancelToken, ip_allowlist: Vec, hostname: String, // for pg_sni router + user_info: ComputeUserInfo, } impl CancelClosure { @@ -256,12 +351,14 @@ impl CancelClosure { cancel_token: CancelToken, ip_allowlist: Vec, hostname: String, + user_info: ComputeUserInfo, ) -> Self { Self { socket_addr, cancel_token, ip_allowlist, hostname, + user_info, } } /// Cancels the query running on user's compute node. @@ -288,6 +385,8 @@ impl CancelClosure { debug!("query was cancelled"); Ok(()) } + + /// Obsolete (will be removed after moving CancelMap to Redis), only for notifications pub(crate) fn set_ip_allowlist(&mut self, ip_allowlist: Vec) { self.ip_allowlist = ip_allowlist; } diff --git a/proxy/src/compute.rs b/proxy/src/compute.rs index 89de6692ad..788bd63fee 100644 --- a/proxy/src/compute.rs +++ b/proxy/src/compute.rs @@ -13,6 +13,7 @@ use thiserror::Error; use tokio::net::TcpStream; use tracing::{debug, error, info, warn}; +use crate::auth::backend::ComputeUserInfo; use crate::auth::parse_endpoint_param; use crate::cancellation::CancelClosure; use crate::config::ComputeConfig; @@ -23,8 +24,10 @@ use crate::control_plane::messages::MetricsAuxInfo; use crate::error::{ReportableError, UserFacingError}; use crate::metrics::{Metrics, NumDbConnectionsGuard}; use crate::proxy::neon_option; +use crate::proxy::NeonOptions; use crate::tls::postgres_rustls::MakeRustlsConnect; use crate::types::Host; +use crate::types::{EndpointId, RoleName}; pub const COULD_NOT_CONNECT: &str = "Couldn't connect to compute node"; @@ -284,6 +287,28 @@ impl ConnCfg { self.0.get_ssl_mode() ); + let compute_info = match parameters.get("user") { + Some(user) => { + match parameters.get("database") { + Some(database) => { + ComputeUserInfo { + user: RoleName::from(user), + options: NeonOptions::default(), // just a shim, we don't need options + endpoint: EndpointId::from(database), + } + } + None => { + warn!("compute node didn't return database name"); + ComputeUserInfo::default() + } + } + } + None => { + warn!("compute node didn't return user name"); + ComputeUserInfo::default() + } + }; + // NB: CancelToken is supposed to hold socket_addr, but we use connect_raw. // Yet another reason to rework the connection establishing code. let cancel_closure = CancelClosure::new( @@ -294,8 +319,9 @@ impl ConnCfg { process_id, secret_key, }, - vec![], + vec![], // TODO: deprecated, will be removed host.to_string(), + compute_info, ); let connection = PostgresConnection { diff --git a/proxy/src/console_redirect_proxy.rs b/proxy/src/console_redirect_proxy.rs index 25a549039c..846f55f9e1 100644 --- a/proxy/src/console_redirect_proxy.rs +++ b/proxy/src/console_redirect_proxy.rs @@ -159,6 +159,7 @@ pub(crate) async fn handle_client( let request_gauge = metrics.connection_requests.guard(proto); let tls = config.tls_config.as_ref(); + let record_handshake_error = !ctx.has_private_peer_addr(); let pause = ctx.latency_timer_pause(crate::metrics::Waiting::Client); let do_handshake = handshake(ctx, stream, tls, record_handshake_error); @@ -171,23 +172,20 @@ pub(crate) async fn handle_client( // spawn a task to cancel the session, but don't wait for it cancellations.spawn({ let cancellation_handler_clone = Arc::clone(&cancellation_handler); - let session_id = ctx.session_id(); - let peer_ip = ctx.peer_addr(); - let cancel_span = tracing::span!(parent: None, tracing::Level::INFO, "cancel_session", session_id = ?session_id); + let ctx = ctx.clone(); + let cancel_span = tracing::span!(parent: None, tracing::Level::INFO, "cancel_session", session_id = ?ctx.session_id()); cancel_span.follows_from(tracing::Span::current()); async move { - drop( - cancellation_handler_clone - .cancel_session( - cancel_key_data, - session_id, - peer_ip, - config.authentication_config.ip_allowlist_check_enabled, - ) - .instrument(cancel_span) - .await, - ); - } + cancellation_handler_clone + .cancel_session_auth( + cancel_key_data, + ctx, + config.authentication_config.ip_allowlist_check_enabled, + backend, + ) + .await + .inspect_err(|e | debug!(error = ?e, "cancel_session failed")).ok(); + }.instrument(cancel_span) }); return Ok(None); diff --git a/proxy/src/control_plane/client/cplane_proxy_v1.rs b/proxy/src/control_plane/client/cplane_proxy_v1.rs index 00038a6ac6..ece03156d1 100644 --- a/proxy/src/control_plane/client/cplane_proxy_v1.rs +++ b/proxy/src/control_plane/client/cplane_proxy_v1.rs @@ -29,7 +29,7 @@ use crate::rate_limiter::WakeComputeRateLimiter; use crate::types::{EndpointCacheKey, EndpointId}; use crate::{compute, http, scram}; -const X_REQUEST_ID: HeaderName = HeaderName::from_static("x-request-id"); +pub(crate) const X_REQUEST_ID: HeaderName = HeaderName::from_static("x-request-id"); #[derive(Clone)] pub struct NeonControlPlaneClient { @@ -78,15 +78,30 @@ impl NeonControlPlaneClient { info!("endpoint is not valid, skipping the request"); return Ok(AuthInfo::default()); } - let request_id = ctx.session_id().to_string(); - let application_name = ctx.console_application_name(); + self.do_get_auth_req(user_info, &ctx.session_id(), Some(ctx)) + .await + } + + async fn do_get_auth_req( + &self, + user_info: &ComputeUserInfo, + session_id: &uuid::Uuid, + ctx: Option<&RequestContext>, + ) -> Result { + let request_id: String = session_id.to_string(); + let application_name = if let Some(ctx) = ctx { + ctx.console_application_name() + } else { + "auth_cancellation".to_string() + }; + async { let request = self .endpoint .get_path("get_endpoint_access_control") .header(X_REQUEST_ID, &request_id) .header(AUTHORIZATION, format!("Bearer {}", &self.jwt)) - .query(&[("session_id", ctx.session_id())]) + .query(&[("session_id", session_id)]) .query(&[ ("application_name", application_name.as_str()), ("endpointish", user_info.endpoint.as_str()), @@ -96,9 +111,16 @@ impl NeonControlPlaneClient { debug!(url = request.url().as_str(), "sending http request"); let start = Instant::now(); - let pause = ctx.latency_timer_pause(crate::metrics::Waiting::Cplane); - let response = self.endpoint.execute(request).await?; - drop(pause); + let response = match ctx { + Some(ctx) => { + let pause = ctx.latency_timer_pause(crate::metrics::Waiting::Cplane); + let rsp = self.endpoint.execute(request).await; + drop(pause); + rsp? + } + None => self.endpoint.execute(request).await?, + }; + info!(duration = ?start.elapsed(), "received http response"); let body = match parse_body::(response).await { Ok(body) => body, diff --git a/proxy/src/proxy/mod.rs b/proxy/src/proxy/mod.rs index 3926c56fec..1f7dba2f9a 100644 --- a/proxy/src/proxy/mod.rs +++ b/proxy/src/proxy/mod.rs @@ -273,23 +273,20 @@ pub(crate) async fn handle_client( // spawn a task to cancel the session, but don't wait for it cancellations.spawn({ let cancellation_handler_clone = Arc::clone(&cancellation_handler); - let session_id = ctx.session_id(); - let peer_ip = ctx.peer_addr(); - let cancel_span = tracing::span!(parent: None, tracing::Level::INFO, "cancel_session", session_id = ?session_id); + let ctx = ctx.clone(); + let cancel_span = tracing::span!(parent: None, tracing::Level::INFO, "cancel_session", session_id = ?ctx.session_id()); cancel_span.follows_from(tracing::Span::current()); async move { - drop( - cancellation_handler_clone - .cancel_session( - cancel_key_data, - session_id, - peer_ip, - config.authentication_config.ip_allowlist_check_enabled, - ) - .instrument(cancel_span) - .await, - ); - } + cancellation_handler_clone + .cancel_session_auth( + cancel_key_data, + ctx, + config.authentication_config.ip_allowlist_check_enabled, + auth_backend, + ) + .await + .inspect_err(|e | debug!(error = ?e, "cancel_session failed")).ok(); + }.instrument(cancel_span) }); return Ok(None); @@ -315,7 +312,7 @@ pub(crate) async fn handle_client( }; let user = user_info.get_user().to_owned(); - let user_info = match user_info + let (user_info, ip_allowlist) = match user_info .authenticate( ctx, &mut stream, @@ -356,6 +353,8 @@ pub(crate) async fn handle_client( .or_else(|e| stream.throw_error(e)) .await?; + node.cancel_closure + .set_ip_allowlist(ip_allowlist.unwrap_or_default()); let session = cancellation_handler.get_session(); prepare_client_connection(&node, &session, &mut stream).await?; From 5b2751397dee2234bfc97d2241f05ab47b95f5ee Mon Sep 17 00:00:00 2001 From: Tristan Partin Date: Thu, 9 Jan 2025 01:05:07 -0600 Subject: [PATCH 010/111] Refactor MigrationRunner::run_migrations() to call a helper (#10232) This will make it easier to add per-db migrations, such as that for CVE-2024-4317. Link: https://www.postgresql.org/support/security/CVE-2024-4317/ Signed-off-by: Tristan Partin Signed-off-by: Tristan Partin --- compute_tools/src/migration.rs | 94 ++++++++++++++++------------------ 1 file changed, 44 insertions(+), 50 deletions(-) diff --git a/compute_tools/src/migration.rs b/compute_tools/src/migration.rs index 1f3de65806..45c33172f7 100644 --- a/compute_tools/src/migration.rs +++ b/compute_tools/src/migration.rs @@ -1,6 +1,6 @@ use anyhow::{Context, Result}; use fail::fail_point; -use postgres::Client; +use postgres::{Client, Transaction}; use tracing::info; /// Runs a series of migrations on a target database @@ -20,11 +20,9 @@ impl<'m> MigrationRunner<'m> { /// Get the current value neon_migration.migration_id fn get_migration_id(&mut self) -> Result { - let query = "SELECT id FROM neon_migration.migration_id"; let row = self .client - .query_one(query, &[]) - .context("run_migrations get migration_id")?; + .query_one("SELECT id FROM neon_migration.migration_id", &[])?; Ok(row.get::<&str, i64>("id")) } @@ -34,7 +32,7 @@ impl<'m> MigrationRunner<'m> { /// This function has a fail point called compute-migration, which can be /// used if you would like to fail the application of a series of migrations /// at some point. - fn update_migration_id(&mut self, migration_id: i64) -> Result<()> { + fn update_migration_id(txn: &mut Transaction, migration_id: i64) -> Result<()> { // We use this fail point in order to check that failing in the // middle of applying a series of migrations fails in an expected // manner @@ -55,12 +53,11 @@ impl<'m> MigrationRunner<'m> { } } - self.client - .query( - "UPDATE neon_migration.migration_id SET id = $1", - &[&migration_id], - ) - .context("run_migrations update id")?; + txn.query( + "UPDATE neon_migration.migration_id SET id = $1", + &[&migration_id], + ) + .with_context(|| format!("update neon_migration.migration_id to {migration_id}"))?; Ok(()) } @@ -81,53 +78,50 @@ impl<'m> MigrationRunner<'m> { Ok(()) } - /// Run the configrured set of migrations + /// Run an individual migration + fn run_migration(txn: &mut Transaction, migration_id: i64, migration: &str) -> Result<()> { + if migration.starts_with("-- SKIP") { + info!("Skipping migration id={}", migration_id); + + // Even though we are skipping the migration, updating the + // migration ID should help keep logic easy to understand when + // trying to understand the state of a cluster. + Self::update_migration_id(txn, migration_id)?; + } else { + info!("Running migration id={}:\n{}\n", migration_id, migration); + + txn.simple_query(migration) + .with_context(|| format!("apply migration {migration_id}"))?; + + Self::update_migration_id(txn, migration_id)?; + } + + Ok(()) + } + + /// Run the configured set of migrations pub fn run_migrations(mut self) -> Result<()> { - self.prepare_database()?; + self.prepare_database() + .context("prepare database to handle migrations")?; let mut current_migration = self.get_migration_id()? as usize; while current_migration < self.migrations.len() { - macro_rules! migration_id { - ($cm:expr) => { - ($cm + 1) as i64 - }; - } + // The index lags the migration ID by 1, so the current migration + // ID is also the next index + let migration_id = (current_migration + 1) as i64; - let migration = self.migrations[current_migration]; + let mut txn = self + .client + .transaction() + .with_context(|| format!("begin transaction for migration {migration_id}"))?; - if migration.starts_with("-- SKIP") { - info!("Skipping migration id={}", migration_id!(current_migration)); + Self::run_migration(&mut txn, migration_id, self.migrations[current_migration]) + .with_context(|| format!("running migration {migration_id}"))?; - // Even though we are skipping the migration, updating the - // migration ID should help keep logic easy to understand when - // trying to understand the state of a cluster. - self.update_migration_id(migration_id!(current_migration))?; - } else { - info!( - "Running migration id={}:\n{}\n", - migration_id!(current_migration), - migration - ); + txn.commit() + .with_context(|| format!("commit transaction for migration {migration_id}"))?; - self.client - .simple_query("BEGIN") - .context("begin migration")?; - - self.client.simple_query(migration).with_context(|| { - format!( - "run_migrations migration id={}", - migration_id!(current_migration) - ) - })?; - - self.update_migration_id(migration_id!(current_migration))?; - - self.client - .simple_query("COMMIT") - .context("commit migration")?; - - info!("Finished migration id={}", migration_id!(current_migration)); - } + info!("Finished migration id={}", migration_id); current_migration += 1; } From 5baa4e7f0a4821802dccd98e936b361cbc7a544b Mon Sep 17 00:00:00 2001 From: John Spray Date: Thu, 9 Jan 2025 11:47:55 +0000 Subject: [PATCH 011/111] docker: don't set LD_LIBRARY_PATH (#10321) ## Problem This was causing storage controller to still use neon-built libpq instead of vanilla libpq. Since https://github.com/neondatabase/neon/pull/10269 we have a vanilla postgres in the system path -- anything that wants a postgres library will use that. ## Summary of changes - Remove LD_LIBRARY_PATH assignment in Dockerfile --- Dockerfile | 5 ----- 1 file changed, 5 deletions(-) diff --git a/Dockerfile b/Dockerfile index 2c157b3b2a..d3659f917a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -103,11 +103,6 @@ RUN mkdir -p /data/.neon/ && \ > /data/.neon/pageserver.toml && \ chown -R neon:neon /data/.neon -# When running a binary that links with libpq, default to using our most recent postgres version. Binaries -# that want a particular postgres version will select it explicitly: this is just a default. -ENV LD_LIBRARY_PATH=/usr/local/v${DEFAULT_PG_VERSION}/lib - - VOLUME ["/data"] USER neon EXPOSE 6400 From 030ab1c0e8dc9f4e20b118ee77361a4b34918652 Mon Sep 17 00:00:00 2001 From: Arseny Sher Date: Thu, 9 Jan 2025 15:26:17 +0300 Subject: [PATCH 012/111] TLA+ spec for safekeeper membership change (#9966) ## Problem We want to define the algorithm for safekeeper membership change. ## Summary of changes Add spec for it, several models and logs of checking them. ref https://github.com/neondatabase/neon/issues/8699 --- .../spec/MCProposerAcceptorReconfig.tla | 41 ++ safekeeper/spec/MCProposerAcceptorStatic.tla | 3 + safekeeper/spec/ProposerAcceptorReconfig.tla | 350 ++++++++++++++++++ safekeeper/spec/ProposerAcceptorStatic.tla | 234 +++++++----- safekeeper/spec/modelcheck.sh | 3 + ...roposerAcceptorReconfig_p2_a2_t2_l2_c3.cfg | 21 ++ ...roposerAcceptorReconfig_p2_a2_t2_l2_c5.cfg | 19 + ...roposerAcceptorReconfig_p2_a3_t2_l2_c3.cfg | 20 + ...roposerAcceptorReconfig_p2_a4_t2_l2_c3.cfg | 19 + safekeeper/spec/remove_interm_progress.awk | 25 ++ safekeeper/spec/remove_interm_progress.sh | 3 + ...2_a2_t2_l2_c3.cfg-2024-12-11--04-24-12.log | 65 ++++ ...2_a2_t2_l2_c5.cfg-2024-12-11--04-26-11.log | 64 ++++ 13 files changed, 784 insertions(+), 83 deletions(-) create mode 100644 safekeeper/spec/MCProposerAcceptorReconfig.tla create mode 100644 safekeeper/spec/ProposerAcceptorReconfig.tla create mode 100644 safekeeper/spec/models/MCProposerAcceptorReconfig_p2_a2_t2_l2_c3.cfg create mode 100644 safekeeper/spec/models/MCProposerAcceptorReconfig_p2_a2_t2_l2_c5.cfg create mode 100644 safekeeper/spec/models/MCProposerAcceptorReconfig_p2_a3_t2_l2_c3.cfg create mode 100644 safekeeper/spec/models/MCProposerAcceptorReconfig_p2_a4_t2_l2_c3.cfg create mode 100644 safekeeper/spec/remove_interm_progress.awk create mode 100755 safekeeper/spec/remove_interm_progress.sh create mode 100644 safekeeper/spec/tlc-results/MCProposerAcceptorReconfig.tla-MCProposerAcceptorReconfig_p2_a2_t2_l2_c3.cfg-2024-12-11--04-24-12.log create mode 100644 safekeeper/spec/tlc-results/MCProposerAcceptorReconfig.tla-MCProposerAcceptorReconfig_p2_a2_t2_l2_c5.cfg-2024-12-11--04-26-11.log diff --git a/safekeeper/spec/MCProposerAcceptorReconfig.tla b/safekeeper/spec/MCProposerAcceptorReconfig.tla new file mode 100644 index 0000000000..a4b25e383a --- /dev/null +++ b/safekeeper/spec/MCProposerAcceptorReconfig.tla @@ -0,0 +1,41 @@ +---- MODULE MCProposerAcceptorReconfig ---- +EXTENDS TLC, ProposerAcceptorReconfig + +\* Augments the spec with model checking constraints. + +\* It slightly duplicates MCProposerAcceptorStatic, but we can't EXTENDS it +\* because it EXTENDS ProposerAcceptorStatic in turn. The duplication isn't big +\* anyway. + +\* For model checking. +CONSTANTS + max_entries, \* model constraint: max log entries acceptor/proposer can hold + max_term, \* model constraint: max allowed term + max_generation \* mode constraint: max config generation + +ASSUME max_entries \in Nat /\ max_term \in Nat /\ max_generation \in Nat + +\* Model space constraint. +StateConstraint == /\ \A p \in proposers: + /\ prop_state[p].term <= max_term + /\ Len(prop_state[p].wal) <= max_entries + /\ conf_store.generation <= max_generation + +\* Sets of proposers and acceptors and symmetric because we don't take any +\* actions depending on some concrete proposer/acceptor (like IF p = p1 THEN +\* ...) +ProposerAcceptorSymmetry == Permutations(proposers) \union Permutations(acceptors) + +\* enforce order of the vars in the error trace with ALIAS +\* Note that ALIAS is supported only since version 1.8.0 which is pre-release +\* as of writing this. +Alias == [ + prop_state |-> prop_state, + prop_conf |-> prop_conf, + acc_state |-> acc_state, + acc_conf |-> acc_conf, + committed |-> committed, + conf_store |-> conf_store + ] + +==== diff --git a/safekeeper/spec/MCProposerAcceptorStatic.tla b/safekeeper/spec/MCProposerAcceptorStatic.tla index be3d99c697..b4eca1965a 100644 --- a/safekeeper/spec/MCProposerAcceptorStatic.tla +++ b/safekeeper/spec/MCProposerAcceptorStatic.tla @@ -3,6 +3,9 @@ EXTENDS TLC, ProposerAcceptorStatic \* Augments the spec with model checking constraints. +\* Note that MCProposerAcceptorReconfig duplicates it and might need to +\* be updated as well. + \* For model checking. CONSTANTS max_entries, \* model constraint: max log entries acceptor/proposer can hold diff --git a/safekeeper/spec/ProposerAcceptorReconfig.tla b/safekeeper/spec/ProposerAcceptorReconfig.tla new file mode 100644 index 0000000000..78de231a39 --- /dev/null +++ b/safekeeper/spec/ProposerAcceptorReconfig.tla @@ -0,0 +1,350 @@ +---- MODULE ProposerAcceptorReconfig ---- + +(* + Spec for https://github.com/neondatabase/neon/blob/538e2312a617c65d489d391892c70b2e4d7407b5/docs/rfcs/035-safekeeper-dynamic-membership-change.md + + Simplifications: + - The ones inherited from ProposerAcceptorStatic. + - We don't model transient state of the configuration change driver process + (storage controller in the implementation). Its actions StartChange and FinishChange + are taken based on the persistent state of safekeepers and conf store. The + justification for that is the following: once new configuration n is + created (e.g with StartChange or FinishChange), any old configuration + change driver working on older conf < n will never be able to commit + it to the conf store because it is protected by CAS. The + propagation of these older confs is still possible though, and + spec allows to do it through acceptors. + Plus the model is already pretty huge. + - Previous point also means that the FinishChange action is + based only on the current state of safekeepers, not from + the past. That's ok because while individual + acceptor may go down, + quorum one never does. So the FinishChange + condition which collects max of the quorum may get + only more strict over time. + + The invariants expectedly break if any of FinishChange + required conditions are removed. +*) + +EXTENDS Integers, Sequences, FiniteSets, TLC + +VARIABLES + \* state which is the same in the static spec + prop_state, + acc_state, + committed, + elected_history, + \* reconfiguration only state + prop_conf, \* prop_conf[p] is current configuration of proposer p + acc_conf, \* acc_conf[a] is current configuration of acceptor a + conf_store \* configuration in the configuration store. + +CONSTANT + acceptors, + proposers + +CONSTANT NULL + +\* Import ProposerAcceptorStatic under PAS. +\* +\* Note that all vars and consts are named the same and thus substituted +\* implicitly. +PAS == INSTANCE ProposerAcceptorStatic + +\******************************************************************************** +\* Helpers +\******************************************************************************** + +\******************************************************************************** +\* Type assertion +\******************************************************************************** + +\* Is c a valid config? +IsConfig(c) == + /\ DOMAIN c = {"generation", "members", "newMembers"} + \* Unique id of the configuration. + /\ c.generation \in Nat + /\ c.members \in SUBSET acceptors + \* newMembers is NULL when it is not a joint conf. + /\ \/ c.newMembers = NULL + \/ c.newMembers \in SUBSET acceptors + +TypeOk == + /\ PAS!TypeOk + /\ \A p \in proposers: IsConfig(prop_conf[p]) + /\ \A a \in acceptors: IsConfig(acc_conf[a]) + /\ IsConfig(conf_store) + +\******************************************************************************** +\* Initial +\******************************************************************************** + +Init == + /\ PAS!Init + /\ \E init_members \in SUBSET acceptors: + LET init_conf == [generation |-> 1, members |-> init_members, newMembers |-> NULL] IN + \* refer to RestartProposer why it is not NULL + /\ prop_conf = [p \in proposers |-> init_conf] + /\ acc_conf = [a \in acceptors |-> init_conf] + /\ conf_store = init_conf + \* We could start with anything, but to reduce state space state with + \* the most reasonable total acceptors - 1 conf size, which e.g. + \* makes basic {a1} -> {a2} change in {a1, a2} acceptors and {a1, a2, + \* a3} -> {a2, a3, a4} in {a1, a2, a3, a4} acceptors models even in + \* the smallest models with single change. + /\ Cardinality(init_members) = Cardinality(acceptors) - 1 + +\******************************************************************************** +\* Actions +\******************************************************************************** + +\* Proposer p loses all state, restarting. In the static spec we bump restarted +\* proposer term to max of some quorum + 1 which is a minimal term which can win +\* election. With reconfigurations it's harder to calculate such a term, so keep +\* it simple and take random acceptor one + 1. +\* +\* Also make proposer to adopt configuration of another random acceptor. In the +\* impl proposer starts with NULL configuration until handshake with first +\* acceptor. Removing this NULL special case makes the spec a bit simpler. +RestartProposer(p) == + /\ \E a \in acceptors: PAS!RestartProposerWithTerm(p, acc_state[a].term + 1) + /\ \E a \in acceptors: prop_conf' = [prop_conf EXCEPT ![p] = acc_conf[a]] + /\ UNCHANGED <> + +\* Acceptor a immediately votes for proposer p. +Vote(p, a) == + \* Configuration must be the same. + /\ prop_conf[p].generation = acc_conf[a].generation + \* And a is expected be a member of it. This is likely redundant as long as + \* becoming leader checks membership (though vote also contributes to max + \* calculation). + /\ \/ a \in prop_conf[p].members + \/ (prop_conf[p].newMembers /= NULL) /\ (a \in prop_conf[p].newMembers) + /\ PAS!Vote(p, a) + /\ UNCHANGED <> + +\* Proposer p gets elected. +BecomeLeader(p) == + /\ prop_state[p].state = "campaign" + \* Votes must form quorum in both sets (if the newMembers exists). + /\ PAS!FormsQuorum(DOMAIN prop_state[p].votes, prop_conf[p].members) + /\ \/ prop_conf[p].newMembers = NULL + \* TLA+ disjunction evaluation doesn't short-circuit for a good reason: + \* https://groups.google.com/g/tlaplus/c/U6tOJ4dsjVM/m/UdOznPCVBwAJ + \* so repeat the null check. + \/ (prop_conf[p].newMembers /= NULL) /\ (PAS!FormsQuorum(DOMAIN prop_state[p].votes, prop_conf[p].newMembers)) + \* DoBecomeLeader will copy WAL of the highest voter to proposer's WAL, so + \* ensure its conf is still the same. In the impl WAL fetching also has to + \* check the configuration. + /\ prop_conf[p].generation = acc_conf[PAS!MaxVoteAcc(p)].generation + /\ \A a \in DOMAIN prop_state[p].votes: prop_conf[p].generation = acc_conf[a].generation + /\ PAS!DoBecomeLeader(p) + /\ UNCHANGED <> + +UpdateTerm(p, a) == + /\ PAS!UpdateTerm(p, a) + /\ UNCHANGED <> + +TruncateWal(p, a) == + /\ prop_state[p].state = "leader" + \* Configuration must be the same. + /\ prop_conf[p].generation = acc_conf[a].generation + /\ PAS!TruncateWal(p, a) + /\ UNCHANGED <> + +NewEntry(p) == + /\ PAS!NewEntry(p) + /\ UNCHANGED <> + +AppendEntry(p, a) == + /\ prop_state[p].state = "leader" + \* Configuration must be the same. + /\ prop_conf[p].generation = acc_conf[a].generation + \* And a is member of it. Ignoring this likely wouldn't hurt, but not useful + \* either. + /\ \/ a \in prop_conf[p].members + \/ (prop_conf[p].newMembers /= NULL) /\ (a \in prop_conf[p].newMembers) + /\ PAS!AppendEntry(p, a) + /\ UNCHANGED <> + +\* see PAS!CommitEntries for comments. +CommitEntries(p) == + /\ prop_state[p].state = "leader" + /\ \E q1 \in PAS!AllMinQuorums(prop_conf[p].members): + LET q1_commit_lsn == PAS!QuorumCommitLsn(p, q1) IN + \* Configuration must be the same. + /\ \A a \in q1: prop_conf[p].generation = acc_conf[a].generation + /\ q1_commit_lsn /= NULL + \* We must collect acks from both quorums, if newMembers is present. + /\ IF prop_conf[p].newMembers = NULL THEN + PAS!DoCommitEntries(p, q1_commit_lsn) + ELSE + \E q2 \in PAS!AllMinQuorums(prop_conf[p].newMembers): + LET q2_commit_lsn == PAS!QuorumCommitLsn(p, q2) IN + \* Configuration must be the same. + /\ \A a \in q1: prop_conf[p].generation = acc_conf[a].generation + /\ q2_commit_lsn /= NULL + /\ PAS!DoCommitEntries(p, PAS!Min(q1_commit_lsn, q2_commit_lsn)) + /\ UNCHANGED <> + +\* Proposer p adopts higher conf c from conf store or from some acceptor. +ProposerSwitchConf(p) == + /\ \E c \in ({conf_store} \union {acc_conf[a]: a \in acceptors}): + \* p's conf is lower than c. + /\ (c.generation > prop_conf[p].generation) + \* We allow to bump conf without restart only when wp is already elected. + \* If it isn't, the votes it has already collected are from the previous + \* configuration and can't be used. + \* + \* So if proposer is in 'campaign' in the impl we would restart preserving + \* conf and increasing term. In the spec this transition is already covered + \* by more a generic RestartProposer, so we don't specify it here. + /\ prop_state[p].state = "leader" + /\ prop_conf' = [prop_conf EXCEPT ![p] = c] + /\ UNCHANGED <> + +\* Do CAS on the conf store, starting change into the new_members conf. +StartChange(new_members) == + \* Possible only if we don't already have the change in progress. + /\ conf_store.newMembers = NULL + \* Not necessary, but reduces space a bit. + /\ new_members /= conf_store.members + /\ conf_store' = [generation |-> conf_store.generation + 1, members |-> conf_store.members, newMembers |-> new_members] + /\ UNCHANGED <> + +\* Acceptor's last_log_term. +AccLastLogTerm(acc) == + PAS!LastLogTerm(PAS!AcceptorTermHistory(acc)) + +\* Do CAS on the conf store, transferring joint conf into the newMembers only. +FinishChange == + \* have joint conf + /\ conf_store.newMembers /= NULL + \* The conditions for finishing the change are: + /\ \E qo \in PAS!AllMinQuorums(conf_store.members): + \* 1) Old majority must be aware of the joint conf. + \* Note: generally the driver can't know current acceptor + \* generation, it can only know that it once had been the + \* expected one, but it might have advanced since then. + \* But as explained at the top of the file if acceptor gen + \* advanced, FinishChange will never be able to complete + \* due to CAS anyway. We use strict equality here because + \* that's what makes sense conceptually (old driver should + \* abandon its attempt if it observes that conf has advanced). + /\ \A a \in qo: conf_store.generation = acc_conf[a].generation + \* 2) New member set must have log synced, i.e. some its majority needs + \* to have at least as high as max of some + \* old majority. + \* 3) Term must be synced, i.e. some majority of the new set must + \* have term >= than max term of some old majority. + \* This ensures that two leaders are never elected with the same + \* term even after config change (which would be bad unless we treat + \* generation as a part of term which we don't). + \* 4) A majority of the new set must be aware of the joint conf. + \* This allows to safely destoy acceptor state if it is not a + \* member of its current conf (which is useful for cleanup after + \* migration as well as for aborts). + /\ LET sync_pos == PAS!MaxTermLsn({[term |-> AccLastLogTerm(a), lsn |-> PAS!FlushLsn(a)]: a \in qo}) + sync_term == PAS!Maximum({acc_state[a].term: a \in qo}) + IN + \E qn \in PAS!AllMinQuorums(conf_store.newMembers): + \A a \in qn: + /\ PAS!TermLsnGE([term |-> AccLastLogTerm(a), lsn |-> PAS!FlushLsn(a)], sync_pos) + /\ acc_state[a].term >= sync_term + \* The same note as above about strict equality applies here. + /\ conf_store.generation = acc_conf[a].generation + /\ conf_store' = [generation |-> conf_store.generation + 1, members |-> conf_store.newMembers, newMembers |-> NULL] + /\ UNCHANGED <> + +\* Do CAS on the conf store, aborting the change in progress. +AbortChange == + \* have joint conf + /\ conf_store.newMembers /= NULL + /\ conf_store' = [generation |-> conf_store.generation + 1, members |-> conf_store.members, newMembers |-> NULL] + /\ UNCHANGED <> + +\* Acceptor a switches to higher configuration from the conf store +\* or from some proposer. +AccSwitchConf(a) == + /\ \E c \in ({conf_store} \union {prop_conf[p]: p \in proposers}): + /\ acc_conf[a].generation < c.generation + /\ acc_conf' = [acc_conf EXCEPT ![a] = c] + /\ UNCHANGED <> + +\* Nuke all acceptor state if it is not a member of its current conf. Models +\* cleanup after migration/abort. +AccReset(a) == + /\ \/ (acc_conf[a].newMembers = NULL) /\ (a \notin acc_conf[a].members) + \/ (acc_conf[a].newMembers /= NULL) /\ (a \notin (acc_conf[a].members \union acc_conf[a].newMembers)) + /\ acc_state' = [acc_state EXCEPT ![a] = PAS!InitAcc] + \* Set nextSendLsn to `a` to NULL everywhere. nextSendLsn serves as a mark + \* that elected proposer performed TruncateWal on the acceptor, which isn't + \* true anymore after state reset. In the impl local deletion is expected to + \* terminate all existing connections. + /\ prop_state' = [p \in proposers |-> [prop_state[p] EXCEPT !.nextSendLsn[a] = NULL]] + /\ UNCHANGED <> + +\******************************************************************************* +\* Final spec +\******************************************************************************* + +Next == + \/ \E p \in proposers: RestartProposer(p) + \/ \E p \in proposers: \E a \in acceptors: Vote(p, a) + \/ \E p \in proposers: BecomeLeader(p) + \/ \E p \in proposers: \E a \in acceptors: UpdateTerm(p, a) + \/ \E p \in proposers: \E a \in acceptors: TruncateWal(p, a) + \/ \E p \in proposers: NewEntry(p) + \/ \E p \in proposers: \E a \in acceptors: AppendEntry(p, a) + \/ \E p \in proposers: CommitEntries(p) + \/ \E new_members \in SUBSET acceptors: StartChange(new_members) + \/ FinishChange + \/ AbortChange + \/ \E p \in proposers: ProposerSwitchConf(p) + \/ \E a \in acceptors: AccSwitchConf(a) + \/ \E a \in acceptors: AccReset(a) + +Spec == Init /\ [][Next]_<> + +\******************************************************************************** +\* Invariants +\******************************************************************************** + +AllConfs == + {conf_store} \union {prop_conf[p]: p \in proposers} \union {acc_conf[a]: a \in acceptors} + +\* Fairly trivial (given the conf store) invariant that different configurations +\* with the same generation are never issued. +ConfigSafety == + \A c1, c2 \in AllConfs: + (c1.generation = c2.generation) => (c1 = c2) + +ElectionSafety == PAS!ElectionSafety + +ElectionSafetyFull == PAS!ElectionSafetyFull + +LogIsMonotonic == PAS!LogIsMonotonic + +LogSafety == PAS!LogSafety + +\******************************************************************************** +\* Invariants which don't need to hold, but useful for playing/debugging. +\******************************************************************************** + +\* Check that we ever switch into non joint conf. +MaxAccConf == ~ \E a \in acceptors: + /\ acc_conf[a].generation = 3 + /\ acc_conf[a].newMembers /= NULL + +CommittedNotTruncated == PAS!CommittedNotTruncated + +MaxTerm == PAS!MaxTerm + +MaxStoreConf == conf_store.generation <= 1 + +MaxAccWalLen == PAS!MaxAccWalLen + +MaxCommitLsn == PAS!MaxCommitLsn + +==== diff --git a/safekeeper/spec/ProposerAcceptorStatic.tla b/safekeeper/spec/ProposerAcceptorStatic.tla index b2d2f005db..fab085bc2e 100644 --- a/safekeeper/spec/ProposerAcceptorStatic.tla +++ b/safekeeper/spec/ProposerAcceptorStatic.tla @@ -18,7 +18,7 @@ \* - old WAL is immediately copied to proposer on its election, without on-demand fetch later. \* Some ideas how to break it to play around to get a feeling: -\* - replace Quorums with BadQuorums. +\* - replace Quorum with BadQuorum. \* - remove 'don't commit entries from previous terms separately' rule in \* CommitEntries and observe figure 8 from the raft paper. \* With p2a3t4l4 32 steps error was found in 1h on 80 cores. @@ -69,16 +69,26 @@ Upsert(f, k, v, l(_)) == \***************** -NumAccs == Cardinality(acceptors) +\* Does set of acceptors `acc_set` form the quorum in the member set `members`? +\* Acceptors not from `members` are excluded (matters only for reconfig). +FormsQuorum(acc_set, members) == + Cardinality(acc_set \intersect members) >= (Cardinality(members) \div 2 + 1) -\* does acc_set form the quorum? -Quorum(acc_set) == Cardinality(acc_set) >= (NumAccs \div 2 + 1) -\* all quorums of acceptors -Quorums == {subset \in SUBSET acceptors: Quorum(subset)} +\* Like FormsQuorum, but for minimal quorum. +FormsMinQuorum(acc_set, members) == + Cardinality(acc_set \intersect members) = (Cardinality(members) \div 2 + 1) -\* For substituting Quorums and seeing what happens. -BadQuorum(acc_set) == Cardinality(acc_set) >= (NumAccs \div 2) -BadQuorums == {subset \in SUBSET acceptors: BadQuorum(subset)} +\* All sets of acceptors forming minimal quorums in the member set `members`. +AllQuorums(members) == {subset \in SUBSET members: FormsQuorum(subset, members)} +AllMinQuorums(members) == {subset \in SUBSET acceptors: FormsMinQuorum(subset, members)} + +\* For substituting Quorum and seeing what happens. +FormsBadQuorum(acc_set, members) == + Cardinality(acc_set \intersect members) >= (Cardinality(members) \div 2) +FormsMinBadQuorum(acc_set, members) == + Cardinality(acc_set \intersect members) = (Cardinality(members) \div 2) +AllBadQuorums(members) == {subset \in SUBSET acceptors: FormsBadQuorum(subset, members)} +AllMinBadQuorums(members) == {subset \in SUBSET acceptors: FormsMinBadQuorum(subset, members)} \* flushLsn (end of WAL, i.e. index of next entry) of acceptor a. FlushLsn(a) == Len(acc_state[a].wal) + 1 @@ -135,10 +145,11 @@ TypeOk == /\ IsWal(prop_state[p].wal) \* Map of acceptor -> next lsn to send. It is set when truncate_wal is \* done so sending entries is allowed only after that. In the impl TCP - \* ensures this ordering. + \* ensures this ordering. We use NULL instead of missing value to use + \* EXCEPT in AccReset. /\ \A a \in DOMAIN prop_state[p].nextSendLsn: /\ a \in acceptors - /\ prop_state[p].nextSendLsn[a] \in Lsns + /\ prop_state[p].nextSendLsn[a] \in Lsns \union {NULL} /\ \A a \in acceptors: /\ DOMAIN acc_state[a] = {"term", "termHistory", "wal"} /\ acc_state[a].term \in Terms @@ -167,6 +178,19 @@ TypeOk == \* Initial \******************************************************************************** +InitAcc == + [ + \* There will be no leader in zero term, 1 is the first + \* real. + term |-> 0, + \* Again, leader in term 0 doesn't exist, but we initialize + \* term histories with it to always have common point in + \* them. Lsn is 1 because TLA+ sequences are indexed from 1 + \* (we don't want to truncate WAL out of range). + termHistory |-> << [term |-> 0, lsn |-> 1] >>, + wal |-> << >> + ] + Init == /\ prop_state = [p \in proposers |-> [ state |-> "campaign", @@ -174,19 +198,9 @@ Init == votes |-> EmptyF, termHistory |-> << >>, wal |-> << >>, - nextSendLsn |-> EmptyF + nextSendLsn |-> [a \in acceptors |-> NULL] ]] - /\ acc_state = [a \in acceptors |-> [ - \* There will be no leader in zero term, 1 is the first - \* real. - term |-> 0, - \* Again, leader in term 0 doesn't exist, but we initialize - \* term histories with it to always have common point in - \* them. Lsn is 1 because TLA+ sequences are indexed from 1 - \* (we don't want to truncate WAL out of range). - termHistory |-> << [term |-> 0, lsn |-> 1] >>, - wal |-> << >> - ]] + /\ acc_state = [a \in acceptors |-> InitAcc] /\ committed = {} /\ elected_history = EmptyF @@ -195,23 +209,35 @@ Init == \* Actions \******************************************************************************** -\* Proposer loses all state. +RestartProposerWithTerm(p, new_term) == + /\ prop_state' = [prop_state EXCEPT ![p].state = "campaign", + ![p].term = new_term, + ![p].votes = EmptyF, + ![p].termHistory = << >>, + ![p].wal = << >>, + ![p].nextSendLsn = [a \in acceptors |-> NULL]] + /\ UNCHANGED <> + +\* Proposer p loses all state, restarting. \* For simplicity (and to reduct state space), we assume it immediately gets \* current state from quorum q of acceptors determining the term he will request \* to vote for. -RestartProposer(p, q) == - /\ Quorum(q) - /\ LET new_term == Maximum({acc_state[a].term : a \in q}) + 1 IN - /\ prop_state' = [prop_state EXCEPT ![p].state = "campaign", - ![p].term = new_term, - ![p].votes = EmptyF, - ![p].termHistory = << >>, - ![p].wal = << >>, - ![p].nextSendLsn = EmptyF] - /\ UNCHANGED <> +RestartProposer(p) == + \E q \in AllQuorums(acceptors): + LET new_term == Maximum({acc_state[a].term : a \in q}) + 1 IN + RestartProposerWithTerm(p, new_term) \* Term history of acceptor a's WAL: the one saved truncated to contain only <= -\* local FlushLsn entries. +\* local FlushLsn entries. Note that FlushLsn is the end LSN of the last entry +\* (and begin LSN of the next). The mental model for non strict comparison is +\* that once proposer is elected it immediately writes log record with zero +\* length. This allows leader to commit existing log without writing any new +\* entries. For example, assume acceptor has WAL +\* 1.1, 1.2 +\* written by prop with term 1; its current +\* is <1, 3>. Now prop with term 2 and max vote from this acc is elected. +\* Once TruncateWAL is done, becomes <2, 3> +\* without any new records explicitly written. AcceptorTermHistory(a) == SelectSeq(acc_state[a].termHistory, LAMBDA th_entry: th_entry.lsn <= FlushLsn(a)) @@ -230,35 +256,52 @@ Vote(p, a) == \* Get lastLogTerm from term history th. LastLogTerm(th) == th[Len(th)].term +\* Compares pairs: returns true if tl1 >= tl2. +TermLsnGE(tl1, tl2) == + /\ tl1.term >= tl2.term + /\ (tl1.term = tl2.term => tl1.lsn >= tl2.lsn) + +\* Choose max pair in the non empty set of them. +MaxTermLsn(term_lsn_set) == + CHOOSE max_tl \in term_lsn_set: \A tl \in term_lsn_set: TermLsnGE(max_tl, tl) + +\* Find acceptor with the highest vote in proposer p's votes. +MaxVoteAcc(p) == + CHOOSE a \in DOMAIN prop_state[p].votes: + LET a_vote == prop_state[p].votes[a] + a_vote_term_lsn == [term |-> LastLogTerm(a_vote.termHistory), lsn |-> a_vote.flushLsn] + vote_term_lsns == {[term |-> LastLogTerm(v.termHistory), lsn |-> v.flushLsn]: v \in Range(prop_state[p].votes)} + IN + a_vote_term_lsn = MaxTermLsn(vote_term_lsns) + +\* Workhorse for BecomeLeader. +\* Assumes the check prop_state[p] votes is quorum has been done *outside*. +DoBecomeLeader(p) == + LET + \* Find acceptor with the highest vote. + max_vote_acc == MaxVoteAcc(p) + max_vote == prop_state[p].votes[max_vote_acc] + prop_th == Append(max_vote.termHistory, [term |-> prop_state[p].term, lsn |-> max_vote.flushLsn]) + IN + \* We copy all log preceding proposer's term from the max vote node so + \* make sure it is still on one term with us. This is a model + \* simplification which can be removed, in impl we fetch WAL on demand + \* from safekeeper which has it later. Note though that in case of on + \* demand fetch we must check on donor not only term match, but that + \* truncate_wal had already been done (if it is not max_vote_acc). + /\ acc_state[max_vote_acc].term = prop_state[p].term + /\ prop_state' = [prop_state EXCEPT ![p].state = "leader", + ![p].termHistory = prop_th, + ![p].wal = acc_state[max_vote_acc].wal + ] + /\ elected_history' = Upsert(elected_history, prop_state[p].term, 1, LAMBDA c: c + 1) + /\ UNCHANGED <> + \* Proposer p gets elected. BecomeLeader(p) == /\ prop_state[p].state = "campaign" - /\ Quorum(DOMAIN prop_state[p].votes) - /\ LET - \* Find acceptor with the highest vote. - max_vote_acc == - CHOOSE a \in DOMAIN prop_state[p].votes: - LET v == prop_state[p].votes[a] - IN \A v2 \in Range(prop_state[p].votes): - /\ LastLogTerm(v.termHistory) >= LastLogTerm(v2.termHistory) - /\ (LastLogTerm(v.termHistory) = LastLogTerm(v2.termHistory) => v.flushLsn >= v2.flushLsn) - max_vote == prop_state[p].votes[max_vote_acc] - prop_th == Append(max_vote.termHistory, [term |-> prop_state[p].term, lsn |-> max_vote.flushLsn]) - IN - \* We copy all log preceding proposer's term from the max vote node so - \* make sure it is still on one term with us. This is a model - \* simplification which can be removed, in impl we fetch WAL on demand - \* from safekeeper which has it later. Note though that in case of on - \* demand fetch we must check on donor not only term match, but that - \* truncate_wal had already been done (if it is not max_vote_acc). - /\ acc_state[max_vote_acc].term = prop_state[p].term - /\ prop_state' = [prop_state EXCEPT ![p].state = "leader", - ![p].termHistory = prop_th, - ![p].wal = acc_state[max_vote_acc].wal - ] - /\ elected_history' = Upsert(elected_history, prop_state[p].term, 1, LAMBDA c: c + 1) - /\ UNCHANGED <> - + /\ FormsQuorum(DOMAIN prop_state[p].votes, acceptors) + /\ DoBecomeLeader(p) \* Acceptor a learns about elected proposer p's term. In impl it matches to \* VoteRequest/VoteResponse exchange when leader is already elected and is not @@ -287,10 +330,11 @@ FindHighestCommonPoint(prop_th, acc_th, acc_flush_lsn) == IN [term |-> last_common_term, lsn |-> Min(acc_common_term_end, prop_common_term_end)] -\* Elected proposer p immediately truncates WAL (and term history) of acceptor a -\* before starting streaming. Establishes nextSendLsn for a. +\* Elected proposer p immediately truncates WAL (and sets term history) of +\* acceptor a before starting streaming. Establishes nextSendLsn for a. \* -\* In impl this happens at each reconnection, here we also allow to do it multiple times. +\* In impl this happens at each reconnection, here we also allow to do it +\* multiple times. TruncateWal(p, a) == /\ prop_state[p].state = "leader" /\ acc_state[a].term = prop_state[p].term @@ -321,8 +365,8 @@ NewEntry(p) == AppendEntry(p, a) == /\ prop_state[p].state = "leader" /\ acc_state[a].term = prop_state[p].term - /\ a \in DOMAIN prop_state[p].nextSendLsn \* did TruncateWal - /\ prop_state[p].nextSendLsn[a] <= Len(prop_state[p].wal) \* have smth to send + /\ prop_state[p].nextSendLsn[a] /= NULL \* did TruncateWal + /\ prop_state[p].nextSendLsn[a] <= Len(prop_state[p].wal) \* have smth to send /\ LET send_lsn == prop_state[p].nextSendLsn[a] entry == prop_state[p].wal[send_lsn] @@ -337,41 +381,65 @@ AppendEntry(p, a) == PropStartLsn(p) == IF prop_state[p].state = "leader" THEN prop_state[p].termHistory[Len(prop_state[p].termHistory)].lsn ELSE NULL -\* Proposer p commits all entries it can using quorum q. Note that unlike -\* will62794/logless-reconfig this allows to commit entries from previous terms -\* (when conditions for that are met). -CommitEntries(p, q) == - /\ prop_state[p].state = "leader" - /\ \A a \in q: +\* LSN which can be committed by proposer p using min quorum q (check that q +\* forms quorum must have been done outside). NULL if there is none. +QuorumCommitLsn(p, q) == + IF + /\ prop_state[p].state = "leader" + /\ \A a \in q: + \* Without explicit responses to appends this ensures that append + \* up to FlushLsn has been accepted. /\ acc_state[a].term = prop_state[p].term \* nextSendLsn existence means TruncateWal has happened, it ensures \* acceptor's WAL (and FlushLsn) are from proper proposer's history. \* Alternatively we could compare LastLogTerm here, but that's closer to \* what we do in the impl (we check flushLsn in AppendResponse, but \* AppendRequest is processed only if HandleElected handling was good). - /\ a \in DOMAIN prop_state[p].nextSendLsn - \* Now find the LSN present on all the quorum. - /\ LET quorum_lsn == Minimum({FlushLsn(a): a \in q}) IN - \* This is the basic Raft rule of not committing entries from previous - \* terms except along with current term entry (commit them only when - \* quorum recovers, i.e. last_log_term on it reaches leader's term). - /\ quorum_lsn >= PropStartLsn(p) - /\ committed' = committed \cup {[term |-> prop_state[p].wal[lsn], lsn |-> lsn]: lsn \in 1..(quorum_lsn - 1)} - /\ UNCHANGED <> + /\ prop_state[p].nextSendLsn[a] /= NULL + THEN + \* Now find the LSN present on all the quorum. + LET quorum_lsn == Minimum({FlushLsn(a): a \in q}) IN + \* This is the basic Raft rule of not committing entries from previous + \* terms except along with current term entry (commit them only when + \* quorum recovers, i.e. last_log_term on it reaches leader's term). + IF quorum_lsn >= PropStartLsn(p) THEN + quorum_lsn + ELSE + NULL + ELSE + NULL + +\* Commit all entries on proposer p with record lsn < commit_lsn. +DoCommitEntries(p, commit_lsn) == + /\ committed' = committed \cup {[term |-> prop_state[p].wal[lsn], lsn |-> lsn]: lsn \in 1..(commit_lsn - 1)} + /\ UNCHANGED <> + +\* Proposer p commits all entries it can using some quorum. Note that unlike +\* will62794/logless-reconfig this allows to commit entries from previous terms +\* (when conditions for that are met). +CommitEntries(p) == + /\ prop_state[p].state = "leader" + \* Using min quorums here is better because 1) QuorumCommitLsn for + \* simplicity checks min across all accs in q. 2) it probably makes + \* evaluation faster. + /\ \E q \in AllMinQuorums(acceptors): + LET commit_lsn == QuorumCommitLsn(p, q) IN + /\ commit_lsn /= NULL + /\ DoCommitEntries(p, commit_lsn) \******************************************************************************* \* Final spec \******************************************************************************* Next == - \/ \E q \in Quorums: \E p \in proposers: RestartProposer(p, q) + \/ \E p \in proposers: RestartProposer(p) \/ \E p \in proposers: \E a \in acceptors: Vote(p, a) \/ \E p \in proposers: BecomeLeader(p) \/ \E p \in proposers: \E a \in acceptors: UpdateTerm(p, a) \/ \E p \in proposers: \E a \in acceptors: TruncateWal(p, a) \/ \E p \in proposers: NewEntry(p) \/ \E p \in proposers: \E a \in acceptors: AppendEntry(p, a) - \/ \E q \in Quorums: \E p \in proposers: CommitEntries(p, q) + \/ \E p \in proposers: CommitEntries(p) Spec == Init /\ [][Next]_<> diff --git a/safekeeper/spec/modelcheck.sh b/safekeeper/spec/modelcheck.sh index 21ead7dad8..0084a8c638 100755 --- a/safekeeper/spec/modelcheck.sh +++ b/safekeeper/spec/modelcheck.sh @@ -2,6 +2,7 @@ # Usage: ./modelcheck.sh , e.g. # ./modelcheck.sh models/MCProposerAcceptorStatic_p2_a3_t3_l3.cfg MCProposerAcceptorStatic.tla +# ./modelcheck.sh models/MCProposerAcceptorReconfig_p2_a3_t3_l3_c3.cfg MCProposerAcceptorReconfig.tla CONFIG=$1 SPEC=$2 @@ -12,6 +13,7 @@ mkdir -p "tlc-results" CONFIG_FILE=$(basename -- "$CONFIG") outfilename="$SPEC-${CONFIG_FILE}-$(date --utc +%Y-%m-%d--%H-%M-%S)".log outfile="tlc-results/$outfilename" +echo "saving results to $outfile" touch $outfile # Save some info about the run. @@ -45,5 +47,6 @@ echo "" >> $outfile # https://docs.tlapl.us/codebase:architecture#fingerprint_sets_fpsets # # Add -simulate to run in infinite simulation mode. +# -coverage 1 is useful for profiling (check how many times actions are taken). java -Xmx$MEM -XX:MaxDirectMemorySize=$MEM -XX:+UseParallelGC -Dtlc2.tool.fp.FPSet.impl=tlc2.tool.fp.OffHeapDiskFPSet \ -cp "${TOOLSPATH}" tlc2.TLC $SPEC -config $CONFIG -workers auto -gzip | tee -a $outfile diff --git a/safekeeper/spec/models/MCProposerAcceptorReconfig_p2_a2_t2_l2_c3.cfg b/safekeeper/spec/models/MCProposerAcceptorReconfig_p2_a2_t2_l2_c3.cfg new file mode 100644 index 0000000000..8d34751083 --- /dev/null +++ b/safekeeper/spec/models/MCProposerAcceptorReconfig_p2_a2_t2_l2_c3.cfg @@ -0,0 +1,21 @@ +CONSTANTS +NULL = NULL +proposers = {p1, p2} +acceptors = {a1, a2} +max_term = 2 +max_entries = 2 +max_generation = 3 +SPECIFICATION Spec +CONSTRAINT StateConstraint +INVARIANT +TypeOk +ConfigSafety +ElectionSafetyFull +LogIsMonotonic +LogSafety +\* As its comment explains generally it is not expected to hold, but +\* in such small model it is true. +CommittedNotTruncated +SYMMETRY ProposerAcceptorSymmetry +CHECK_DEADLOCK FALSE +ALIAS Alias diff --git a/safekeeper/spec/models/MCProposerAcceptorReconfig_p2_a2_t2_l2_c5.cfg b/safekeeper/spec/models/MCProposerAcceptorReconfig_p2_a2_t2_l2_c5.cfg new file mode 100644 index 0000000000..eb7e0768ff --- /dev/null +++ b/safekeeper/spec/models/MCProposerAcceptorReconfig_p2_a2_t2_l2_c5.cfg @@ -0,0 +1,19 @@ +CONSTANTS +NULL = NULL +proposers = {p1, p2} +acceptors = {a1, a2} +max_term = 2 +max_entries = 2 +max_generation = 5 +SPECIFICATION Spec +CONSTRAINT StateConstraint +INVARIANT +TypeOk +ConfigSafety +ElectionSafetyFull +LogIsMonotonic +LogSafety +CommittedNotTruncated +SYMMETRY ProposerAcceptorSymmetry +CHECK_DEADLOCK FALSE +ALIAS Alias diff --git a/safekeeper/spec/models/MCProposerAcceptorReconfig_p2_a3_t2_l2_c3.cfg b/safekeeper/spec/models/MCProposerAcceptorReconfig_p2_a3_t2_l2_c3.cfg new file mode 100644 index 0000000000..b5fae13880 --- /dev/null +++ b/safekeeper/spec/models/MCProposerAcceptorReconfig_p2_a3_t2_l2_c3.cfg @@ -0,0 +1,20 @@ +CONSTANTS +NULL = NULL +proposers = {p1, p2} +acceptors = {a1, a2, a3} +max_term = 2 +max_entries = 2 +max_generation = 3 +SPECIFICATION Spec +CONSTRAINT StateConstraint +INVARIANT +TypeOk +ConfigSafety +ElectionSafetyFull +LogIsMonotonic +LogSafety +CommittedNotTruncated +SYMMETRY ProposerAcceptorSymmetry +CHECK_DEADLOCK FALSE +ALIAS Alias + diff --git a/safekeeper/spec/models/MCProposerAcceptorReconfig_p2_a4_t2_l2_c3.cfg b/safekeeper/spec/models/MCProposerAcceptorReconfig_p2_a4_t2_l2_c3.cfg new file mode 100644 index 0000000000..71af9fa367 --- /dev/null +++ b/safekeeper/spec/models/MCProposerAcceptorReconfig_p2_a4_t2_l2_c3.cfg @@ -0,0 +1,19 @@ +CONSTANTS +NULL = NULL +proposers = {p1, p2} +acceptors = {a1, a2, a3, a4} +max_term = 2 +max_entries = 2 +max_generation = 3 +SPECIFICATION Spec +CONSTRAINT StateConstraint +INVARIANT +TypeOk +ElectionSafetyFull +LogIsMonotonic +LogSafety +CommittedNotTruncated +SYMMETRY ProposerAcceptorSymmetry +CHECK_DEADLOCK FALSE +ALIAS Alias + diff --git a/safekeeper/spec/remove_interm_progress.awk b/safekeeper/spec/remove_interm_progress.awk new file mode 100644 index 0000000000..6203f6fa4f --- /dev/null +++ b/safekeeper/spec/remove_interm_progress.awk @@ -0,0 +1,25 @@ +# Print all lines, but thin out lines starting with Progress: +# leave only first and last 5 ones in the beginning, and only 1 of 1440 +# of others (once a day). +# Also remove checkpointing logs. +{ + lines[NR] = $0 +} +$0 ~ /^Progress/ { + ++pcount +} +END { + progress_idx = 0 + for (i = 1; i <= NR; i++) { + if (lines[i] ~ /^Progress/) { + if (progress_idx < 5 || progress_idx >= pcount - 5 || progress_idx % 1440 == 0) { + print lines[i] + } + progress_idx++ + } + else if (lines[i] ~ /^Checkpointing/) {} + else { + print lines[i] + } + } +} \ No newline at end of file diff --git a/safekeeper/spec/remove_interm_progress.sh b/safekeeper/spec/remove_interm_progress.sh new file mode 100755 index 0000000000..a8724a2b92 --- /dev/null +++ b/safekeeper/spec/remove_interm_progress.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +awk -f remove_interm_progress.awk $1 > $1.thin \ No newline at end of file diff --git a/safekeeper/spec/tlc-results/MCProposerAcceptorReconfig.tla-MCProposerAcceptorReconfig_p2_a2_t2_l2_c3.cfg-2024-12-11--04-24-12.log b/safekeeper/spec/tlc-results/MCProposerAcceptorReconfig.tla-MCProposerAcceptorReconfig_p2_a2_t2_l2_c3.cfg-2024-12-11--04-24-12.log new file mode 100644 index 0000000000..8aac9eb58c --- /dev/null +++ b/safekeeper/spec/tlc-results/MCProposerAcceptorReconfig.tla-MCProposerAcceptorReconfig_p2_a2_t2_l2_c3.cfg-2024-12-11--04-24-12.log @@ -0,0 +1,65 @@ +git revision: 9e386917a +Platform: Linux neon-dev-arm64-1 6.8.0-49-generic #49-Ubuntu SMP PREEMPT_DYNAMIC Sun Nov 3 21:21:58 UTC 2024 aarch64 aarch64 aarch64 GNU/Linux +CPU Info Linux: Neoverse-N1 +CPU Cores Linux: 80 +CPU Info Mac: +CPU Cores Mac: +Spec: MCProposerAcceptorReconfig.tla +Config: models/MCProposerAcceptorReconfig_p2_a2_t2_l2_c3.cfg +---- +CONSTANTS +NULL = NULL +proposers = {p1, p2} +acceptors = {a1, a2} +max_term = 2 +max_entries = 2 +max_generation = 3 +SPECIFICATION Spec +CONSTRAINT StateConstraint +INVARIANT +TypeOk +ElectionSafetyFull +LogIsMonotonic +LogSafety +\* CommittedNotTruncated +SYMMETRY ProposerAcceptorSymmetry +CHECK_DEADLOCK FALSE +ALIAS Alias + +---- + +TLC2 Version 2.20 of Day Month 20?? (rev: f68cb71) +Running breadth-first search Model-Checking with fp 99 and seed -9189733667206762985 with 35 workers on 80 cores with 27307MB heap and 30720MB offheap memory [pid: 391272] (Linux 6.8.0-49-generic aarch64, Ubuntu 21.0.5 x86_64, OffHeapDiskFPSet, DiskStateQueue). +Parsing file /home/arseny/neon2/safekeeper/spec/MCProposerAcceptorReconfig.tla +Parsing file /tmp/tlc-3211535543066978921/TLC.tla (jar:file:/home/arseny/tla2tools.jar!/tla2sany/StandardModules/TLC.tla) +Parsing file /home/arseny/neon2/safekeeper/spec/ProposerAcceptorReconfig.tla +Parsing file /tmp/tlc-3211535543066978921/_TLCTrace.tla (jar:file:/home/arseny/tla2tools.jar!/tla2sany/StandardModules/_TLCTrace.tla) +Parsing file /tmp/tlc-3211535543066978921/Integers.tla (jar:file:/home/arseny/tla2tools.jar!/tla2sany/StandardModules/Integers.tla) +Parsing file /tmp/tlc-3211535543066978921/Sequences.tla (jar:file:/home/arseny/tla2tools.jar!/tla2sany/StandardModules/Sequences.tla) +Parsing file /tmp/tlc-3211535543066978921/FiniteSets.tla (jar:file:/home/arseny/tla2tools.jar!/tla2sany/StandardModules/FiniteSets.tla) +Parsing file /tmp/tlc-3211535543066978921/Naturals.tla (jar:file:/home/arseny/tla2tools.jar!/tla2sany/StandardModules/Naturals.tla) +Parsing file /home/arseny/neon2/safekeeper/spec/ProposerAcceptorStatic.tla +Parsing file /tmp/tlc-3211535543066978921/TLCExt.tla (jar:file:/home/arseny/tla2tools.jar!/tla2sany/StandardModules/TLCExt.tla) +Semantic processing of module Naturals +Semantic processing of module Sequences +Semantic processing of module FiniteSets +Semantic processing of module TLC +Semantic processing of module Integers +Semantic processing of module ProposerAcceptorStatic +Semantic processing of module ProposerAcceptorReconfig +Semantic processing of module TLCExt +Semantic processing of module _TLCTrace +Semantic processing of module MCProposerAcceptorReconfig +Starting... (2024-12-11 04:24:13) +Computing initial states... +Finished computing initial states: 2 states generated, with 1 of them distinct at 2024-12-11 04:24:15. +Progress(16) at 2024-12-11 04:24:18: 1,427,589 states generated (1,427,589 s/min), 142,472 distinct states found (142,472 ds/min), 47,162 states left on queue. +Model checking completed. No error has been found. + Estimates of the probability that TLC did not check all reachable states + because two distinct states had the same fingerprint: + calculated (optimistic): val = 1.0E-6 + based on the actual fingerprints: val = 4.2E-8 +17746857 states generated, 1121659 distinct states found, 0 states left on queue. +The depth of the complete state graph search is 37. +The average outdegree of the complete state graph is 1 (minimum is 0, the maximum 9 and the 95th percentile is 3). +Finished in 33s at (2024-12-11 04:24:46) diff --git a/safekeeper/spec/tlc-results/MCProposerAcceptorReconfig.tla-MCProposerAcceptorReconfig_p2_a2_t2_l2_c5.cfg-2024-12-11--04-26-11.log b/safekeeper/spec/tlc-results/MCProposerAcceptorReconfig.tla-MCProposerAcceptorReconfig_p2_a2_t2_l2_c5.cfg-2024-12-11--04-26-11.log new file mode 100644 index 0000000000..40e7611ae3 --- /dev/null +++ b/safekeeper/spec/tlc-results/MCProposerAcceptorReconfig.tla-MCProposerAcceptorReconfig_p2_a2_t2_l2_c5.cfg-2024-12-11--04-26-11.log @@ -0,0 +1,64 @@ +git revision: 9e386917a +Platform: Linux neon-dev-arm64-1 6.8.0-49-generic #49-Ubuntu SMP PREEMPT_DYNAMIC Sun Nov 3 21:21:58 UTC 2024 aarch64 aarch64 aarch64 GNU/Linux +CPU Info Linux: Neoverse-N1 +CPU Cores Linux: 80 +CPU Info Mac: +CPU Cores Mac: +Spec: MCProposerAcceptorReconfig.tla +Config: models/MCProposerAcceptorReconfig_p2_a2_t2_l2_c5.cfg +---- +CONSTANTS +NULL = NULL +proposers = {p1, p2} +acceptors = {a1, a2} +max_term = 2 +max_entries = 2 +max_generation = 5 +SPECIFICATION Spec +CONSTRAINT StateConstraint +INVARIANT +TypeOk +ElectionSafetyFull +LogIsMonotonic +LogSafety +\* CommittedNotTruncated +SYMMETRY ProposerAcceptorSymmetry +CHECK_DEADLOCK FALSE +ALIAS Alias + +---- + +TLC2 Version 2.20 of Day Month 20?? (rev: f68cb71) +Running breadth-first search Model-Checking with fp 114 and seed -8099467489737745861 with 35 workers on 80 cores with 27307MB heap and 30720MB offheap memory [pid: 392020] (Linux 6.8.0-49-generic aarch64, Ubuntu 21.0.5 x86_64, OffHeapDiskFPSet, DiskStateQueue). +Parsing file /home/arseny/neon2/safekeeper/spec/MCProposerAcceptorReconfig.tla +Parsing file /tmp/tlc-11757875725969857497/TLC.tla (jar:file:/home/arseny/tla2tools.jar!/tla2sany/StandardModules/TLC.tla) +Parsing file /home/arseny/neon2/safekeeper/spec/ProposerAcceptorReconfig.tla +Parsing file /tmp/tlc-11757875725969857497/_TLCTrace.tla (jar:file:/home/arseny/tla2tools.jar!/tla2sany/StandardModules/_TLCTrace.tla) +Parsing file /tmp/tlc-11757875725969857497/Integers.tla (jar:file:/home/arseny/tla2tools.jar!/tla2sany/StandardModules/Integers.tla) +Parsing file /tmp/tlc-11757875725969857497/Sequences.tla (jar:file:/home/arseny/tla2tools.jar!/tla2sany/StandardModules/Sequences.tla) +Parsing file /tmp/tlc-11757875725969857497/FiniteSets.tla (jar:file:/home/arseny/tla2tools.jar!/tla2sany/StandardModules/FiniteSets.tla) +Parsing file /tmp/tlc-11757875725969857497/Naturals.tla (jar:file:/home/arseny/tla2tools.jar!/tla2sany/StandardModules/Naturals.tla) +Parsing file /home/arseny/neon2/safekeeper/spec/ProposerAcceptorStatic.tla +Parsing file /tmp/tlc-11757875725969857497/TLCExt.tla (jar:file:/home/arseny/tla2tools.jar!/tla2sany/StandardModules/TLCExt.tla) +Semantic processing of module Naturals +Semantic processing of module Sequences +Semantic processing of module FiniteSets +Semantic processing of module TLC +Semantic processing of module Integers +Semantic processing of module ProposerAcceptorStatic +Semantic processing of module ProposerAcceptorReconfig +Semantic processing of module TLCExt +Semantic processing of module _TLCTrace +Semantic processing of module MCProposerAcceptorReconfig +Starting... (2024-12-11 04:26:12) +Computing initial states... +Finished computing initial states: 2 states generated, with 1 of them distinct at 2024-12-11 04:26:14. +Progress(14) at 2024-12-11 04:26:17: 1,519,385 states generated (1,519,385 s/min), 231,263 distinct states found (231,263 ds/min), 121,410 states left on queue. +Progress(20) at 2024-12-11 04:27:17: 42,757,204 states generated (41,237,819 s/min), 4,198,386 distinct states found (3,967,123 ds/min), 1,308,109 states left on queue. +Progress(22) at 2024-12-11 04:28:17: 83,613,929 states generated (40,856,725 s/min), 7,499,873 distinct states found (3,301,487 ds/min), 1,929,464 states left on queue. +Progress(23) at 2024-12-11 04:29:17: 124,086,758 states generated (40,472,829 s/min), 10,569,712 distinct states found (3,069,839 ds/min), 2,386,988 states left on queue. +Progress(24) at 2024-12-11 04:30:17: 163,412,538 states generated (39,325,780 s/min), 13,314,303 distinct states found (2,744,591 ds/min), 2,610,637 states left on queue. +Progress(25) at 2024-12-11 04:31:17: 202,643,708 states generated (39,231,170 s/min), 15,960,583 distinct states found (2,646,280 ds/min), 2,759,681 states left on queue. +Progress(26) at 2024-12-11 04:32:17: 240,681,633 states generated (38,037,925 s/min), 18,443,440 distinct states found (2,482,857 ds/min), 2,852,177 states left on queue. +Progress(27) at 2024-12-11 04:33:17: 278,559,134 states generated (37,877,501 s/min), 20,878,067 distinct states found (2,434,627 ds/min), 2,904,400 states left on queue. +Progress(28) at 2024-12-11 04:34:17: 316,699,911 states generated (38,140,777 s/min), 23,212,229 distinct states found (2,334,162 ds/min), 2,864,969 states left on queue. From f4739d49e3f20fae6a88b7a050084ab73c6319e4 Mon Sep 17 00:00:00 2001 From: Vlad Lazar Date: Thu, 9 Jan 2025 12:31:02 +0000 Subject: [PATCH 013/111] pageserver: tweak interpreted ingest record metrics (#10291) ## Problem The filtered record metric doesn't make sense for interpreted ingest. ## Summary of changes While of dubious utility in the first place, this patch replaces them with records received and records observed metrics for interpreted ingest: * received records cause the pageserver to do _something_: write a key, value pair to storage, update some metadata or flush pending modifications * observed records are a shard 0 concept and contain only key metadata used in tracking relation sizes (received records include observed records) --- libs/wal_decoder/src/models.rs | 8 +++ libs/wal_decoder/src/serialized_batch.rs | 5 ++ pageserver/src/metrics.rs | 10 ++- .../walreceiver/walreceiver_connection.rs | 72 +++++++------------ test_runner/regress/test_sharding.py | 54 +++++++++----- 5 files changed, 83 insertions(+), 66 deletions(-) diff --git a/libs/wal_decoder/src/models.rs b/libs/wal_decoder/src/models.rs index af22de5d95..6576dd0eba 100644 --- a/libs/wal_decoder/src/models.rs +++ b/libs/wal_decoder/src/models.rs @@ -95,6 +95,14 @@ impl InterpretedWalRecord { && self.metadata_record.is_none() && matches!(self.flush_uncommitted, FlushUncommittedRecords::No) } + + /// Checks if the WAL record is observed (i.e. contains only metadata + /// for observed values) + pub fn is_observed(&self) -> bool { + self.batch.is_observed() + && self.metadata_record.is_none() + && matches!(self.flush_uncommitted, FlushUncommittedRecords::No) + } } /// The interpreted part of the Postgres WAL record which requires metadata diff --git a/libs/wal_decoder/src/serialized_batch.rs b/libs/wal_decoder/src/serialized_batch.rs index 41294da7a0..af2b179e05 100644 --- a/libs/wal_decoder/src/serialized_batch.rs +++ b/libs/wal_decoder/src/serialized_batch.rs @@ -501,6 +501,11 @@ impl SerializedValueBatch { !self.has_data() && self.metadata.is_empty() } + /// Checks if the batch contains only observed values + pub fn is_observed(&self) -> bool { + !self.has_data() && !self.metadata.is_empty() + } + /// Checks if the batch contains data /// /// Note that if this returns false, it may still contain observed values or diff --git a/pageserver/src/metrics.rs b/pageserver/src/metrics.rs index b4e20cb8b9..e825d538a2 100644 --- a/pageserver/src/metrics.rs +++ b/pageserver/src/metrics.rs @@ -2337,13 +2337,15 @@ macro_rules! redo_bytes_histogram_count_buckets { pub(crate) struct WalIngestMetrics { pub(crate) bytes_received: IntCounter, pub(crate) records_received: IntCounter, + pub(crate) records_observed: IntCounter, pub(crate) records_committed: IntCounter, pub(crate) records_filtered: IntCounter, pub(crate) gap_blocks_zeroed_on_rel_extend: IntCounter, pub(crate) clear_vm_bits_unknown: IntCounterVec, } -pub(crate) static WAL_INGEST: Lazy = Lazy::new(|| WalIngestMetrics { +pub(crate) static WAL_INGEST: Lazy = Lazy::new(|| { + WalIngestMetrics { bytes_received: register_int_counter!( "pageserver_wal_ingest_bytes_received", "Bytes of WAL ingested from safekeepers", @@ -2354,6 +2356,11 @@ pub(crate) static WAL_INGEST: Lazy = Lazy::new(|| WalIngestMet "Number of WAL records received from safekeepers" ) .expect("failed to define a metric"), + records_observed: register_int_counter!( + "pageserver_wal_ingest_records_observed", + "Number of WAL records observed from safekeepers. These are metadata only records for shard 0." + ) + .expect("failed to define a metric"), records_committed: register_int_counter!( "pageserver_wal_ingest_records_committed", "Number of WAL records which resulted in writes to pageserver storage" @@ -2375,6 +2382,7 @@ pub(crate) static WAL_INGEST: Lazy = Lazy::new(|| WalIngestMet &["entity"], ) .expect("failed to define a metric"), +} }); pub(crate) static PAGESERVER_TIMELINE_WAL_RECORDS_RECEIVED: Lazy = Lazy::new(|| { diff --git a/pageserver/src/tenant/timeline/walreceiver/walreceiver_connection.rs b/pageserver/src/tenant/timeline/walreceiver/walreceiver_connection.rs index 3f10eeda60..d74faa1af5 100644 --- a/pageserver/src/tenant/timeline/walreceiver/walreceiver_connection.rs +++ b/pageserver/src/tenant/timeline/walreceiver/walreceiver_connection.rs @@ -319,27 +319,11 @@ pub(super) async fn handle_walreceiver_connection( return Ok(()); } - async fn commit( - modification: &mut DatadirModification<'_>, - uncommitted: &mut u64, - filtered: &mut u64, - ctx: &RequestContext, - ) -> anyhow::Result<()> { - WAL_INGEST - .records_committed - .inc_by(*uncommitted - *filtered); - modification.commit(ctx).await?; - *uncommitted = 0; - *filtered = 0; - Ok(()) - } - let status_update = match replication_message { ReplicationMessage::RawInterpretedWalRecords(raw) => { WAL_INGEST.bytes_received.inc_by(raw.data().len() as u64); let mut uncommitted_records = 0; - let mut filtered_records = 0; // This is the end LSN of the raw WAL from which the records // were interpreted. @@ -380,31 +364,23 @@ pub(super) async fn handle_walreceiver_connection( if matches!(interpreted.flush_uncommitted, FlushUncommittedRecords::Yes) && uncommitted_records > 0 { - commit( - &mut modification, - &mut uncommitted_records, - &mut filtered_records, - &ctx, - ) - .await?; + modification.commit(&ctx).await?; + uncommitted_records = 0; } let local_next_record_lsn = interpreted.next_record_lsn; - let ingested = walingest + + if interpreted.is_observed() { + WAL_INGEST.records_observed.inc(); + } + + walingest .ingest_record(interpreted, &mut modification, &ctx) .await .with_context(|| { format!("could not ingest record at {local_next_record_lsn}") })?; - if !ingested { - tracing::debug!( - "ingest: filtered out record @ LSN {local_next_record_lsn}" - ); - WAL_INGEST.records_filtered.inc(); - filtered_records += 1; - } - uncommitted_records += 1; // FIXME: this cannot be made pausable_failpoint without fixing the @@ -418,13 +394,8 @@ pub(super) async fn handle_walreceiver_connection( || modification.approx_pending_bytes() > DatadirModification::MAX_PENDING_BYTES { - commit( - &mut modification, - &mut uncommitted_records, - &mut filtered_records, - &ctx, - ) - .await?; + modification.commit(&ctx).await?; + uncommitted_records = 0; } } @@ -442,13 +413,7 @@ pub(super) async fn handle_walreceiver_connection( if uncommitted_records > 0 || needs_last_record_lsn_advance { // Commit any uncommitted records - commit( - &mut modification, - &mut uncommitted_records, - &mut filtered_records, - &ctx, - ) - .await?; + modification.commit(&ctx).await?; } if !caught_up && streaming_lsn >= end_of_wal { @@ -469,6 +434,21 @@ pub(super) async fn handle_walreceiver_connection( } ReplicationMessage::XLogData(xlog_data) => { + async fn commit( + modification: &mut DatadirModification<'_>, + uncommitted: &mut u64, + filtered: &mut u64, + ctx: &RequestContext, + ) -> anyhow::Result<()> { + WAL_INGEST + .records_committed + .inc_by(*uncommitted - *filtered); + modification.commit(ctx).await?; + *uncommitted = 0; + *filtered = 0; + Ok(()) + } + // Pass the WAL data to the decoder, and see if we can decode // more records as a result. let data = xlog_data.data(); diff --git a/test_runner/regress/test_sharding.py b/test_runner/regress/test_sharding.py index 4c381b563f..673904a1cd 100644 --- a/test_runner/regress/test_sharding.py +++ b/test_runner/regress/test_sharding.py @@ -561,11 +561,17 @@ def test_sharding_split_smoke( workload.write_rows(256) # Note which pageservers initially hold a shard after tenant creation - pre_split_pageserver_ids = [loc["node_id"] for loc in env.storage_controller.locate(tenant_id)] - log.info("Pre-split pageservers: {pre_split_pageserver_ids}") + pre_split_pageserver_ids = dict() + for loc in env.storage_controller.locate(tenant_id): + shard_no = TenantShardId.parse(loc["shard_id"]).shard_number + pre_split_pageserver_ids[loc["node_id"]] = shard_no + log.info(f"Pre-split pageservers: {pre_split_pageserver_ids}") # For pageservers holding a shard, validate their ingest statistics # reflect a proper splitting of the WAL. + + observed_on_shard_zero = 0 + received_on_non_zero_shard = 0 for pageserver in env.pageservers: if pageserver.id not in pre_split_pageserver_ids: continue @@ -573,28 +579,38 @@ def test_sharding_split_smoke( metrics = pageserver.http_client().get_metrics_values( [ "pageserver_wal_ingest_records_received_total", - "pageserver_wal_ingest_records_committed_total", - "pageserver_wal_ingest_records_filtered_total", + "pageserver_wal_ingest_records_observed_total", ] ) log.info(f"Pageserver {pageserver.id} metrics: {metrics}") - # Not everything received was committed - assert ( - metrics["pageserver_wal_ingest_records_received_total"] - > metrics["pageserver_wal_ingest_records_committed_total"] - ) + received = metrics["pageserver_wal_ingest_records_received_total"] + observed = metrics["pageserver_wal_ingest_records_observed_total"] - # Something was committed - assert metrics["pageserver_wal_ingest_records_committed_total"] > 0 + shard_number: int | None = pre_split_pageserver_ids.get(pageserver.id, None) + if shard_number is None: + assert received == 0 + assert observed == 0 + elif shard_number == 0: + # Shard 0 receives its own records and observes records of other shards + # for relation size tracking. + assert observed > 0 + assert received > 0 + observed_on_shard_zero = int(observed) + else: + # Non zero shards do not observe any records, but only receive their own. + assert observed == 0 + assert received > 0 + received_on_non_zero_shard += int(received) - # Counts are self consistent - assert ( - metrics["pageserver_wal_ingest_records_received_total"] - == metrics["pageserver_wal_ingest_records_committed_total"] - + metrics["pageserver_wal_ingest_records_filtered_total"] - ) + # Some records are sent to multiple shards and some shard 0 records include both value observations + # and other metadata. Hence, we do a sanity check below that shard 0 observes the majority of records + # received by other shards. + assert ( + observed_on_shard_zero <= received_on_non_zero_shard + and observed_on_shard_zero >= received_on_non_zero_shard // 2 + ) # TODO: validate that shards have different sizes @@ -633,7 +649,7 @@ def test_sharding_split_smoke( # We should have split into 8 shards, on the same 4 pageservers we started on. assert len(post_split_pageserver_ids) == split_shard_count assert len(set(post_split_pageserver_ids)) == shard_count - assert set(post_split_pageserver_ids) == set(pre_split_pageserver_ids) + assert set(post_split_pageserver_ids) == set(pre_split_pageserver_ids.keys()) # The old parent shards should no longer exist on disk assert not shards_on_disk(old_shard_ids) @@ -739,7 +755,7 @@ def test_sharding_split_smoke( # all the pageservers that originally held an attached shard should still hold one, otherwise # it would indicate that we had done some unnecessary migration. log.info(f"attached: {attached}") - for ps_id in pre_split_pageserver_ids: + for ps_id in pre_split_pageserver_ids.keys(): log.info("Pre-split pageserver {ps_id} should still hold an attached location") assert ps_id in attached From 20c40eb733f401903adfa1c1da47dbed54098cae Mon Sep 17 00:00:00 2001 From: Konstantin Knizhnik Date: Thu, 9 Jan 2025 15:12:04 +0200 Subject: [PATCH 014/111] Add response tag to getpage request in V3 protocol version (#8686) ## Problem We have several serious data corruption incidents caused by mismatch of get-age requests: https://neondb.slack.com/archives/C07FJS4QF7V/p1723032720164359 We hope that the problem is fixed now. But it is better to prevent such kind of problems in future. Part of https://github.com/neondatabase/cloud/issues/16472 ## Summary of changes This PR introduce new V3 version of compute<->pageserver protocol, adding tag to getpage response. So now compute is able to check if it really gets response to the requested page. ## Checklist before requesting a review - [ ] I have performed a self-review of my code. - [ ] If it is a core feature, I have added thorough tests. - [ ] Do we need to implement analytics? if so did you add the relevant metrics to the dashboard? - [ ] If this PR requires public announcement, mark it with /release-notes label and add several sentences in this section. ## Checklist before merging - [ ] Do not forget to reformat commit message to not include the above checklist --------- Co-authored-by: Konstantin Knizhnik Co-authored-by: Heikki Linnakangas --- libs/pageserver_api/src/models.rs | 395 ++++++++++++---- pageserver/client/src/page_service.rs | 2 +- .../pagebench/src/cmd/getpage_latest_lsn.rs | 15 +- pageserver/src/metrics.rs | 1 + pageserver/src/page_service.rs | 181 ++++--- pgxn/neon/libpagestore.c | 9 +- pgxn/neon/pagestore_client.h | 51 +- pgxn/neon/pagestore_smgr.c | 445 ++++++++++++------ 8 files changed, 784 insertions(+), 315 deletions(-) diff --git a/libs/pageserver_api/src/models.rs b/libs/pageserver_api/src/models.rs index f3fc9fad76..39390d7647 100644 --- a/libs/pageserver_api/src/models.rs +++ b/libs/pageserver_api/src/models.rs @@ -1460,75 +1460,91 @@ impl TryFrom for PagestreamBeMessageTag { // interface allows sending both LSNs, and let the pageserver do the right thing. There was no // difference in the responses between V1 and V2. // -#[derive(Clone, Copy)] +// V3 version of protocol adds request ID to all requests. This request ID is also included in response +// as well as other fields from requests, which allows to verify that we receive response for our request. +// We copy fields from request to response to make checking more reliable: request ID is formed from process ID +// and local counter, so in principle there can be duplicated requests IDs if process PID is reused. +// +#[derive(Debug, PartialEq, Eq, Clone, Copy)] pub enum PagestreamProtocolVersion { V2, + V3, } -#[derive(Debug, PartialEq, Eq)] +pub type RequestId = u64; + +#[derive(Debug, PartialEq, Eq, Clone, Copy)] +pub struct PagestreamRequest { + pub reqid: RequestId, + pub request_lsn: Lsn, + pub not_modified_since: Lsn, +} + +#[derive(Debug, PartialEq, Eq, Clone, Copy)] pub struct PagestreamExistsRequest { - pub request_lsn: Lsn, - pub not_modified_since: Lsn, + pub hdr: PagestreamRequest, pub rel: RelTag, } -#[derive(Debug, PartialEq, Eq)] +#[derive(Debug, PartialEq, Eq, Clone, Copy)] pub struct PagestreamNblocksRequest { - pub request_lsn: Lsn, - pub not_modified_since: Lsn, + pub hdr: PagestreamRequest, pub rel: RelTag, } -#[derive(Debug, PartialEq, Eq)] +#[derive(Debug, PartialEq, Eq, Clone, Copy)] pub struct PagestreamGetPageRequest { - pub request_lsn: Lsn, - pub not_modified_since: Lsn, + pub hdr: PagestreamRequest, pub rel: RelTag, pub blkno: u32, } -#[derive(Debug, PartialEq, Eq)] +#[derive(Debug, PartialEq, Eq, Clone, Copy)] pub struct PagestreamDbSizeRequest { - pub request_lsn: Lsn, - pub not_modified_since: Lsn, + pub hdr: PagestreamRequest, pub dbnode: u32, } -#[derive(Debug, PartialEq, Eq)] +#[derive(Debug, PartialEq, Eq, Clone, Copy)] pub struct PagestreamGetSlruSegmentRequest { - pub request_lsn: Lsn, - pub not_modified_since: Lsn, + pub hdr: PagestreamRequest, pub kind: u8, pub segno: u32, } #[derive(Debug)] pub struct PagestreamExistsResponse { + pub req: PagestreamExistsRequest, pub exists: bool, } #[derive(Debug)] pub struct PagestreamNblocksResponse { + pub req: PagestreamNblocksRequest, pub n_blocks: u32, } #[derive(Debug)] pub struct PagestreamGetPageResponse { + pub req: PagestreamGetPageRequest, pub page: Bytes, } #[derive(Debug)] pub struct PagestreamGetSlruSegmentResponse { + pub req: PagestreamGetSlruSegmentRequest, pub segment: Bytes, } #[derive(Debug)] pub struct PagestreamErrorResponse { + pub req: PagestreamRequest, pub message: String, } #[derive(Debug)] pub struct PagestreamDbSizeResponse { + pub req: PagestreamDbSizeRequest, pub db_size: i64, } @@ -1545,15 +1561,16 @@ pub struct TenantHistorySize { impl PagestreamFeMessage { /// Serialize a compute -> pageserver message. This is currently only used in testing - /// tools. Always uses protocol version 2. + /// tools. Always uses protocol version 3. pub fn serialize(&self) -> Bytes { let mut bytes = BytesMut::new(); match self { Self::Exists(req) => { bytes.put_u8(0); - bytes.put_u64(req.request_lsn.0); - bytes.put_u64(req.not_modified_since.0); + bytes.put_u64(req.hdr.reqid); + bytes.put_u64(req.hdr.request_lsn.0); + bytes.put_u64(req.hdr.not_modified_since.0); bytes.put_u32(req.rel.spcnode); bytes.put_u32(req.rel.dbnode); bytes.put_u32(req.rel.relnode); @@ -1562,8 +1579,9 @@ impl PagestreamFeMessage { Self::Nblocks(req) => { bytes.put_u8(1); - bytes.put_u64(req.request_lsn.0); - bytes.put_u64(req.not_modified_since.0); + bytes.put_u64(req.hdr.reqid); + bytes.put_u64(req.hdr.request_lsn.0); + bytes.put_u64(req.hdr.not_modified_since.0); bytes.put_u32(req.rel.spcnode); bytes.put_u32(req.rel.dbnode); bytes.put_u32(req.rel.relnode); @@ -1572,8 +1590,9 @@ impl PagestreamFeMessage { Self::GetPage(req) => { bytes.put_u8(2); - bytes.put_u64(req.request_lsn.0); - bytes.put_u64(req.not_modified_since.0); + bytes.put_u64(req.hdr.reqid); + bytes.put_u64(req.hdr.request_lsn.0); + bytes.put_u64(req.hdr.not_modified_since.0); bytes.put_u32(req.rel.spcnode); bytes.put_u32(req.rel.dbnode); bytes.put_u32(req.rel.relnode); @@ -1583,15 +1602,17 @@ impl PagestreamFeMessage { Self::DbSize(req) => { bytes.put_u8(3); - bytes.put_u64(req.request_lsn.0); - bytes.put_u64(req.not_modified_since.0); + bytes.put_u64(req.hdr.reqid); + bytes.put_u64(req.hdr.request_lsn.0); + bytes.put_u64(req.hdr.not_modified_since.0); bytes.put_u32(req.dbnode); } Self::GetSlruSegment(req) => { bytes.put_u8(4); - bytes.put_u64(req.request_lsn.0); - bytes.put_u64(req.not_modified_since.0); + bytes.put_u64(req.hdr.reqid); + bytes.put_u64(req.hdr.request_lsn.0); + bytes.put_u64(req.hdr.not_modified_since.0); bytes.put_u8(req.kind); bytes.put_u32(req.segno); } @@ -1600,21 +1621,35 @@ impl PagestreamFeMessage { bytes.into() } - pub fn parse(body: &mut R) -> anyhow::Result { + pub fn parse( + body: &mut R, + protocol_version: PagestreamProtocolVersion, + ) -> anyhow::Result { // these correspond to the NeonMessageTag enum in pagestore_client.h // // TODO: consider using protobuf or serde bincode for less error prone // serialization. let msg_tag = body.read_u8()?; - - // these two fields are the same for every request type - let request_lsn = Lsn::from(body.read_u64::()?); - let not_modified_since = Lsn::from(body.read_u64::()?); + let (reqid, request_lsn, not_modified_since) = match protocol_version { + PagestreamProtocolVersion::V2 => ( + 0, + Lsn::from(body.read_u64::()?), + Lsn::from(body.read_u64::()?), + ), + PagestreamProtocolVersion::V3 => ( + body.read_u64::()?, + Lsn::from(body.read_u64::()?), + Lsn::from(body.read_u64::()?), + ), + }; match msg_tag { 0 => Ok(PagestreamFeMessage::Exists(PagestreamExistsRequest { - request_lsn, - not_modified_since, + hdr: PagestreamRequest { + reqid, + request_lsn, + not_modified_since, + }, rel: RelTag { spcnode: body.read_u32::()?, dbnode: body.read_u32::()?, @@ -1623,8 +1658,11 @@ impl PagestreamFeMessage { }, })), 1 => Ok(PagestreamFeMessage::Nblocks(PagestreamNblocksRequest { - request_lsn, - not_modified_since, + hdr: PagestreamRequest { + reqid, + request_lsn, + not_modified_since, + }, rel: RelTag { spcnode: body.read_u32::()?, dbnode: body.read_u32::()?, @@ -1633,8 +1671,11 @@ impl PagestreamFeMessage { }, })), 2 => Ok(PagestreamFeMessage::GetPage(PagestreamGetPageRequest { - request_lsn, - not_modified_since, + hdr: PagestreamRequest { + reqid, + request_lsn, + not_modified_since, + }, rel: RelTag { spcnode: body.read_u32::()?, dbnode: body.read_u32::()?, @@ -1644,14 +1685,20 @@ impl PagestreamFeMessage { blkno: body.read_u32::()?, })), 3 => Ok(PagestreamFeMessage::DbSize(PagestreamDbSizeRequest { - request_lsn, - not_modified_since, + hdr: PagestreamRequest { + reqid, + request_lsn, + not_modified_since, + }, dbnode: body.read_u32::()?, })), 4 => Ok(PagestreamFeMessage::GetSlruSegment( PagestreamGetSlruSegmentRequest { - request_lsn, - not_modified_since, + hdr: PagestreamRequest { + reqid, + request_lsn, + not_modified_since, + }, kind: body.read_u8()?, segno: body.read_u32::()?, }, @@ -1662,43 +1709,114 @@ impl PagestreamFeMessage { } impl PagestreamBeMessage { - pub fn serialize(&self) -> Bytes { + pub fn serialize(&self, protocol_version: PagestreamProtocolVersion) -> Bytes { let mut bytes = BytesMut::new(); use PagestreamBeMessageTag as Tag; - match self { - Self::Exists(resp) => { - bytes.put_u8(Tag::Exists as u8); - bytes.put_u8(resp.exists as u8); - } + match protocol_version { + PagestreamProtocolVersion::V2 => { + match self { + Self::Exists(resp) => { + bytes.put_u8(Tag::Exists as u8); + bytes.put_u8(resp.exists as u8); + } - Self::Nblocks(resp) => { - bytes.put_u8(Tag::Nblocks as u8); - bytes.put_u32(resp.n_blocks); - } + Self::Nblocks(resp) => { + bytes.put_u8(Tag::Nblocks as u8); + bytes.put_u32(resp.n_blocks); + } - Self::GetPage(resp) => { - bytes.put_u8(Tag::GetPage as u8); - bytes.put(&resp.page[..]); - } + Self::GetPage(resp) => { + bytes.put_u8(Tag::GetPage as u8); + bytes.put(&resp.page[..]) + } - Self::Error(resp) => { - bytes.put_u8(Tag::Error as u8); - bytes.put(resp.message.as_bytes()); - bytes.put_u8(0); // null terminator - } - Self::DbSize(resp) => { - bytes.put_u8(Tag::DbSize as u8); - bytes.put_i64(resp.db_size); - } + Self::Error(resp) => { + bytes.put_u8(Tag::Error as u8); + bytes.put(resp.message.as_bytes()); + bytes.put_u8(0); // null terminator + } + Self::DbSize(resp) => { + bytes.put_u8(Tag::DbSize as u8); + bytes.put_i64(resp.db_size); + } - Self::GetSlruSegment(resp) => { - bytes.put_u8(Tag::GetSlruSegment as u8); - bytes.put_u32((resp.segment.len() / BLCKSZ as usize) as u32); - bytes.put(&resp.segment[..]); + Self::GetSlruSegment(resp) => { + bytes.put_u8(Tag::GetSlruSegment as u8); + bytes.put_u32((resp.segment.len() / BLCKSZ as usize) as u32); + bytes.put(&resp.segment[..]); + } + } + } + PagestreamProtocolVersion::V3 => { + match self { + Self::Exists(resp) => { + bytes.put_u8(Tag::Exists as u8); + bytes.put_u64(resp.req.hdr.reqid); + bytes.put_u64(resp.req.hdr.request_lsn.0); + bytes.put_u64(resp.req.hdr.not_modified_since.0); + bytes.put_u32(resp.req.rel.spcnode); + bytes.put_u32(resp.req.rel.dbnode); + bytes.put_u32(resp.req.rel.relnode); + bytes.put_u8(resp.req.rel.forknum); + bytes.put_u8(resp.exists as u8); + } + + Self::Nblocks(resp) => { + bytes.put_u8(Tag::Nblocks as u8); + bytes.put_u64(resp.req.hdr.reqid); + bytes.put_u64(resp.req.hdr.request_lsn.0); + bytes.put_u64(resp.req.hdr.not_modified_since.0); + bytes.put_u32(resp.req.rel.spcnode); + bytes.put_u32(resp.req.rel.dbnode); + bytes.put_u32(resp.req.rel.relnode); + bytes.put_u8(resp.req.rel.forknum); + bytes.put_u32(resp.n_blocks); + } + + Self::GetPage(resp) => { + bytes.put_u8(Tag::GetPage as u8); + bytes.put_u64(resp.req.hdr.reqid); + bytes.put_u64(resp.req.hdr.request_lsn.0); + bytes.put_u64(resp.req.hdr.not_modified_since.0); + bytes.put_u32(resp.req.rel.spcnode); + bytes.put_u32(resp.req.rel.dbnode); + bytes.put_u32(resp.req.rel.relnode); + bytes.put_u8(resp.req.rel.forknum); + bytes.put_u32(resp.req.blkno); + bytes.put(&resp.page[..]) + } + + Self::Error(resp) => { + bytes.put_u8(Tag::Error as u8); + bytes.put_u64(resp.req.reqid); + bytes.put_u64(resp.req.request_lsn.0); + bytes.put_u64(resp.req.not_modified_since.0); + bytes.put(resp.message.as_bytes()); + bytes.put_u8(0); // null terminator + } + Self::DbSize(resp) => { + bytes.put_u8(Tag::DbSize as u8); + bytes.put_u64(resp.req.hdr.reqid); + bytes.put_u64(resp.req.hdr.request_lsn.0); + bytes.put_u64(resp.req.hdr.not_modified_since.0); + bytes.put_u32(resp.req.dbnode); + bytes.put_i64(resp.db_size); + } + + Self::GetSlruSegment(resp) => { + bytes.put_u8(Tag::GetSlruSegment as u8); + bytes.put_u64(resp.req.hdr.reqid); + bytes.put_u64(resp.req.hdr.request_lsn.0); + bytes.put_u64(resp.req.hdr.not_modified_since.0); + bytes.put_u8(resp.req.kind); + bytes.put_u32(resp.req.segno); + bytes.put_u32((resp.segment.len() / BLCKSZ as usize) as u32); + bytes.put(&resp.segment[..]); + } + } } } - bytes.into() } @@ -1710,38 +1828,131 @@ impl PagestreamBeMessage { let ok = match Tag::try_from(msg_tag).map_err(|tag: u8| anyhow::anyhow!("invalid tag {tag}"))? { Tag::Exists => { - let exists = buf.read_u8()?; + let reqid = buf.read_u64::()?; + let request_lsn = Lsn(buf.read_u64::()?); + let not_modified_since = Lsn(buf.read_u64::()?); + let rel = RelTag { + spcnode: buf.read_u32::()?, + dbnode: buf.read_u32::()?, + relnode: buf.read_u32::()?, + forknum: buf.read_u8()?, + }; + let exists = buf.read_u8()? != 0; Self::Exists(PagestreamExistsResponse { - exists: exists != 0, + req: PagestreamExistsRequest { + hdr: PagestreamRequest { + reqid, + request_lsn, + not_modified_since, + }, + rel, + }, + exists, }) } Tag::Nblocks => { + let reqid = buf.read_u64::()?; + let request_lsn = Lsn(buf.read_u64::()?); + let not_modified_since = Lsn(buf.read_u64::()?); + let rel = RelTag { + spcnode: buf.read_u32::()?, + dbnode: buf.read_u32::()?, + relnode: buf.read_u32::()?, + forknum: buf.read_u8()?, + }; let n_blocks = buf.read_u32::()?; - Self::Nblocks(PagestreamNblocksResponse { n_blocks }) + Self::Nblocks(PagestreamNblocksResponse { + req: PagestreamNblocksRequest { + hdr: PagestreamRequest { + reqid, + request_lsn, + not_modified_since, + }, + rel, + }, + n_blocks, + }) } Tag::GetPage => { + let reqid = buf.read_u64::()?; + let request_lsn = Lsn(buf.read_u64::()?); + let not_modified_since = Lsn(buf.read_u64::()?); + let rel = RelTag { + spcnode: buf.read_u32::()?, + dbnode: buf.read_u32::()?, + relnode: buf.read_u32::()?, + forknum: buf.read_u8()?, + }; + let blkno = buf.read_u32::()?; let mut page = vec![0; 8192]; // TODO: use MaybeUninit buf.read_exact(&mut page)?; - PagestreamBeMessage::GetPage(PagestreamGetPageResponse { page: page.into() }) + Self::GetPage(PagestreamGetPageResponse { + req: PagestreamGetPageRequest { + hdr: PagestreamRequest { + reqid, + request_lsn, + not_modified_since, + }, + rel, + blkno, + }, + page: page.into(), + }) } Tag::Error => { + let reqid = buf.read_u64::()?; + let request_lsn = Lsn(buf.read_u64::()?); + let not_modified_since = Lsn(buf.read_u64::()?); let mut msg = Vec::new(); buf.read_until(0, &mut msg)?; let cstring = std::ffi::CString::from_vec_with_nul(msg)?; let rust_str = cstring.to_str()?; - PagestreamBeMessage::Error(PagestreamErrorResponse { + Self::Error(PagestreamErrorResponse { + req: PagestreamRequest { + reqid, + request_lsn, + not_modified_since, + }, message: rust_str.to_owned(), }) } Tag::DbSize => { + let reqid = buf.read_u64::()?; + let request_lsn = Lsn(buf.read_u64::()?); + let not_modified_since = Lsn(buf.read_u64::()?); + let dbnode = buf.read_u32::()?; let db_size = buf.read_i64::()?; - Self::DbSize(PagestreamDbSizeResponse { db_size }) + Self::DbSize(PagestreamDbSizeResponse { + req: PagestreamDbSizeRequest { + hdr: PagestreamRequest { + reqid, + request_lsn, + not_modified_since, + }, + dbnode, + }, + db_size, + }) } Tag::GetSlruSegment => { + let reqid = buf.read_u64::()?; + let request_lsn = Lsn(buf.read_u64::()?); + let not_modified_since = Lsn(buf.read_u64::()?); + let kind = buf.read_u8()?; + let segno = buf.read_u32::()?; let n_blocks = buf.read_u32::()?; let mut segment = vec![0; n_blocks as usize * BLCKSZ as usize]; buf.read_exact(&mut segment)?; Self::GetSlruSegment(PagestreamGetSlruSegmentResponse { + req: PagestreamGetSlruSegmentRequest { + hdr: PagestreamRequest { + reqid, + request_lsn, + not_modified_since, + }, + kind, + segno, + }, segment: segment.into(), }) } @@ -1780,8 +1991,11 @@ mod tests { // Test serialization/deserialization of PagestreamFeMessage let messages = vec![ PagestreamFeMessage::Exists(PagestreamExistsRequest { - request_lsn: Lsn(4), - not_modified_since: Lsn(3), + hdr: PagestreamRequest { + reqid: 0, + request_lsn: Lsn(4), + not_modified_since: Lsn(3), + }, rel: RelTag { forknum: 1, spcnode: 2, @@ -1790,8 +2004,11 @@ mod tests { }, }), PagestreamFeMessage::Nblocks(PagestreamNblocksRequest { - request_lsn: Lsn(4), - not_modified_since: Lsn(4), + hdr: PagestreamRequest { + reqid: 0, + request_lsn: Lsn(4), + not_modified_since: Lsn(4), + }, rel: RelTag { forknum: 1, spcnode: 2, @@ -1800,8 +2017,11 @@ mod tests { }, }), PagestreamFeMessage::GetPage(PagestreamGetPageRequest { - request_lsn: Lsn(4), - not_modified_since: Lsn(3), + hdr: PagestreamRequest { + reqid: 0, + request_lsn: Lsn(4), + not_modified_since: Lsn(3), + }, rel: RelTag { forknum: 1, spcnode: 2, @@ -1811,14 +2031,19 @@ mod tests { blkno: 7, }), PagestreamFeMessage::DbSize(PagestreamDbSizeRequest { - request_lsn: Lsn(4), - not_modified_since: Lsn(3), + hdr: PagestreamRequest { + reqid: 0, + request_lsn: Lsn(4), + not_modified_since: Lsn(3), + }, dbnode: 7, }), ]; for msg in messages { let bytes = msg.serialize(); - let reconstructed = PagestreamFeMessage::parse(&mut bytes.reader()).unwrap(); + let reconstructed = + PagestreamFeMessage::parse(&mut bytes.reader(), PagestreamProtocolVersion::V3) + .unwrap(); assert!(msg == reconstructed); } } diff --git a/pageserver/client/src/page_service.rs b/pageserver/client/src/page_service.rs index f9507fc47a..207ec4166c 100644 --- a/pageserver/client/src/page_service.rs +++ b/pageserver/client/src/page_service.rs @@ -60,7 +60,7 @@ impl Client { ) -> anyhow::Result { let copy_both: tokio_postgres::CopyBothDuplex = self .client - .copy_both_simple(&format!("pagestream_v2 {tenant_id} {timeline_id}")) + .copy_both_simple(&format!("pagestream_v3 {tenant_id} {timeline_id}")) .await?; let Client { cancel_on_client_drop, diff --git a/pageserver/pagebench/src/cmd/getpage_latest_lsn.rs b/pageserver/pagebench/src/cmd/getpage_latest_lsn.rs index b2df01714d..9f3984f1bd 100644 --- a/pageserver/pagebench/src/cmd/getpage_latest_lsn.rs +++ b/pageserver/pagebench/src/cmd/getpage_latest_lsn.rs @@ -2,7 +2,7 @@ use anyhow::Context; use camino::Utf8PathBuf; use pageserver_api::key::Key; use pageserver_api::keyspace::KeySpaceAccum; -use pageserver_api::models::PagestreamGetPageRequest; +use pageserver_api::models::{PagestreamGetPageRequest, PagestreamRequest}; use pageserver_api::shard::TenantShardId; use tokio_util::sync::CancellationToken; @@ -322,12 +322,15 @@ async fn main_impl( .to_rel_block() .expect("we filter non-rel-block keys out above"); PagestreamGetPageRequest { - request_lsn: if rng.gen_bool(args.req_latest_probability) { - Lsn::MAX - } else { - r.timeline_lsn + hdr: PagestreamRequest { + reqid: 0, + request_lsn: if rng.gen_bool(args.req_latest_probability) { + Lsn::MAX + } else { + r.timeline_lsn + }, + not_modified_since: r.timeline_lsn, }, - not_modified_since: r.timeline_lsn, rel: rel_tag, blkno: block_no, } diff --git a/pageserver/src/metrics.rs b/pageserver/src/metrics.rs index e825d538a2..a313a64080 100644 --- a/pageserver/src/metrics.rs +++ b/pageserver/src/metrics.rs @@ -1854,6 +1854,7 @@ pub(crate) static LIVE_CONNECTIONS: Lazy = Lazy::new(|| { #[derive(Clone, Copy, enum_map::Enum, IntoStaticStr)] pub(crate) enum ComputeCommandKind { + PageStreamV3, PageStreamV2, Basebackup, Fullbackup, diff --git a/pageserver/src/page_service.rs b/pageserver/src/page_service.rs index d00ec11a76..0c4a1b18f5 100644 --- a/pageserver/src/page_service.rs +++ b/pageserver/src/page_service.rs @@ -17,7 +17,7 @@ use pageserver_api::models::{ PagestreamErrorResponse, PagestreamExistsRequest, PagestreamExistsResponse, PagestreamFeMessage, PagestreamGetPageRequest, PagestreamGetSlruSegmentRequest, PagestreamGetSlruSegmentResponse, PagestreamNblocksRequest, PagestreamNblocksResponse, - PagestreamProtocolVersion, + PagestreamProtocolVersion, PagestreamRequest, }; use pageserver_api::shard::TenantShardId; use postgres_backend::{ @@ -67,7 +67,7 @@ use crate::tenant::PageReconstructError; use crate::tenant::Timeline; use crate::{basebackup, timed_after_cancellation}; use pageserver_api::key::rel_block_to_key; -use pageserver_api::reltag::{BlockNumber, RelTag, SlruKind}; +use pageserver_api::reltag::SlruKind; use postgres_ffi::pg_constants::DEFAULTTABLESPACE_OID; use postgres_ffi::BLCKSZ; @@ -537,6 +537,23 @@ impl From for QueryError { } } +#[derive(thiserror::Error, Debug)] +struct BatchedPageStreamError { + req: PagestreamRequest, + err: PageStreamError, +} + +impl std::fmt::Display for BatchedPageStreamError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + self.err.fmt(f) + } +} + +struct BatchedGetPageRequest { + req: PagestreamGetPageRequest, + timer: SmgrOpTimer, +} + enum BatchedFeMessage { Exists { span: Span, @@ -554,7 +571,7 @@ enum BatchedFeMessage { span: Span, shard: timeline::handle::Handle, effective_request_lsn: Lsn, - pages: smallvec::SmallVec<[(RelTag, BlockNumber, SmgrOpTimer); 1]>, + pages: smallvec::SmallVec<[BatchedGetPageRequest; 1]>, }, DbSize { span: Span, @@ -570,7 +587,7 @@ enum BatchedFeMessage { }, RespondError { span: Span, - error: PageStreamError, + error: BatchedPageStreamError, }, } @@ -595,7 +612,7 @@ impl BatchedFeMessage { BatchedFeMessage::GetPage { shard, pages, .. } => ( shard, pages.len(), - itertools::Either::Right(pages.iter_mut().map(|(_, _, timer)| timer)), + itertools::Either::Right(pages.iter_mut().map(|p| &mut p.timer)), ), BatchedFeMessage::RespondError { .. } => return Ok(()), }; @@ -654,6 +671,7 @@ impl PageServerHandler { ) } + #[allow(clippy::too_many_arguments)] async fn pagestream_read_message( pgb: &mut PostgresBackendReader, tenant_id: TenantId, @@ -661,6 +679,7 @@ impl PageServerHandler { timeline_handles: &mut TimelineHandles, cancel: &CancellationToken, ctx: &RequestContext, + protocol_version: PagestreamProtocolVersion, parent_span: Span, ) -> Result, QueryError> where @@ -695,11 +714,12 @@ impl PageServerHandler { fail::fail_point!("ps::handle-pagerequest-message"); // parse request - let neon_fe_msg = PagestreamFeMessage::parse(&mut copy_data_bytes.reader())?; + let neon_fe_msg = + PagestreamFeMessage::parse(&mut copy_data_bytes.reader(), protocol_version)?; let batched_msg = match neon_fe_msg { PagestreamFeMessage::Exists(req) => { - let span = tracing::info_span!(parent: parent_span, "handle_get_rel_exists_request", rel = %req.rel, req_lsn = %req.request_lsn); + let span = tracing::info_span!(parent: parent_span, "handle_get_rel_exists_request", rel = %req.rel, req_lsn = %req.hdr.request_lsn); let shard = timeline_handles .get(tenant_id, timeline_id, ShardSelector::Zero) .instrument(span.clone()) // sets `shard_id` field @@ -715,7 +735,7 @@ impl PageServerHandler { } } PagestreamFeMessage::Nblocks(req) => { - let span = tracing::info_span!(parent: parent_span, "handle_get_nblocks_request", rel = %req.rel, req_lsn = %req.request_lsn); + let span = tracing::info_span!(parent: parent_span, "handle_get_nblocks_request", rel = %req.rel, req_lsn = %req.hdr.request_lsn); let shard = timeline_handles .get(tenant_id, timeline_id, ShardSelector::Zero) .instrument(span.clone()) // sets `shard_id` field @@ -731,7 +751,7 @@ impl PageServerHandler { } } PagestreamFeMessage::DbSize(req) => { - let span = tracing::info_span!(parent: parent_span, "handle_db_size_request", dbnode = %req.dbnode, req_lsn = %req.request_lsn); + let span = tracing::info_span!(parent: parent_span, "handle_db_size_request", dbnode = %req.dbnode, req_lsn = %req.hdr.request_lsn); let shard = timeline_handles .get(tenant_id, timeline_id, ShardSelector::Zero) .instrument(span.clone()) // sets `shard_id` field @@ -747,7 +767,7 @@ impl PageServerHandler { } } PagestreamFeMessage::GetSlruSegment(req) => { - let span = tracing::info_span!(parent: parent_span, "handle_get_slru_segment_request", kind = %req.kind, segno = %req.segno, req_lsn = %req.request_lsn); + let span = tracing::info_span!(parent: parent_span, "handle_get_slru_segment_request", kind = %req.kind, segno = %req.segno, req_lsn = %req.hdr.request_lsn); let shard = timeline_handles .get(tenant_id, timeline_id, ShardSelector::Zero) .instrument(span.clone()) // sets `shard_id` field @@ -762,25 +782,23 @@ impl PageServerHandler { req, } } - PagestreamFeMessage::GetPage(PagestreamGetPageRequest { - request_lsn, - not_modified_since, - rel, - blkno, - }) => { - let span = tracing::info_span!(parent: parent_span, "handle_get_page_at_lsn_request_batched", req_lsn = %request_lsn); + PagestreamFeMessage::GetPage(req) => { + let span = tracing::info_span!(parent: parent_span, "handle_get_page_at_lsn_request_batched", req_lsn = %req.hdr.request_lsn); macro_rules! respond_error { ($error:expr) => {{ let error = BatchedFeMessage::RespondError { span, - error: $error, + error: BatchedPageStreamError { + req: req.hdr, + err: $error, + }, }; Ok(Some(error)) }}; } - let key = rel_block_to_key(rel, blkno); + let key = rel_block_to_key(req.rel, req.blkno); let shard = match timeline_handles .get(tenant_id, timeline_id, ShardSelector::Page(key)) .instrument(span.clone()) // sets `shard_id` field @@ -814,8 +832,8 @@ impl PageServerHandler { let effective_request_lsn = match Self::wait_or_get_last_lsn( &shard, - request_lsn, - not_modified_since, + req.hdr.request_lsn, + req.hdr.not_modified_since, &shard.get_latest_gc_cutoff_lsn(), ctx, ) @@ -831,7 +849,7 @@ impl PageServerHandler { span, shard, effective_request_lsn, - pages: smallvec::smallvec![(rel, blkno, timer)], + pages: smallvec::smallvec![BatchedGetPageRequest { req, timer }], } } }; @@ -910,6 +928,7 @@ impl PageServerHandler { pgb_writer: &mut PostgresBackend, batch: BatchedFeMessage, cancel: &CancellationToken, + protocol_version: PagestreamProtocolVersion, ctx: &RequestContext, ) -> Result<(), QueryError> where @@ -917,7 +936,7 @@ impl PageServerHandler { { // invoke handler function let (handler_results, span): ( - Vec>, + Vec>, _, ) = match batch { BatchedFeMessage::Exists { @@ -932,7 +951,8 @@ impl PageServerHandler { .handle_get_rel_exists_request(&shard, &req, ctx) .instrument(span.clone()) .await - .map(|msg| (msg, timer))], + .map(|msg| (msg, timer)) + .map_err(|err| BatchedPageStreamError { err, req: req.hdr })], span, ) } @@ -948,7 +968,8 @@ impl PageServerHandler { .handle_get_nblocks_request(&shard, &req, ctx) .instrument(span.clone()) .await - .map(|msg| (msg, timer))], + .map(|msg| (msg, timer)) + .map_err(|err| BatchedPageStreamError { err, req: req.hdr })], span, ) } @@ -990,7 +1011,8 @@ impl PageServerHandler { .handle_db_size_request(&shard, &req, ctx) .instrument(span.clone()) .await - .map(|msg| (msg, timer))], + .map(|msg| (msg, timer)) + .map_err(|err| BatchedPageStreamError { err, req: req.hdr })], span, ) } @@ -1006,7 +1028,8 @@ impl PageServerHandler { .handle_get_slru_segment_request(&shard, &req, ctx) .instrument(span.clone()) .await - .map(|msg| (msg, timer))], + .map(|msg| (msg, timer)) + .map_err(|err| BatchedPageStreamError { err, req: req.hdr })], span, ) } @@ -1022,7 +1045,7 @@ impl PageServerHandler { // Other handler errors are sent back as an error message and we stay in pagestream protocol. for handler_result in handler_results { let (response_msg, timer) = match handler_result { - Err(e) => match &e { + Err(e) => match &e.err { PageStreamError::Shutdown => { // If we fail to fulfil a request during shutdown, which may be _because_ of // shutdown, then do not send the error to the client. Instead just drop the @@ -1041,13 +1064,14 @@ impl PageServerHandler { // print the all details to the log with {:#}, but for the client the // error message is enough. Do not log if shutting down, as the anyhow::Error // here includes cancellation which is not an error. - let full = utils::error::report_compact_sources(&e); + let full = utils::error::report_compact_sources(&e.err); span.in_scope(|| { error!("error reading relation or page version: {full:#}") }); ( PagestreamBeMessage::Error(PagestreamErrorResponse { - message: e.to_string(), + req: e.req, + message: e.err.to_string(), }), None, // TODO: measure errors ) @@ -1060,7 +1084,9 @@ impl PageServerHandler { // marshal & transmit response message // - pgb_writer.write_message_noflush(&BeMessage::CopyData(&response_msg.serialize()))?; + pgb_writer.write_message_noflush(&BeMessage::CopyData( + &response_msg.serialize(protocol_version), + ))?; // We purposefully don't count flush time into the timer. // @@ -1123,7 +1149,7 @@ impl PageServerHandler { pgb: &mut PostgresBackend, tenant_id: TenantId, timeline_id: TimelineId, - _protocol_version: PagestreamProtocolVersion, + protocol_version: PagestreamProtocolVersion, ctx: RequestContext, ) -> Result<(), QueryError> where @@ -1163,6 +1189,7 @@ impl PageServerHandler { timeline_handles, request_span, pipelining_config, + protocol_version, &ctx, ) .await @@ -1175,6 +1202,7 @@ impl PageServerHandler { timeline_id, timeline_handles, request_span, + protocol_version, &ctx, ) .await @@ -1201,6 +1229,7 @@ impl PageServerHandler { timeline_id: TimelineId, mut timeline_handles: TimelineHandles, request_span: Span, + protocol_version: PagestreamProtocolVersion, ctx: &RequestContext, ) -> ( (PostgresBackendReader, TimelineHandles), @@ -1218,6 +1247,7 @@ impl PageServerHandler { &mut timeline_handles, &cancel, ctx, + protocol_version, request_span.clone(), ) .await; @@ -1238,7 +1268,7 @@ impl PageServerHandler { } let err = self - .pagesteam_handle_batched_message(pgb_writer, msg, &cancel, ctx) + .pagesteam_handle_batched_message(pgb_writer, msg, &cancel, protocol_version, ctx) .await; match err { Ok(()) => {} @@ -1261,6 +1291,7 @@ impl PageServerHandler { mut timeline_handles: TimelineHandles, request_span: Span, pipelining_config: PageServicePipeliningConfigPipelined, + protocol_version: PagestreamProtocolVersion, ctx: &RequestContext, ) -> ( (PostgresBackendReader, TimelineHandles), @@ -1358,6 +1389,7 @@ impl PageServerHandler { &mut timeline_handles, &cancel_batcher, &ctx, + protocol_version, request_span.clone(), ) .await; @@ -1403,8 +1435,14 @@ impl PageServerHandler { batch .throttle_and_record_start_processing(&self.cancel) .await?; - self.pagesteam_handle_batched_message(pgb_writer, batch, &cancel, &ctx) - .await?; + self.pagesteam_handle_batched_message( + pgb_writer, + batch, + &cancel, + protocol_version, + &ctx, + ) + .await?; } } }); @@ -1578,8 +1616,8 @@ impl PageServerHandler { let latest_gc_cutoff_lsn = timeline.get_latest_gc_cutoff_lsn(); let lsn = Self::wait_or_get_last_lsn( timeline, - req.request_lsn, - req.not_modified_since, + req.hdr.request_lsn, + req.hdr.not_modified_since, &latest_gc_cutoff_lsn, ctx, ) @@ -1590,6 +1628,7 @@ impl PageServerHandler { .await?; Ok(PagestreamBeMessage::Exists(PagestreamExistsResponse { + req: *req, exists, })) } @@ -1604,8 +1643,8 @@ impl PageServerHandler { let latest_gc_cutoff_lsn = timeline.get_latest_gc_cutoff_lsn(); let lsn = Self::wait_or_get_last_lsn( timeline, - req.request_lsn, - req.not_modified_since, + req.hdr.request_lsn, + req.hdr.not_modified_since, &latest_gc_cutoff_lsn, ctx, ) @@ -1616,6 +1655,7 @@ impl PageServerHandler { .await?; Ok(PagestreamBeMessage::Nblocks(PagestreamNblocksResponse { + req: *req, n_blocks, })) } @@ -1630,8 +1670,8 @@ impl PageServerHandler { let latest_gc_cutoff_lsn = timeline.get_latest_gc_cutoff_lsn(); let lsn = Self::wait_or_get_last_lsn( timeline, - req.request_lsn, - req.not_modified_since, + req.hdr.request_lsn, + req.hdr.not_modified_since, &latest_gc_cutoff_lsn, ctx, ) @@ -1643,6 +1683,7 @@ impl PageServerHandler { let db_size = total_blocks as i64 * BLCKSZ as i64; Ok(PagestreamBeMessage::DbSize(PagestreamDbSizeResponse { + req: *req, db_size, })) } @@ -1652,9 +1693,9 @@ impl PageServerHandler { &mut self, timeline: &Timeline, effective_lsn: Lsn, - requests: smallvec::SmallVec<[(RelTag, BlockNumber, SmgrOpTimer); 1]>, + requests: smallvec::SmallVec<[BatchedGetPageRequest; 1]>, ctx: &RequestContext, - ) -> Vec> { + ) -> Vec> { debug_assert_current_span_has_tenant_and_timeline_id(); timeline @@ -1663,7 +1704,7 @@ impl PageServerHandler { let results = timeline .get_rel_page_at_lsn_batched( - requests.iter().map(|(reltag, blkno, _)| (reltag, blkno)), + requests.iter().map(|p| (&p.req.rel, &p.req.blkno)), effective_lsn, ctx, ) @@ -1675,16 +1716,20 @@ impl PageServerHandler { requests .into_iter() .zip(results.into_iter()) - .map(|((_, _, timer), res)| { + .map(|(req, res)| { res.map(|page| { ( PagestreamBeMessage::GetPage(models::PagestreamGetPageResponse { + req: req.req, page, }), - timer, + req.timer, ) }) - .map_err(PageStreamError::from) + .map_err(|e| BatchedPageStreamError { + err: PageStreamError::from(e), + req: req.req.hdr, + }) }), ) } @@ -1699,8 +1744,8 @@ impl PageServerHandler { let latest_gc_cutoff_lsn = timeline.get_latest_gc_cutoff_lsn(); let lsn = Self::wait_or_get_last_lsn( timeline, - req.request_lsn, - req.not_modified_since, + req.hdr.request_lsn, + req.hdr.not_modified_since, &latest_gc_cutoff_lsn, ctx, ) @@ -1711,7 +1756,7 @@ impl PageServerHandler { let segment = timeline.get_slru_segment(kind, req.segno, lsn, ctx).await?; Ok(PagestreamBeMessage::GetSlruSegment( - PagestreamGetSlruSegmentResponse { segment }, + PagestreamGetSlruSegmentResponse { req: *req, segment }, )) } @@ -1906,6 +1951,7 @@ struct FullBackupCmd { struct PageStreamCmd { tenant_id: TenantId, timeline_id: TimelineId, + protocol_version: PagestreamProtocolVersion, } /// `lease lsn tenant timeline lsn` @@ -1926,7 +1972,7 @@ enum PageServiceCmd { } impl PageStreamCmd { - fn parse(query: &str) -> anyhow::Result { + fn parse(query: &str, protocol_version: PagestreamProtocolVersion) -> anyhow::Result { let parameters = query.split_whitespace().collect_vec(); if parameters.len() != 2 { bail!( @@ -1941,6 +1987,7 @@ impl PageStreamCmd { Ok(Self { tenant_id, timeline_id, + protocol_version, }) } } @@ -2078,7 +2125,14 @@ impl PageServiceCmd { bail!("cannot parse query: {query}") }; match cmd.to_ascii_lowercase().as_str() { - "pagestream_v2" => Ok(Self::PageStream(PageStreamCmd::parse(other)?)), + "pagestream_v2" => Ok(Self::PageStream(PageStreamCmd::parse( + other, + PagestreamProtocolVersion::V2, + )?)), + "pagestream_v3" => Ok(Self::PageStream(PageStreamCmd::parse( + other, + PagestreamProtocolVersion::V3, + )?)), "basebackup" => Ok(Self::BaseBackup(BaseBackupCmd::parse(other)?)), "fullbackup" => Ok(Self::FullBackup(FullBackupCmd::parse(other)?)), "lease" => { @@ -2160,25 +2214,21 @@ where PageServiceCmd::PageStream(PageStreamCmd { tenant_id, timeline_id, + protocol_version, }) => { tracing::Span::current() .record("tenant_id", field::display(tenant_id)) .record("timeline_id", field::display(timeline_id)); self.check_permission(Some(tenant_id))?; + let command_kind = match protocol_version { + PagestreamProtocolVersion::V2 => ComputeCommandKind::PageStreamV2, + PagestreamProtocolVersion::V3 => ComputeCommandKind::PageStreamV3, + }; + COMPUTE_COMMANDS_COUNTERS.for_command(command_kind).inc(); - COMPUTE_COMMANDS_COUNTERS - .for_command(ComputeCommandKind::PageStreamV2) - .inc(); - - self.handle_pagerequests( - pgb, - tenant_id, - timeline_id, - PagestreamProtocolVersion::V2, - ctx, - ) - .await?; + self.handle_pagerequests(pgb, tenant_id, timeline_id, protocol_version, ctx) + .await?; } PageServiceCmd::BaseBackup(BaseBackupCmd { tenant_id, @@ -2357,7 +2407,8 @@ mod tests { cmd, PageServiceCmd::PageStream(PageStreamCmd { tenant_id, - timeline_id + timeline_id, + protocol_version: PagestreamProtocolVersion::V2, }) ); let cmd = PageServiceCmd::parse(&format!("basebackup {tenant_id} {timeline_id}")).unwrap(); diff --git a/pgxn/neon/libpagestore.c b/pgxn/neon/libpagestore.c index fa2a570ea8..769befb4e5 100644 --- a/pgxn/neon/libpagestore.c +++ b/pgxn/neon/libpagestore.c @@ -556,6 +556,9 @@ pageserver_connect(shardno_t shard_no, int elevel) switch (neon_protocol_version) { + case 3: + pagestream_query = psprintf("pagestream_v3 %s %s", neon_tenant, neon_timeline); + break; case 2: pagestream_query = psprintf("pagestream_v2 %s %s", neon_tenant, neon_timeline); break; @@ -1135,9 +1138,9 @@ pg_init_libpagestore(void) "Version of compute<->page server protocol", NULL, &neon_protocol_version, - 2, /* use protocol version 2 */ - 2, /* min */ - 2, /* max */ + 2, /* use protocol version 2 */ + 2, /* min */ + 3, /* max */ PGC_SU_BACKEND, 0, /* no flags required */ NULL, NULL, NULL); diff --git a/pgxn/neon/pagestore_client.h b/pgxn/neon/pagestore_client.h index f905e3b0fa..37bc4f7886 100644 --- a/pgxn/neon/pagestore_client.h +++ b/pgxn/neon/pagestore_client.h @@ -44,10 +44,15 @@ typedef enum T_NeonGetSlruSegmentResponse, } NeonMessageTag; +typedef uint64 NeonRequestId; + /* base struct for c-style inheritance */ typedef struct { NeonMessageTag tag; + NeonRequestId reqid; + XLogRecPtr lsn; + XLogRecPtr not_modified_since; } NeonMessage; #define messageTag(m) (((const NeonMessage *)(m))->tag) @@ -67,6 +72,7 @@ typedef enum { SLRU_MULTIXACT_OFFSETS } SlruKind; + /*-- * supertype of all the Neon*Request structs below. * @@ -87,37 +93,37 @@ typedef enum { * * These structs describe the V2 of these requests. (The old now-defunct V1 * protocol contained just one LSN and a boolean 'latest' flag.) + * + * V3 version of protocol adds request ID to all requests. This request ID is also included in response + * as well as other fields from requests, which allows to verify that we receive response for our request. + * We copy fields from request to response to make checking more reliable: request ID is formed from process ID + * and local counter, so in principle there can be duplicated requests IDs if process PID is reused. */ -typedef struct -{ - NeonMessageTag tag; - XLogRecPtr lsn; - XLogRecPtr not_modified_since; -} NeonRequest; +typedef NeonMessage NeonRequest; typedef struct { - NeonRequest req; + NeonRequest hdr; NRelFileInfo rinfo; ForkNumber forknum; } NeonExistsRequest; typedef struct { - NeonRequest req; + NeonRequest hdr; NRelFileInfo rinfo; ForkNumber forknum; } NeonNblocksRequest; typedef struct { - NeonRequest req; + NeonRequest hdr; Oid dbNode; } NeonDbSizeRequest; typedef struct { - NeonRequest req; + NeonRequest hdr; NRelFileInfo rinfo; ForkNumber forknum; BlockNumber blkno; @@ -125,32 +131,29 @@ typedef struct typedef struct { - NeonRequest req; - SlruKind kind; - int segno; + NeonRequest hdr; + SlruKind kind; + int segno; } NeonGetSlruSegmentRequest; /* supertype of all the Neon*Response structs below */ -typedef struct -{ - NeonMessageTag tag; -} NeonResponse; +typedef NeonMessage NeonResponse; typedef struct { - NeonMessageTag tag; + NeonExistsRequest req; bool exists; } NeonExistsResponse; typedef struct { - NeonMessageTag tag; + NeonNblocksRequest req; uint32 n_blocks; } NeonNblocksResponse; typedef struct { - NeonMessageTag tag; + NeonGetPageRequest req; char page[FLEXIBLE_ARRAY_MEMBER]; } NeonGetPageResponse; @@ -158,21 +161,21 @@ typedef struct typedef struct { - NeonMessageTag tag; + NeonDbSizeRequest req; int64 db_size; } NeonDbSizeResponse; typedef struct { - NeonMessageTag tag; + NeonResponse req; char message[FLEXIBLE_ARRAY_MEMBER]; /* null-terminated error * message */ } NeonErrorResponse; typedef struct { - NeonMessageTag tag; - int n_blocks; + NeonGetSlruSegmentRequest req; + int n_blocks; char data[BLCKSZ * SLRU_PAGES_PER_SEGMENT]; } NeonGetSlruSegmentResponse; diff --git a/pgxn/neon/pagestore_smgr.c b/pgxn/neon/pagestore_smgr.c index b733807026..7a4c0ef487 100644 --- a/pgxn/neon/pagestore_smgr.c +++ b/pgxn/neon/pagestore_smgr.c @@ -120,6 +120,9 @@ static bool (*old_redo_read_buffer_filter) (XLogReaderState *record, uint8 block static BlockNumber neon_nblocks(SMgrRelation reln, ForkNumber forknum); +static uint32 local_request_counter; +#define GENERATE_REQUEST_ID() (((NeonRequestId)MyProcPid << 32) | ++local_request_counter) + /* * Prefetch implementation: * @@ -188,15 +191,11 @@ typedef struct PrefetchRequest uint8 status; /* see PrefetchStatus for valid values */ uint8 flags; /* see PrefetchRequestFlags */ neon_request_lsns request_lsns; + NeonRequestId reqid; NeonResponse *response; /* may be null */ uint64 my_ring_index; } PrefetchRequest; -StaticAssertDecl(sizeof(PrefetchRequest) == 64, - "We prefer to have a power-of-2 size for this struct. Please" - " try to find an alternative solution before reaching to" - " increase the expected size here"); - /* prefetch buffer lookup hash table */ typedef struct PrfHashEntry @@ -365,6 +364,7 @@ compact_prefetch_buffers(void) target_slot->shard_no = source_slot->shard_no; target_slot->status = source_slot->status; target_slot->response = source_slot->response; + target_slot->reqid = source_slot->reqid; target_slot->request_lsns = source_slot->request_lsns; target_slot->my_ring_index = empty_ring_index; @@ -798,7 +798,8 @@ prefetch_do_request(PrefetchRequest *slot, neon_request_lsns *force_request_lsns uint64 mySlotNo PG_USED_FOR_ASSERTS_ONLY = slot->my_ring_index; NeonGetPageRequest request = { - .req.tag = T_NeonGetPageRequest, + .hdr.tag = T_NeonGetPageRequest, + .hdr.reqid = GENERATE_REQUEST_ID(), /* lsn and not_modified_since are filled in below */ .rinfo = BufTagGetNRelFileInfo(slot->buftag), .forknum = slot->buftag.forkNum, @@ -807,14 +808,16 @@ prefetch_do_request(PrefetchRequest *slot, neon_request_lsns *force_request_lsns Assert(mySlotNo == MyPState->ring_unused); + slot->reqid = request.hdr.reqid; + if (force_request_lsns) slot->request_lsns = *force_request_lsns; else neon_get_request_lsns(BufTagGetNRelFileInfo(slot->buftag), slot->buftag.forkNum, slot->buftag.blockNum, &slot->request_lsns, 1, NULL); - request.req.lsn = slot->request_lsns.request_lsn; - request.req.not_modified_since = slot->request_lsns.not_modified_since; + request.hdr.lsn = slot->request_lsns.request_lsn; + request.hdr.not_modified_since = slot->request_lsns.not_modified_since; Assert(slot->response == NULL); Assert(slot->my_ring_index == MyPState->ring_unused); @@ -1102,6 +1105,12 @@ Retry: return min_ring_index; } +static bool +equal_requests(NeonRequest* a, NeonRequest* b) +{ + return a->reqid == b->reqid && a->lsn == b->lsn && a->not_modified_since == b->not_modified_since; +} + /* * Note: this function can get canceled and use a long jump to the next catch @@ -1184,6 +1193,10 @@ nm_pack_request(NeonRequest *msg) initStringInfo(&s); pq_sendbyte(&s, msg->tag); + if (neon_protocol_version >= 3) + { + pq_sendint64(&s, msg->reqid); + } pq_sendint64(&s, msg->lsn); pq_sendint64(&s, msg->not_modified_since); @@ -1261,8 +1274,16 @@ NeonResponse * nm_unpack_response(StringInfo s) { NeonMessageTag tag = pq_getmsgbyte(s); + NeonResponse resp_hdr = {0}; /* make valgrind happy */ NeonResponse *resp = NULL; + resp_hdr.tag = tag; + if (neon_protocol_version >= 3) + { + resp_hdr.reqid = pq_getmsgint64(s); + resp_hdr.lsn = pq_getmsgint64(s); + resp_hdr.not_modified_since = pq_getmsgint64(s); + } switch (tag) { /* pagestore -> pagestore_client */ @@ -1270,7 +1291,14 @@ nm_unpack_response(StringInfo s) { NeonExistsResponse *msg_resp = palloc0(sizeof(NeonExistsResponse)); - msg_resp->tag = tag; + if (neon_protocol_version >= 3) + { + NInfoGetSpcOid(msg_resp->req.rinfo) = pq_getmsgint(s, 4); + NInfoGetDbOid(msg_resp->req.rinfo) = pq_getmsgint(s, 4); + NInfoGetRelNumber(msg_resp->req.rinfo) = pq_getmsgint(s, 4); + msg_resp->req.forknum = pq_getmsgbyte(s); + } + msg_resp->req.hdr = resp_hdr; msg_resp->exists = pq_getmsgbyte(s); pq_getmsgend(s); @@ -1282,7 +1310,14 @@ nm_unpack_response(StringInfo s) { NeonNblocksResponse *msg_resp = palloc0(sizeof(NeonNblocksResponse)); - msg_resp->tag = tag; + if (neon_protocol_version >= 3) + { + NInfoGetSpcOid(msg_resp->req.rinfo) = pq_getmsgint(s, 4); + NInfoGetDbOid(msg_resp->req.rinfo) = pq_getmsgint(s, 4); + NInfoGetRelNumber(msg_resp->req.rinfo) = pq_getmsgint(s, 4); + msg_resp->req.forknum = pq_getmsgbyte(s); + } + msg_resp->req.hdr = resp_hdr; msg_resp->n_blocks = pq_getmsgint(s, 4); pq_getmsgend(s); @@ -1295,12 +1330,20 @@ nm_unpack_response(StringInfo s) NeonGetPageResponse *msg_resp; msg_resp = MemoryContextAllocZero(MyPState->bufctx, PS_GETPAGERESPONSE_SIZE); - msg_resp->tag = tag; + if (neon_protocol_version >= 3) + { + NInfoGetSpcOid(msg_resp->req.rinfo) = pq_getmsgint(s, 4); + NInfoGetDbOid(msg_resp->req.rinfo) = pq_getmsgint(s, 4); + NInfoGetRelNumber(msg_resp->req.rinfo) = pq_getmsgint(s, 4); + msg_resp->req.forknum = pq_getmsgbyte(s); + msg_resp->req.blkno = pq_getmsgint(s, 4); + } + msg_resp->req.hdr = resp_hdr; /* XXX: should be varlena */ memcpy(msg_resp->page, pq_getmsgbytes(s, BLCKSZ), BLCKSZ); pq_getmsgend(s); - Assert(msg_resp->tag == T_NeonGetPageResponse); + Assert(msg_resp->req.hdr.tag == T_NeonGetPageResponse); resp = (NeonResponse *) msg_resp; break; @@ -1310,7 +1353,11 @@ nm_unpack_response(StringInfo s) { NeonDbSizeResponse *msg_resp = palloc0(sizeof(NeonDbSizeResponse)); - msg_resp->tag = tag; + if (neon_protocol_version >= 3) + { + msg_resp->req.dbNode = pq_getmsgint(s, 4); + } + msg_resp->req.hdr = resp_hdr; msg_resp->db_size = pq_getmsgint64(s); pq_getmsgend(s); @@ -1328,7 +1375,7 @@ nm_unpack_response(StringInfo s) msglen = strlen(msgtext); msg_resp = palloc0(sizeof(NeonErrorResponse) + msglen + 1); - msg_resp->tag = tag; + msg_resp->req = resp_hdr; memcpy(msg_resp->message, msgtext, msglen + 1); pq_getmsgend(s); @@ -1339,9 +1386,17 @@ nm_unpack_response(StringInfo s) case T_NeonGetSlruSegmentResponse: { NeonGetSlruSegmentResponse *msg_resp; - int n_blocks = pq_getmsgint(s, 4); - msg_resp = palloc(sizeof(NeonGetSlruSegmentResponse)); - msg_resp->tag = tag; + int n_blocks; + msg_resp = palloc0(sizeof(NeonGetSlruSegmentResponse)); + + if (neon_protocol_version >= 3) + { + msg_resp->req.kind = pq_getmsgbyte(s); + msg_resp->req.segno = pq_getmsgint(s, 4); + } + msg_resp->req.hdr = resp_hdr; + + n_blocks = pq_getmsgint(s, 4); msg_resp->n_blocks = n_blocks; memcpy(msg_resp->data, pq_getmsgbytes(s, n_blocks * BLCKSZ), n_blocks * BLCKSZ); pq_getmsgend(s); @@ -1386,8 +1441,8 @@ nm_to_string(NeonMessage *msg) appendStringInfoString(&s, "{\"type\": \"NeonExistsRequest\""); appendStringInfo(&s, ", \"rinfo\": \"%u/%u/%u\"", RelFileInfoFmt(msg_req->rinfo)); appendStringInfo(&s, ", \"forknum\": %d", msg_req->forknum); - appendStringInfo(&s, ", \"lsn\": \"%X/%X\"", LSN_FORMAT_ARGS(msg_req->req.lsn)); - appendStringInfo(&s, ", \"not_modified_since\": \"%X/%X\"", LSN_FORMAT_ARGS(msg_req->req.not_modified_since)); + appendStringInfo(&s, ", \"lsn\": \"%X/%X\"", LSN_FORMAT_ARGS(msg_req->hdr.lsn)); + appendStringInfo(&s, ", \"not_modified_since\": \"%X/%X\"", LSN_FORMAT_ARGS(msg_req->hdr.not_modified_since)); appendStringInfoChar(&s, '}'); break; } @@ -1399,8 +1454,8 @@ nm_to_string(NeonMessage *msg) appendStringInfoString(&s, "{\"type\": \"NeonNblocksRequest\""); appendStringInfo(&s, ", \"rinfo\": \"%u/%u/%u\"", RelFileInfoFmt(msg_req->rinfo)); appendStringInfo(&s, ", \"forknum\": %d", msg_req->forknum); - appendStringInfo(&s, ", \"lsn\": \"%X/%X\"", LSN_FORMAT_ARGS(msg_req->req.lsn)); - appendStringInfo(&s, ", \"not_modified_since\": \"%X/%X\"", LSN_FORMAT_ARGS(msg_req->req.not_modified_since)); + appendStringInfo(&s, ", \"lsn\": \"%X/%X\"", LSN_FORMAT_ARGS(msg_req->hdr.lsn)); + appendStringInfo(&s, ", \"not_modified_since\": \"%X/%X\"", LSN_FORMAT_ARGS(msg_req->hdr.not_modified_since)); appendStringInfoChar(&s, '}'); break; } @@ -1413,8 +1468,8 @@ nm_to_string(NeonMessage *msg) appendStringInfo(&s, ", \"rinfo\": \"%u/%u/%u\"", RelFileInfoFmt(msg_req->rinfo)); appendStringInfo(&s, ", \"forknum\": %d", msg_req->forknum); appendStringInfo(&s, ", \"blkno\": %u", msg_req->blkno); - appendStringInfo(&s, ", \"lsn\": \"%X/%X\"", LSN_FORMAT_ARGS(msg_req->req.lsn)); - appendStringInfo(&s, ", \"not_modified_since\": \"%X/%X\"", LSN_FORMAT_ARGS(msg_req->req.not_modified_since)); + appendStringInfo(&s, ", \"lsn\": \"%X/%X\"", LSN_FORMAT_ARGS(msg_req->hdr.lsn)); + appendStringInfo(&s, ", \"not_modified_since\": \"%X/%X\"", LSN_FORMAT_ARGS(msg_req->hdr.not_modified_since)); appendStringInfoChar(&s, '}'); break; } @@ -1424,8 +1479,8 @@ nm_to_string(NeonMessage *msg) appendStringInfoString(&s, "{\"type\": \"NeonDbSizeRequest\""); appendStringInfo(&s, ", \"dbnode\": \"%u\"", msg_req->dbNode); - appendStringInfo(&s, ", \"lsn\": \"%X/%X\"", LSN_FORMAT_ARGS(msg_req->req.lsn)); - appendStringInfo(&s, ", \"not_modified_since\": \"%X/%X\"", LSN_FORMAT_ARGS(msg_req->req.not_modified_since)); + appendStringInfo(&s, ", \"lsn\": \"%X/%X\"", LSN_FORMAT_ARGS(msg_req->hdr.lsn)); + appendStringInfo(&s, ", \"not_modified_since\": \"%X/%X\"", LSN_FORMAT_ARGS(msg_req->hdr.not_modified_since)); appendStringInfoChar(&s, '}'); break; } @@ -1436,8 +1491,8 @@ nm_to_string(NeonMessage *msg) appendStringInfoString(&s, "{\"type\": \"NeonGetSlruSegmentRequest\""); appendStringInfo(&s, ", \"kind\": %u", msg_req->kind); appendStringInfo(&s, ", \"segno\": %u", msg_req->segno); - appendStringInfo(&s, ", \"lsn\": \"%X/%X\"", LSN_FORMAT_ARGS(msg_req->req.lsn)); - appendStringInfo(&s, ", \"not_modified_since\": \"%X/%X\"", LSN_FORMAT_ARGS(msg_req->req.not_modified_since)); + appendStringInfo(&s, ", \"lsn\": \"%X/%X\"", LSN_FORMAT_ARGS(msg_req->hdr.lsn)); + appendStringInfo(&s, ", \"not_modified_since\": \"%X/%X\"", LSN_FORMAT_ARGS(msg_req->hdr.not_modified_since)); appendStringInfoChar(&s, '}'); break; } @@ -2312,39 +2367,64 @@ neon_exists(SMgrRelation reln, ForkNumber forkNum) REL_METADATA_PSEUDO_BLOCKNO, &request_lsns, 1, NULL); { NeonExistsRequest request = { - .req.tag = T_NeonExistsRequest, - .req.lsn = request_lsns.request_lsn, - .req.not_modified_since = request_lsns.not_modified_since, + .hdr.tag = T_NeonExistsRequest, + .hdr.reqid = GENERATE_REQUEST_ID(), + .hdr.lsn = request_lsns.request_lsn, + .hdr.not_modified_since = request_lsns.not_modified_since, .rinfo = InfoFromSMgrRel(reln), .forknum = forkNum }; resp = page_server_request(&request); + + switch (resp->tag) + { + case T_NeonExistsResponse: + { + NeonExistsResponse* exists_resp = (NeonExistsResponse *) resp; + if (neon_protocol_version >= 3) + { + if (!equal_requests(resp, &request.hdr) || + !RelFileInfoEquals(exists_resp->req.rinfo, request.rinfo) || + exists_resp->req.forknum != request.forknum) + { + NEON_PANIC_CONNECTION_STATE(-1, PANIC, + "Unexpect response {reqid=%lx,lsn=%X/%08X, since=%X/%08X, rel=%u/%u/%u.%u} to exits request {reqid=%lx,lsn=%X/%08X, since=%X/%08X, rel=%u/%u/%u.%u}", + resp->reqid, LSN_FORMAT_ARGS(resp->lsn), LSN_FORMAT_ARGS(resp->not_modified_since), RelFileInfoFmt(exists_resp->req.rinfo), exists_resp->req.forknum, + request.hdr.reqid, LSN_FORMAT_ARGS(request.hdr.lsn), LSN_FORMAT_ARGS(request.hdr.not_modified_since), RelFileInfoFmt(request.rinfo), request.forknum); + } + } + exists = exists_resp->exists; + break; + } + case T_NeonErrorResponse: + if (neon_protocol_version >= 3) + { + if (!equal_requests(resp, &request.hdr)) + { + elog(WARNING, NEON_TAG "Error message {reqid=%lx,lsn=%X/%08X, since=%X/%08X} doesn't match exists request {reqid=%lx,lsn=%X/%08X, since=%X/%08X}", + resp->reqid, LSN_FORMAT_ARGS(resp->lsn), LSN_FORMAT_ARGS(resp->not_modified_since), + request.hdr.reqid, LSN_FORMAT_ARGS(request.hdr.lsn), LSN_FORMAT_ARGS(request.hdr.not_modified_since)); + } + } + ereport(ERROR, + (errcode(ERRCODE_IO_ERROR), + errmsg(NEON_TAG "[reqid %lx] could not read relation existence of rel %u/%u/%u.%u from page server at lsn %X/%08X", + resp->reqid, + RelFileInfoFmt(InfoFromSMgrRel(reln)), + forkNum, + LSN_FORMAT_ARGS(request_lsns.effective_request_lsn)), + errdetail("page server returned error: %s", + ((NeonErrorResponse *) resp)->message))); + break; + + default: + NEON_PANIC_CONNECTION_STATE(-1, PANIC, + "Expected Exists (0x%02x) or Error (0x%02x) response to ExistsRequest, but got 0x%02x", + T_NeonExistsResponse, T_NeonErrorResponse, resp->tag); + } + pfree(resp); } - - switch (resp->tag) - { - case T_NeonExistsResponse: - exists = ((NeonExistsResponse *) resp)->exists; - break; - - case T_NeonErrorResponse: - ereport(ERROR, - (errcode(ERRCODE_IO_ERROR), - errmsg(NEON_TAG "could not read relation existence of rel %u/%u/%u.%u from page server at lsn %X/%08X", - RelFileInfoFmt(InfoFromSMgrRel(reln)), - forkNum, - LSN_FORMAT_ARGS(request_lsns.effective_request_lsn)), - errdetail("page server returned error: %s", - ((NeonErrorResponse *) resp)->message))); - break; - - default: - NEON_PANIC_CONNECTION_STATE(-1, PANIC, - "Expected Exists (0x%02x) or Error (0x%02x) response to ExistsRequest, but got 0x%02x", - T_NeonExistsResponse, T_NeonErrorResponse, resp->tag); - } - pfree(resp); return exists; } @@ -2952,15 +3032,43 @@ Retry: switch (resp->tag) { case T_NeonGetPageResponse: - memcpy(buffer, ((NeonGetPageResponse *) resp)->page, BLCKSZ); + { + NeonGetPageResponse* getpage_resp = (NeonGetPageResponse *) resp; + if (neon_protocol_version >= 3) + { + if (resp->reqid != slot->reqid || + resp->lsn != slot->request_lsns.request_lsn || + resp->not_modified_since != slot->request_lsns.not_modified_since || + !RelFileInfoEquals(getpage_resp->req.rinfo, rinfo) || + getpage_resp->req.forknum != forkNum || + getpage_resp->req.blkno != base_blockno + i) + { + NEON_PANIC_CONNECTION_STATE(-1, PANIC, + "Unexpect response {reqid=%lx,lsn=%X/%08X, since=%X/%08X, rel=%u/%u/%u.%u, block=%u} to get page request {reqid=%lx,lsn=%X/%08X, since=%X/%08X, rel=%u/%u/%u.%u, block=%u}", + resp->reqid, LSN_FORMAT_ARGS(resp->lsn), LSN_FORMAT_ARGS(resp->not_modified_since), RelFileInfoFmt(getpage_resp->req.rinfo), getpage_resp->req.forknum, getpage_resp->req.blkno, + slot->reqid, LSN_FORMAT_ARGS(slot->request_lsns.request_lsn), LSN_FORMAT_ARGS(slot->request_lsns.not_modified_since), RelFileInfoFmt(rinfo), forkNum, base_blockno + i); + } + } + memcpy(buffer, getpage_resp->page, BLCKSZ); lfc_write(rinfo, forkNum, blockno, buffer); break; - + } case T_NeonErrorResponse: + if (neon_protocol_version >= 3) + { + if (resp->reqid != slot->reqid || + resp->lsn != slot->request_lsns.request_lsn || + resp->not_modified_since != slot->request_lsns.not_modified_since) + { + elog(WARNING, NEON_TAG "Error message {reqid=%lx,lsn=%X/%08X, since=%X/%08X} doesn't match get relsize request {reqid=%lx,lsn=%X/%08X, since=%X/%08X}", + resp->reqid, LSN_FORMAT_ARGS(resp->lsn), LSN_FORMAT_ARGS(resp->not_modified_since), + slot->reqid, LSN_FORMAT_ARGS(slot->request_lsns.request_lsn), LSN_FORMAT_ARGS(slot->request_lsns.not_modified_since)); + } + } ereport(ERROR, (errcode(ERRCODE_IO_ERROR), - errmsg(NEON_TAG "[shard %d] could not read block %u in rel %u/%u/%u.%u from page server at lsn %X/%08X", - slot->shard_no, blockno, RelFileInfoFmt(rinfo), + errmsg(NEON_TAG "[shard %d, reqid %lx] could not read block %u in rel %u/%u/%u.%u from page server at lsn %X/%08X", + slot->shard_no, resp->reqid, blockno, RelFileInfoFmt(rinfo), forkNum, LSN_FORMAT_ARGS(reqlsns->effective_request_lsn)), errdetail("page server returned error: %s", ((NeonErrorResponse *) resp)->message))); @@ -3443,47 +3551,72 @@ neon_nblocks(SMgrRelation reln, ForkNumber forknum) { NeonNblocksRequest request = { - .req.tag = T_NeonNblocksRequest, - .req.lsn = request_lsns.request_lsn, - .req.not_modified_since = request_lsns.not_modified_since, + .hdr.tag = T_NeonNblocksRequest, + .hdr.reqid = GENERATE_REQUEST_ID(), + .hdr.lsn = request_lsns.request_lsn, + .hdr.not_modified_since = request_lsns.not_modified_since, .rinfo = InfoFromSMgrRel(reln), .forknum = forknum, }; resp = page_server_request(&request); + + switch (resp->tag) + { + case T_NeonNblocksResponse: + { + NeonNblocksResponse * relsize_resp = (NeonNblocksResponse *) resp; + if (neon_protocol_version >= 3) + { + if (!equal_requests(resp, &request.hdr) || + !RelFileInfoEquals(relsize_resp->req.rinfo, request.rinfo) || + relsize_resp->req.forknum != forknum) + { + NEON_PANIC_CONNECTION_STATE(-1, PANIC, + "Unexpect response {reqid=%lx,lsn=%X/%08X, since=%X/%08X, rel=%u/%u/%u.%u} to get relsize request {reqid=%lx,lsn=%X/%08X, since=%X/%08X, rel=%u/%u/%u.%u}", + resp->reqid, LSN_FORMAT_ARGS(resp->lsn), LSN_FORMAT_ARGS(resp->not_modified_since), RelFileInfoFmt(relsize_resp->req.rinfo), relsize_resp->req.forknum, + request.hdr.reqid, LSN_FORMAT_ARGS(request.hdr.lsn), LSN_FORMAT_ARGS(request.hdr.not_modified_since), RelFileInfoFmt(request.rinfo), forknum); + } + } + n_blocks = relsize_resp->n_blocks; + break; + } + case T_NeonErrorResponse: + if (neon_protocol_version >= 3) + { + if (!equal_requests(resp, &request.hdr)) + { + elog(WARNING, NEON_TAG "Error message {reqid=%lx,lsn=%X/%08X, since=%X/%08X} doesn't match get relsize request {reqid=%lx,lsn=%X/%08X, since=%X/%08X}", + resp->reqid, LSN_FORMAT_ARGS(resp->lsn), LSN_FORMAT_ARGS(resp->not_modified_since), + request.hdr.reqid, LSN_FORMAT_ARGS(request.hdr.lsn), LSN_FORMAT_ARGS(request.hdr.not_modified_since)); + } + } + ereport(ERROR, + (errcode(ERRCODE_IO_ERROR), + errmsg(NEON_TAG "[reqid %lx] could not read relation size of rel %u/%u/%u.%u from page server at lsn %X/%08X", + resp->reqid, + RelFileInfoFmt(InfoFromSMgrRel(reln)), + forknum, + LSN_FORMAT_ARGS(request_lsns.effective_request_lsn)), + errdetail("page server returned error: %s", + ((NeonErrorResponse *) resp)->message))); + break; + + default: + NEON_PANIC_CONNECTION_STATE(-1, PANIC, + "Expected Nblocks (0x%02x) or Error (0x%02x) response to NblocksRequest, but got 0x%02x", + T_NeonNblocksResponse, T_NeonErrorResponse, resp->tag); + } + update_cached_relsize(InfoFromSMgrRel(reln), forknum, n_blocks); + + neon_log(SmgrTrace, "neon_nblocks: rel %u/%u/%u fork %u (request LSN %X/%08X): %u blocks", + RelFileInfoFmt(InfoFromSMgrRel(reln)), + forknum, + LSN_FORMAT_ARGS(request_lsns.effective_request_lsn), + n_blocks); + + pfree(resp); } - - switch (resp->tag) - { - case T_NeonNblocksResponse: - n_blocks = ((NeonNblocksResponse *) resp)->n_blocks; - break; - - case T_NeonErrorResponse: - ereport(ERROR, - (errcode(ERRCODE_IO_ERROR), - errmsg(NEON_TAG "could not read relation size of rel %u/%u/%u.%u from page server at lsn %X/%08X", - RelFileInfoFmt(InfoFromSMgrRel(reln)), - forknum, - LSN_FORMAT_ARGS(request_lsns.effective_request_lsn)), - errdetail("page server returned error: %s", - ((NeonErrorResponse *) resp)->message))); - break; - - default: - NEON_PANIC_CONNECTION_STATE(-1, PANIC, - "Expected Nblocks (0x%02x) or Error (0x%02x) response to NblocksRequest, but got 0x%02x", - T_NeonNblocksResponse, T_NeonErrorResponse, resp->tag); - } - update_cached_relsize(InfoFromSMgrRel(reln), forknum, n_blocks); - - neon_log(SmgrTrace, "neon_nblocks: rel %u/%u/%u fork %u (request LSN %X/%08X): %u blocks", - RelFileInfoFmt(InfoFromSMgrRel(reln)), - forknum, - LSN_FORMAT_ARGS(request_lsns.effective_request_lsn), - n_blocks); - - pfree(resp); return n_blocks; } @@ -3503,40 +3636,64 @@ neon_dbsize(Oid dbNode) { NeonDbSizeRequest request = { - .req.tag = T_NeonDbSizeRequest, - .req.lsn = request_lsns.request_lsn, - .req.not_modified_since = request_lsns.not_modified_since, + .hdr.tag = T_NeonDbSizeRequest, + .hdr.reqid = GENERATE_REQUEST_ID(), + .hdr.lsn = request_lsns.request_lsn, + .hdr.not_modified_since = request_lsns.not_modified_since, .dbNode = dbNode, }; resp = page_server_request(&request); + + switch (resp->tag) + { + case T_NeonDbSizeResponse: + { + NeonDbSizeResponse* dbsize_resp = (NeonDbSizeResponse *) resp; + if (neon_protocol_version >= 3) + { + if (!equal_requests(resp, &request.hdr) || + dbsize_resp->req.dbNode != dbNode) + { + NEON_PANIC_CONNECTION_STATE(-1, PANIC, + "Unexpect response {reqid=%lx,lsn=%X/%08X, since=%X/%08X, dbNode=%u} to get DB size request {reqid=%lx,lsn=%X/%08X, since=%X/%08X, dbNode=%u}", + resp->reqid, LSN_FORMAT_ARGS(resp->lsn), LSN_FORMAT_ARGS(resp->not_modified_since), dbsize_resp->req.dbNode, + request.hdr.reqid, LSN_FORMAT_ARGS(request.hdr.lsn), LSN_FORMAT_ARGS(request.hdr.not_modified_since), dbNode); + } + } + db_size = dbsize_resp->db_size; + break; + } + case T_NeonErrorResponse: + if (neon_protocol_version >= 3) + { + if (!equal_requests(resp, &request.hdr)) + { + elog(WARNING, NEON_TAG "Error message {reqid=%lx,lsn=%X/%08X, since=%X/%08X} doesn't match get DB size request {reqid=%lx,lsn=%X/%08X, since=%X/%08X}", + resp->reqid, LSN_FORMAT_ARGS(resp->lsn), LSN_FORMAT_ARGS(resp->not_modified_since), + request.hdr.reqid, LSN_FORMAT_ARGS(request.hdr.lsn), LSN_FORMAT_ARGS(request.hdr.not_modified_since)); + } + } + ereport(ERROR, + (errcode(ERRCODE_IO_ERROR), + errmsg(NEON_TAG "[reqid %lx] could not read db size of db %u from page server at lsn %X/%08X", + resp->reqid, + dbNode, LSN_FORMAT_ARGS(request_lsns.effective_request_lsn)), + errdetail("page server returned error: %s", + ((NeonErrorResponse *) resp)->message))); + break; + + default: + NEON_PANIC_CONNECTION_STATE(-1, PANIC, + "Expected DbSize (0x%02x) or Error (0x%02x) response to DbSizeRequest, but got 0x%02x", + T_NeonDbSizeResponse, T_NeonErrorResponse, resp->tag); + } + + neon_log(SmgrTrace, "neon_dbsize: db %u (request LSN %X/%08X): %ld bytes", + dbNode, LSN_FORMAT_ARGS(request_lsns.effective_request_lsn), db_size); + + pfree(resp); } - - switch (resp->tag) - { - case T_NeonDbSizeResponse: - db_size = ((NeonDbSizeResponse *) resp)->db_size; - break; - - case T_NeonErrorResponse: - ereport(ERROR, - (errcode(ERRCODE_IO_ERROR), - errmsg(NEON_TAG "could not read db size of db %u from page server at lsn %X/%08X", - dbNode, LSN_FORMAT_ARGS(request_lsns.effective_request_lsn)), - errdetail("page server returned error: %s", - ((NeonErrorResponse *) resp)->message))); - break; - - default: - NEON_PANIC_CONNECTION_STATE(-1, PANIC, - "Expected DbSize (0x%02x) or Error (0x%02x) response to DbSizeRequest, but got 0x%02x", - T_NeonDbSizeResponse, T_NeonErrorResponse, resp->tag); - } - - neon_log(SmgrTrace, "neon_dbsize: db %u (request LSN %X/%08X): %ld bytes", - dbNode, LSN_FORMAT_ARGS(request_lsns.effective_request_lsn), db_size); - - pfree(resp); return db_size; } @@ -3868,16 +4025,17 @@ neon_read_slru_segment(SMgrRelation reln, const char* path, int segno, void* buf return -1; request = (NeonGetSlruSegmentRequest) { - .req.tag = T_NeonGetSlruSegmentRequest, - .req.lsn = request_lsn, - .req.not_modified_since = not_modified_since, + .hdr.tag = T_NeonGetSlruSegmentRequest, + .hdr.reqid = GENERATE_REQUEST_ID(), + .hdr.lsn = request_lsn, + .hdr.not_modified_since = not_modified_since, .kind = kind, .segno = segno }; do { - while (!page_server->send(shard_no, &request.req) || !page_server->flush(shard_no)); + while (!page_server->send(shard_no, &request.hdr) || !page_server->flush(shard_no)); consume_prefetch_responses(); @@ -3887,14 +4045,38 @@ neon_read_slru_segment(SMgrRelation reln, const char* path, int segno, void* buf switch (resp->tag) { case T_NeonGetSlruSegmentResponse: - n_blocks = ((NeonGetSlruSegmentResponse *) resp)->n_blocks; - memcpy(buffer, ((NeonGetSlruSegmentResponse *) resp)->data, n_blocks*BLCKSZ); + { + NeonGetSlruSegmentResponse* slru_resp = (NeonGetSlruSegmentResponse *) resp; + if (neon_protocol_version >= 3) + { + if (!equal_requests(resp, &request.hdr) || + slru_resp->req.kind != kind || + slru_resp->req.segno != segno) + { + NEON_PANIC_CONNECTION_STATE(-1, PANIC, + "Unexpect response {reqid=%lx,lsn=%X/%08X, since=%X/%08X, kind=%u, segno=%u} to get SLRU segment request {reqid=%lx,lsn=%X/%08X, since=%X/%08X, kind=%u, segno=%u}", + resp->reqid, LSN_FORMAT_ARGS(resp->lsn), LSN_FORMAT_ARGS(resp->not_modified_since), slru_resp->req.kind, slru_resp->req.segno, + request.hdr.reqid, LSN_FORMAT_ARGS(request.hdr.lsn), LSN_FORMAT_ARGS(request.hdr.not_modified_since), kind, segno); + } + } + n_blocks = slru_resp->n_blocks; + memcpy(buffer, slru_resp->data, n_blocks*BLCKSZ); break; - + } case T_NeonErrorResponse: + if (neon_protocol_version >= 3) + { + if (!equal_requests(resp, &request.hdr)) + { + elog(WARNING, NEON_TAG "Error message {reqid=%lx,lsn=%X/%08X, since=%X/%08X} doesn't match get SLRU segment request {reqid=%lx,lsn=%X/%08X, since=%X/%08X}", + resp->reqid, LSN_FORMAT_ARGS(resp->lsn), LSN_FORMAT_ARGS(resp->not_modified_since), + request.hdr.reqid, LSN_FORMAT_ARGS(request.hdr.lsn), LSN_FORMAT_ARGS(request.hdr.not_modified_since)); + } + } ereport(ERROR, (errcode(ERRCODE_IO_ERROR), - errmsg(NEON_TAG "could not read SLRU %d segment %d at lsn %X/%08X", + errmsg(NEON_TAG "[reqid %lx] could not read SLRU %d segment %d at lsn %X/%08X", + resp->reqid, kind, segno, LSN_FORMAT_ARGS(request_lsn)), @@ -4033,8 +4215,9 @@ neon_extend_rel_size(NRelFileInfo rinfo, ForkNumber forknum, BlockNumber blkno, NeonResponse *response; NeonNblocksResponse *nbresponse; NeonNblocksRequest request = { - .req = (NeonRequest) { + .hdr = (NeonRequest) { .tag = T_NeonNblocksRequest, + .reqid = GENERATE_REQUEST_ID(), .lsn = end_recptr, .not_modified_since = end_recptr, }, From 640ac4fc9efcdadad442f7dfafe15e7f9d816906 Mon Sep 17 00:00:00 2001 From: "Alex Chi Z." <4198311+skyzh@users.noreply.github.com> Date: Thu, 9 Jan 2025 09:43:20 -0500 Subject: [PATCH 015/111] fix(pageserver): report timestamp is in the past if the key is missing (#10210) ## Problem If for some reasons we already garbage-collected the data under an LSN but the caller uses a past LSN for the find_time_cutoff function, now we will report a missing key error and GC will never proceed. Note that missing key error can also happen if the key is really missing (i.e., during the past offload incidents) ## Summary of changes Make sure GC proceeds by bumping the LSN. When time_cutoff=None, we will not increase the time_cutoff (it will be set to latest_gc_cutoff). If we really need to bump the GC LSN for maintenance purpose, we need a separate API to do that. Signed-off-by: Alex Chi Z --- pageserver/src/pgdatadir_mapping.rs | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/pageserver/src/pgdatadir_mapping.rs b/pageserver/src/pgdatadir_mapping.rs index 14c7e0d2f8..b65fe6cf7c 100644 --- a/pageserver/src/pgdatadir_mapping.rs +++ b/pageserver/src/pgdatadir_mapping.rs @@ -627,7 +627,7 @@ impl Timeline { // cannot overflow, high and low are both smaller than u64::MAX / 2 let mid = (high + low) / 2; - let cmp = self + let cmp = match self .is_latest_commit_timestamp_ge_than( search_timestamp, Lsn(mid * 8), @@ -635,7 +635,16 @@ impl Timeline { &mut found_larger, ctx, ) - .await?; + .await + { + Ok(res) => res, + Err(PageReconstructError::MissingKey(e)) => { + warn!("Missing key while find_lsn_for_timestamp. Either we might have already garbage-collected that data or the key is really missing. Last error: {:#}", e); + // Return that we didn't find any requests smaller than the LSN, and logging the error. + return Ok(LsnForTimestamp::Past(min_lsn)); + } + Err(e) => return Err(e), + }; if cmp { high = mid; @@ -643,6 +652,7 @@ impl Timeline { low = mid + 1; } } + // If `found_smaller == true`, `low = t + 1` where `t` is the target LSN, // so the LSN of the last commit record before or at `search_timestamp`. // Remove one from `low` to get `t`. From ac6cca17acae975ab261c9e82c8fb1bc8a06bcce Mon Sep 17 00:00:00 2001 From: John Spray Date: Thu, 9 Jan 2025 15:33:44 +0000 Subject: [PATCH 016/111] storcon: don't log a heartbeat error during shutdown (#10325) ## Problem Occasionally we see an unexpected error like: ``` ERROR spawn_heartbeat_driver: Failed to update node state 1 after heartbeat round: Shutting down\n') Hint: use scripts/check_allowed_errors.sh to test any new allowed_error you add ``` https://neon-github-public-dev.s3.amazonaws.com/reports/pr-10324/12690404952/index.html#/testresult/63406a0687bf6eca ## Summary of changes - Explicitly handle ApiError::ShuttingDown as a no-op when mutating node status --- storage_controller/src/service.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/storage_controller/src/service.rs b/storage_controller/src/service.rs index fd4ee7fd10..cf2d1ef97b 100644 --- a/storage_controller/src/service.rs +++ b/storage_controller/src/service.rs @@ -1047,6 +1047,9 @@ impl Service { // on a snapshot of the nodes. tracing::info!("Node {} was not found after heartbeat round", node_id); } + Err(ApiError::ShuttingDown) => { + // No-op: we're shutting down, no need to try and update any nodes' statuses + } Err(err) => { // Transition to active involves reconciling: if a node responds to a heartbeat then // becomes unavailable again, we may get an error here. From ad51622568a24b6be53ff165f02d32e004476a73 Mon Sep 17 00:00:00 2001 From: John Spray Date: Thu, 9 Jan 2025 15:34:06 +0000 Subject: [PATCH 017/111] remote_storage: enable Azure connection pooling by default (#10324) ## Problem Initially we defaulted this to zero to reduce risk. We have now been using pooling in staging for some time without issues, so let's make it the default for anyone using this software without setting the config explicitly. Closes: https://github.com/neondatabase/cloud/issues/20971 ## Summary of changes - Set Azure blob storage connection pool size to 8 by default --- libs/remote_storage/src/config.rs | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/libs/remote_storage/src/config.rs b/libs/remote_storage/src/config.rs index dd49d4d5e7..49b1d9dc87 100644 --- a/libs/remote_storage/src/config.rs +++ b/libs/remote_storage/src/config.rs @@ -115,13 +115,15 @@ fn default_max_keys_per_list_response() -> Option { } fn default_azure_conn_pool_size() -> usize { - // Conservative default: no connection pooling. At time of writing this is the Azure - // SDK's default as well, due to historic reports of hard-to-reproduce issues + // By default, the Azure SDK does no connection pooling, due to historic reports of hard-to-reproduce issues // (https://github.com/hyperium/hyper/issues/2312) // // However, using connection pooling is important to avoid exhausting client ports when // doing huge numbers of requests (https://github.com/neondatabase/cloud/issues/20971) - 0 + // + // We therefore enable a modest pool size by default: this may be configured to zero if + // issues like the alleged upstream hyper issue appear. + 8 } impl Debug for S3Config { From bebc46e71336104a3282e9f2497c2889583caffa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arpad=20M=C3=BCller?= Date: Thu, 9 Jan 2025 16:55:02 +0100 Subject: [PATCH 018/111] Add scheduling_policy column to safekeepers table (#10205) Add a `scheduling_policy` column to the safekeepers table of the storage controller. Part of #9981 --- control_plane/storcon_cli/src/main.rs | 13 +++- libs/pageserver_api/src/controller_api.rs | 33 +++++++++ .../down.sql | 1 + .../up.sql | 1 + storage_controller/src/http.rs | 4 +- storage_controller/src/persistence.rs | 69 ++++++++++++++----- storage_controller/src/schema.rs | 1 + storage_controller/src/service.rs | 9 ++- .../regress/test_storage_controller.py | 2 +- 9 files changed, 106 insertions(+), 27 deletions(-) create mode 100644 storage_controller/migrations/2024-12-12-212515_safekeepers_scheduling_policy/down.sql create mode 100644 storage_controller/migrations/2024-12-12-212515_safekeepers_scheduling_policy/up.sql diff --git a/control_plane/storcon_cli/src/main.rs b/control_plane/storcon_cli/src/main.rs index 6ee1044c18..617b2cd1ba 100644 --- a/control_plane/storcon_cli/src/main.rs +++ b/control_plane/storcon_cli/src/main.rs @@ -1035,7 +1035,15 @@ async fn main() -> anyhow::Result<()> { resp.sort_by(|a, b| a.id.cmp(&b.id)); let mut table = comfy_table::Table::new(); - table.set_header(["Id", "Version", "Host", "Port", "Http Port", "AZ Id"]); + table.set_header([ + "Id", + "Version", + "Host", + "Port", + "Http Port", + "AZ Id", + "Scheduling", + ]); for sk in resp { table.add_row([ format!("{}", sk.id), @@ -1043,7 +1051,8 @@ async fn main() -> anyhow::Result<()> { sk.host, format!("{}", sk.port), format!("{}", sk.http_port), - sk.availability_zone_id.to_string(), + sk.availability_zone_id.clone(), + String::from(sk.scheduling_policy), ]); } println!("{table}"); diff --git a/libs/pageserver_api/src/controller_api.rs b/libs/pageserver_api/src/controller_api.rs index faf11e487c..7eb3547183 100644 --- a/libs/pageserver_api/src/controller_api.rs +++ b/libs/pageserver_api/src/controller_api.rs @@ -320,6 +320,38 @@ impl From for String { } } +#[derive(Serialize, Deserialize, Clone, Copy, Eq, PartialEq, Debug)] +pub enum SkSchedulingPolicy { + Active, + Disabled, + Decomissioned, +} + +impl FromStr for SkSchedulingPolicy { + type Err = anyhow::Error; + + fn from_str(s: &str) -> Result { + Ok(match s { + "active" => Self::Active, + "disabled" => Self::Disabled, + "decomissioned" => Self::Decomissioned, + _ => return Err(anyhow::anyhow!("Unknown scheduling state '{s}'")), + }) + } +} + +impl From for String { + fn from(value: SkSchedulingPolicy) -> String { + use SkSchedulingPolicy::*; + match value { + Active => "active", + Disabled => "disabled", + Decomissioned => "decomissioned", + } + .to_string() + } +} + /// Controls how tenant shards are mapped to locations on pageservers, e.g. whether /// to create secondary locations. #[derive(Clone, Serialize, Deserialize, Debug, PartialEq, Eq)] @@ -387,6 +419,7 @@ pub struct SafekeeperDescribeResponse { pub port: i32, pub http_port: i32, pub availability_zone_id: String, + pub scheduling_policy: SkSchedulingPolicy, } #[cfg(test)] diff --git a/storage_controller/migrations/2024-12-12-212515_safekeepers_scheduling_policy/down.sql b/storage_controller/migrations/2024-12-12-212515_safekeepers_scheduling_policy/down.sql new file mode 100644 index 0000000000..e26bff798f --- /dev/null +++ b/storage_controller/migrations/2024-12-12-212515_safekeepers_scheduling_policy/down.sql @@ -0,0 +1 @@ +ALTER TABLE safekeepers DROP scheduling_policy; diff --git a/storage_controller/migrations/2024-12-12-212515_safekeepers_scheduling_policy/up.sql b/storage_controller/migrations/2024-12-12-212515_safekeepers_scheduling_policy/up.sql new file mode 100644 index 0000000000..d83cc6cc46 --- /dev/null +++ b/storage_controller/migrations/2024-12-12-212515_safekeepers_scheduling_policy/up.sql @@ -0,0 +1 @@ +ALTER TABLE safekeepers ADD scheduling_policy VARCHAR NOT NULL DEFAULT 'disabled'; diff --git a/storage_controller/src/http.rs b/storage_controller/src/http.rs index 24fd4c341a..5385e4ee0b 100644 --- a/storage_controller/src/http.rs +++ b/storage_controller/src/http.rs @@ -3,7 +3,7 @@ use crate::metrics::{ HttpRequestLatencyLabelGroup, HttpRequestStatusLabelGroup, PageserverRequestLabelGroup, METRICS_REGISTRY, }; -use crate::persistence::SafekeeperPersistence; +use crate::persistence::SafekeeperUpsert; use crate::reconciler::ReconcileError; use crate::service::{LeadershipStatus, Service, RECONCILE_TIMEOUT, STARTUP_RECONCILE_TIMEOUT}; use anyhow::Context; @@ -1249,7 +1249,7 @@ async fn handle_get_safekeeper(req: Request) -> Result, Api async fn handle_upsert_safekeeper(mut req: Request) -> Result, ApiError> { check_permissions(&req, Scope::Infra)?; - let body = json_request::(&mut req).await?; + let body = json_request::(&mut req).await?; let id = parse_request_param::(&req, "id")?; if id != body.id { diff --git a/storage_controller/src/persistence.rs b/storage_controller/src/persistence.rs index c5eb106f24..cebf3e9594 100644 --- a/storage_controller/src/persistence.rs +++ b/storage_controller/src/persistence.rs @@ -13,6 +13,7 @@ use pageserver_api::controller_api::AvailabilityZone; use pageserver_api::controller_api::MetadataHealthRecord; use pageserver_api::controller_api::SafekeeperDescribeResponse; use pageserver_api::controller_api::ShardSchedulingPolicy; +use pageserver_api::controller_api::SkSchedulingPolicy; use pageserver_api::controller_api::{NodeSchedulingPolicy, PlacementPolicy}; use pageserver_api::models::TenantConfig; use pageserver_api::shard::ShardConfigError; @@ -1075,12 +1076,14 @@ impl Persistence { pub(crate) async fn safekeeper_upsert( &self, - record: SafekeeperPersistence, + record: SafekeeperUpsert, ) -> Result<(), DatabaseError> { use crate::schema::safekeepers::dsl::*; self.with_conn(move |conn| -> DatabaseResult<()> { - let bind = record.as_insert_or_update(); + let bind = record + .as_insert_or_update() + .map_err(|e| DatabaseError::Logical(format!("{e}")))?; let inserted_updated = diesel::insert_into(safekeepers) .values(&bind) @@ -1243,6 +1246,7 @@ pub(crate) struct ControllerPersistence { pub(crate) started_at: chrono::DateTime, } +// What we store in the database #[derive(Serialize, Deserialize, Queryable, Selectable, Eq, PartialEq, Debug, Clone)] #[diesel(table_name = crate::schema::safekeepers)] pub(crate) struct SafekeeperPersistence { @@ -1257,11 +1261,51 @@ pub(crate) struct SafekeeperPersistence { pub(crate) active: bool, pub(crate) http_port: i32, pub(crate) availability_zone_id: String, + pub(crate) scheduling_policy: String, } impl SafekeeperPersistence { - fn as_insert_or_update(&self) -> InsertUpdateSafekeeper<'_> { - InsertUpdateSafekeeper { + pub(crate) fn as_describe_response(&self) -> Result { + let scheduling_policy = + SkSchedulingPolicy::from_str(&self.scheduling_policy).map_err(|e| { + DatabaseError::Logical(format!("can't construct SkSchedulingPolicy: {e:?}")) + })?; + // omit the `active` flag on purpose: it is deprecated. + Ok(SafekeeperDescribeResponse { + id: NodeId(self.id as u64), + region_id: self.region_id.clone(), + version: self.version, + host: self.host.clone(), + port: self.port, + http_port: self.http_port, + availability_zone_id: self.availability_zone_id.clone(), + scheduling_policy, + }) + } +} + +/// What we expect from the upsert http api +#[derive(Serialize, Deserialize, Eq, PartialEq, Debug, Clone)] +pub(crate) struct SafekeeperUpsert { + pub(crate) id: i64, + pub(crate) region_id: String, + /// 1 is special, it means just created (not currently posted to storcon). + /// Zero or negative is not really expected. + /// Otherwise the number from `release-$(number_of_commits_on_branch)` tag. + pub(crate) version: i64, + pub(crate) host: String, + pub(crate) port: i32, + pub(crate) active: bool, + pub(crate) http_port: i32, + pub(crate) availability_zone_id: String, +} + +impl SafekeeperUpsert { + fn as_insert_or_update(&self) -> anyhow::Result> { + if self.version < 0 { + anyhow::bail!("negative version: {}", self.version); + } + Ok(InsertUpdateSafekeeper { id: self.id, region_id: &self.region_id, version: self.version, @@ -1270,19 +1314,9 @@ impl SafekeeperPersistence { active: self.active, http_port: self.http_port, availability_zone_id: &self.availability_zone_id, - } - } - pub(crate) fn as_describe_response(&self) -> SafekeeperDescribeResponse { - // omit the `active` flag on purpose: it is deprecated. - SafekeeperDescribeResponse { - id: NodeId(self.id as u64), - region_id: self.region_id.clone(), - version: self.version, - host: self.host.clone(), - port: self.port, - http_port: self.http_port, - availability_zone_id: self.availability_zone_id.clone(), - } + // None means a wish to not update this column. We expose abilities to update it via other means. + scheduling_policy: None, + }) } } @@ -1297,4 +1331,5 @@ struct InsertUpdateSafekeeper<'a> { active: bool, http_port: i32, availability_zone_id: &'a str, + scheduling_policy: Option<&'a str>, } diff --git a/storage_controller/src/schema.rs b/storage_controller/src/schema.rs index 9e005ab932..44c91619ab 100644 --- a/storage_controller/src/schema.rs +++ b/storage_controller/src/schema.rs @@ -39,6 +39,7 @@ diesel::table! { active -> Bool, http_port -> Int4, availability_zone_id -> Text, + scheduling_policy -> Varchar, } } diff --git a/storage_controller/src/service.rs b/storage_controller/src/service.rs index cf2d1ef97b..265b2798d2 100644 --- a/storage_controller/src/service.rs +++ b/storage_controller/src/service.rs @@ -7350,13 +7350,12 @@ impl Service { pub(crate) async fn safekeepers_list( &self, ) -> Result, DatabaseError> { - Ok(self - .persistence + self.persistence .list_safekeepers() .await? .into_iter() .map(|v| v.as_describe_response()) - .collect::>()) + .collect::, _>>() } pub(crate) async fn get_safekeeper( @@ -7366,12 +7365,12 @@ impl Service { self.persistence .safekeeper_get(id) .await - .map(|v| v.as_describe_response()) + .and_then(|v| v.as_describe_response()) } pub(crate) async fn upsert_safekeeper( &self, - record: crate::persistence::SafekeeperPersistence, + record: crate::persistence::SafekeeperUpsert, ) -> Result<(), DatabaseError> { self.persistence.safekeeper_upsert(record).await } diff --git a/test_runner/regress/test_storage_controller.py b/test_runner/regress/test_storage_controller.py index 207f55a214..da6d5b8622 100644 --- a/test_runner/regress/test_storage_controller.py +++ b/test_runner/regress/test_storage_controller.py @@ -3019,7 +3019,7 @@ def test_safekeeper_deployment_time_update(neon_env_builder: NeonEnvBuilder): def eq_safekeeper_records(a: dict[str, Any], b: dict[str, Any]) -> bool: compared = [dict(a), dict(b)] - masked_keys = ["created_at", "updated_at", "active"] + masked_keys = ["created_at", "updated_at", "active", "scheduling_policy"] for d in compared: # keep deleting these in case we are comparing the body as it will be uploaded by real scripts From f37eeb56addb3c7946ac6299cd7e415f4dc47a9d Mon Sep 17 00:00:00 2001 From: Alexey Kondratov Date: Thu, 9 Jan 2025 17:39:53 +0100 Subject: [PATCH 019/111] fix(compute_ctl): Resolve issues with dropping roles having dangling permissions (#10299) ## Problem In Postgres, one cannot drop a role if it has any dependent objects in the DB. In `compute_ctl`, we automatically reassign all dependent objects in every DB to the corresponding DB owner. Yet, it seems that it doesn't help with some implicit permissions. The issue is reproduced by installing a `postgis` extension because it creates some views and tables in the public schema. ## Summary of changes Added a repro test without using a `postgis`: i) create a role via `compute_ctl` (with `neon_superuser` grant); ii) create a test role, a table in schema public, and grant permissions via the role in `neon_superuser`. To fix the issue, I added a new `compute_ctl` code that removes such dangling permissions before dropping the role. It's done in the least invasive way, i.e., only touches the schema public, because i) that's the problem we had with PostGIS; ii) it creates a smaller chance of messing anything up and getting a stuck operation again, just for a different reason. Properly, any API-based catalog operations should fail gracefully and provide an actionable error and status code to the control plane, allowing the latter to unwind the operation and propagate an error message and hint to the user. In this sense, it's aligned with another feature request https://github.com/neondatabase/cloud/issues/21611 Resolve neondatabase/cloud#13582 --- compute_tools/src/spec_apply.rs | 14 ++- .../sql/pre_drop_role_revoke_privileges.sql | 28 +++++ test_runner/regress/test_compute_catalog.py | 115 ++++++++++++++++++ 3 files changed, 156 insertions(+), 1 deletion(-) create mode 100644 compute_tools/src/sql/pre_drop_role_revoke_privileges.sql diff --git a/compute_tools/src/spec_apply.rs b/compute_tools/src/spec_apply.rs index 695a722d6d..7401de2e60 100644 --- a/compute_tools/src/spec_apply.rs +++ b/compute_tools/src/spec_apply.rs @@ -75,7 +75,7 @@ pub struct MutableApplyContext { pub dbs: HashMap, } -/// Appply the operations that belong to the given spec apply phase. +/// Apply the operations that belong to the given spec apply phase. /// /// Commands within a single phase are executed in order of Iterator yield. /// Commands of ApplySpecPhase::RunInEachDatabase will execute in the database @@ -498,7 +498,19 @@ async fn get_operations<'a>( ), comment: None, }, + // Revoke some potentially blocking privileges (Neon-specific currently) + Operation { + query: format!( + include_str!("sql/pre_drop_role_revoke_privileges.sql"), + role_name = quoted, + ), + comment: None, + }, // This now will only drop privileges of the role + // TODO: this is obviously not 100% true because of the above case, + // there could be still some privileges that are not revoked. Maybe this + // only drops privileges that were granted *by this* role, not *to this* role, + // but this has to be checked. Operation { query: format!("DROP OWNED BY {}", quoted), comment: None, diff --git a/compute_tools/src/sql/pre_drop_role_revoke_privileges.sql b/compute_tools/src/sql/pre_drop_role_revoke_privileges.sql new file mode 100644 index 0000000000..cdaa7071d3 --- /dev/null +++ b/compute_tools/src/sql/pre_drop_role_revoke_privileges.sql @@ -0,0 +1,28 @@ +SET SESSION ROLE neon_superuser; + +DO $$ +DECLARE + schema TEXT; + revoke_query TEXT; +BEGIN + FOR schema IN + SELECT schema_name + FROM information_schema.schemata + -- So far, we only had issues with 'public' schema. Probably, because we do some additional grants, + -- e.g., make DB owner the owner of 'public' schema automatically (when created via API). + -- See https://github.com/neondatabase/cloud/issues/13582 for the context. + -- Still, keep the loop because i) it efficiently handles the case when there is no 'public' schema, + -- ii) it's easy to add more schemas to the list if needed. + WHERE schema_name IN ('public') + LOOP + revoke_query := format( + 'REVOKE ALL PRIVILEGES ON ALL TABLES IN SCHEMA %I FROM {role_name} GRANTED BY neon_superuser;', + schema + ); + + EXECUTE revoke_query; + END LOOP; +END; +$$; + +RESET ROLE; diff --git a/test_runner/regress/test_compute_catalog.py b/test_runner/regress/test_compute_catalog.py index e411aad97d..f0878b2631 100644 --- a/test_runner/regress/test_compute_catalog.py +++ b/test_runner/regress/test_compute_catalog.py @@ -250,3 +250,118 @@ def test_dropdb_with_subscription(neon_simple_env: NeonEnv): assert curr_db is not None assert len(curr_db) == 1 assert curr_db[0] == "publisher_db" + + +def test_compute_drop_role(neon_simple_env: NeonEnv): + """ + Test that compute_ctl can drop a role even if it has some depending objects + like permissions in one of the databases. + Reproduction test for https://github.com/neondatabase/cloud/issues/13582 + """ + env = neon_simple_env + TEST_DB_NAME = "db_with_permissions" + + endpoint = env.endpoints.create_start("main") + + endpoint.respec_deep( + **{ + "skip_pg_catalog_updates": False, + "cluster": { + "roles": [ + { + # We need to create role via compute_ctl, because in this case it will receive + # additional grants equivalent to our real environment, so we can repro some + # issues. + "name": "neon", + # Some autocomplete-suggested hash, no specific meaning. + "encrypted_password": "SCRAM-SHA-256$4096:hBT22QjqpydQWqEulorfXA==$miBogcoj68JWYdsNB5PW1X6PjSLBEcNuctuhtGkb4PY=:hxk2gxkwxGo6P7GCtfpMlhA9zwHvPMsCz+NQf2HfvWk=", + "options": [], + }, + ], + "databases": [ + { + "name": TEST_DB_NAME, + "owner": "neon", + }, + ], + }, + } + ) + endpoint.reconfigure() + + with endpoint.cursor(dbname=TEST_DB_NAME) as cursor: + # Create table and view as `cloud_admin`. This is the case when, for example, + # PostGIS extensions creates tables in `public` schema. + cursor.execute("create table test_table (id int)") + cursor.execute("create view test_view as select * from test_table") + + with endpoint.cursor(dbname=TEST_DB_NAME, user="neon") as cursor: + cursor.execute("create role readonly") + # We (`compute_ctl`) make 'neon' the owner of schema 'public' in the owned database. + # Postgres has all sorts of permissions and grants that we may not handle well, + # but this is the shortest repro grant for the issue + # https://github.com/neondatabase/cloud/issues/13582 + cursor.execute("grant select on all tables in schema public to readonly") + + # Check that role was created + with endpoint.cursor() as cursor: + cursor.execute("SELECT rolname FROM pg_roles WHERE rolname = 'readonly'") + role = cursor.fetchone() + assert role is not None + + # Confirm that we actually have some permissions for 'readonly' role + # that may block our ability to drop the role. + with endpoint.cursor(dbname=TEST_DB_NAME) as cursor: + cursor.execute( + "select grantor from information_schema.role_table_grants where grantee = 'readonly'" + ) + res = cursor.fetchall() + assert len(res) == 2, f"Expected 2 table grants, got {len(res)}" + for row in res: + assert row[0] == "neon_superuser" + + # Drop role via compute_ctl + endpoint.respec_deep( + **{ + "skip_pg_catalog_updates": False, + "delta_operations": [ + { + "action": "delete_role", + "name": "readonly", + }, + ], + } + ) + endpoint.reconfigure() + + # Check that role is dropped + with endpoint.cursor() as cursor: + cursor.execute("SELECT rolname FROM pg_roles WHERE rolname = 'readonly'") + role = cursor.fetchone() + assert role is None + + # + # Drop schema 'public' and check that we can still drop the role + # + with endpoint.cursor(dbname=TEST_DB_NAME) as cursor: + cursor.execute("create role readonly2") + cursor.execute("grant select on all tables in schema public to readonly2") + cursor.execute("drop schema public cascade") + + endpoint.respec_deep( + **{ + "skip_pg_catalog_updates": False, + "delta_operations": [ + { + "action": "delete_role", + "name": "readonly2", + }, + ], + } + ) + endpoint.reconfigure() + + with endpoint.cursor() as cursor: + cursor.execute("SELECT rolname FROM pg_roles WHERE rolname = 'readonly2'") + role = cursor.fetchone() + assert role is None From 99b5a6705f1dfa41a6cc7ae60b74b2f769d0eb85 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arpad=20M=C3=BCller?= Date: Thu, 9 Jan 2025 19:29:09 +0100 Subject: [PATCH 020/111] Update rust to 1.84.0 (#10328) We keep the practice of keeping the compiler up to date, pointing to the latest release. This is done by many other projects in the Rust ecosystem as well. [Release notes](https://releases.rs/docs/1.84.0/). Prior update was in #9926. --- build-tools.Dockerfile | 2 +- rust-toolchain.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/build-tools.Dockerfile b/build-tools.Dockerfile index fa84e467ad..79210a2e1b 100644 --- a/build-tools.Dockerfile +++ b/build-tools.Dockerfile @@ -258,7 +258,7 @@ WORKDIR /home/nonroot # Rust # Please keep the version of llvm (installed above) in sync with rust llvm (`rustc --version --verbose | grep LLVM`) -ENV RUSTC_VERSION=1.83.0 +ENV RUSTC_VERSION=1.84.0 ENV RUSTUP_HOME="/home/nonroot/.rustup" ENV PATH="/home/nonroot/.cargo/bin:${PATH}" ARG RUSTFILT_VERSION=0.2.1 diff --git a/rust-toolchain.toml b/rust-toolchain.toml index f0661a32e0..06746d3e1d 100644 --- a/rust-toolchain.toml +++ b/rust-toolchain.toml @@ -1,5 +1,5 @@ [toolchain] -channel = "1.83.0" +channel = "1.84.0" profile = "default" # The default profile includes rustc, rust-std, cargo, rust-docs, rustfmt and clippy. # https://rust-lang.github.io/rustup/concepts/profiles.html From 49756a0d01530c6df00b6095da15ad05027c149d Mon Sep 17 00:00:00 2001 From: Tristan Partin Date: Thu, 9 Jan 2025 14:08:26 -0600 Subject: [PATCH 021/111] Implement compute_ctl management API in Axum (#10099) This is a refactor to create better abstractions related to our management server. It cleans up the code, and prepares everything for authorized communication to and from the control plane. Signed-off-by: Tristan Partin --- Cargo.lock | 105 ++- Cargo.toml | 6 +- compute_tools/Cargo.toml | 5 +- compute_tools/src/bin/compute_ctl.rs | 9 +- compute_tools/src/catalog.rs | 6 +- compute_tools/src/http/api.rs | 606 ------------------ compute_tools/src/http/extract/json.rs | 48 ++ compute_tools/src/http/extract/mod.rs | 7 + compute_tools/src/http/extract/path.rs | 48 ++ compute_tools/src/http/extract/query.rs | 48 ++ compute_tools/src/http/mod.rs | 57 +- compute_tools/src/http/openapi_spec.yaml | 2 +- .../src/http/routes/check_writability.rs | 20 + compute_tools/src/http/routes/configure.rs | 91 +++ .../src/http/routes/database_schema.rs | 34 + .../src/http/routes/dbs_and_roles.rs | 16 + .../src/http/routes/extension_server.rs | 67 ++ compute_tools/src/http/routes/extensions.rs | 45 ++ compute_tools/src/http/routes/failpoints.rs | 35 + compute_tools/src/http/routes/grants.rs | 48 ++ compute_tools/src/http/routes/info.rs | 11 + compute_tools/src/http/routes/insights.rs | 18 + .../src/http/routes/installed_extensions.rs | 33 + compute_tools/src/http/routes/metrics.rs | 32 + compute_tools/src/http/routes/metrics_json.rs | 12 + compute_tools/src/http/routes/mod.rs | 38 ++ compute_tools/src/http/routes/status.rs | 14 + compute_tools/src/http/routes/terminate.rs | 58 ++ compute_tools/src/http/server.rs | 165 +++++ compute_tools/src/lib.rs | 2 - control_plane/src/endpoint.rs | 4 +- libs/compute_api/src/responses.rs | 23 +- workspace_hack/Cargo.toml | 3 +- 33 files changed, 1065 insertions(+), 651 deletions(-) delete mode 100644 compute_tools/src/http/api.rs create mode 100644 compute_tools/src/http/extract/json.rs create mode 100644 compute_tools/src/http/extract/mod.rs create mode 100644 compute_tools/src/http/extract/path.rs create mode 100644 compute_tools/src/http/extract/query.rs create mode 100644 compute_tools/src/http/routes/check_writability.rs create mode 100644 compute_tools/src/http/routes/configure.rs create mode 100644 compute_tools/src/http/routes/database_schema.rs create mode 100644 compute_tools/src/http/routes/dbs_and_roles.rs create mode 100644 compute_tools/src/http/routes/extension_server.rs create mode 100644 compute_tools/src/http/routes/extensions.rs create mode 100644 compute_tools/src/http/routes/failpoints.rs create mode 100644 compute_tools/src/http/routes/grants.rs create mode 100644 compute_tools/src/http/routes/info.rs create mode 100644 compute_tools/src/http/routes/insights.rs create mode 100644 compute_tools/src/http/routes/installed_extensions.rs create mode 100644 compute_tools/src/http/routes/metrics.rs create mode 100644 compute_tools/src/http/routes/metrics_json.rs create mode 100644 compute_tools/src/http/routes/mod.rs create mode 100644 compute_tools/src/http/routes/status.rs create mode 100644 compute_tools/src/http/routes/terminate.rs create mode 100644 compute_tools/src/http/server.rs diff --git a/Cargo.lock b/Cargo.lock index 9e0e343996..44143fa0da 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -718,13 +718,13 @@ dependencies = [ [[package]] name = "axum" -version = "0.7.5" +version = "0.7.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a6c9af12842a67734c9a2e355436e5d03b22383ed60cf13cd0c18fbfe3dcbcf" +checksum = "edca88bc138befd0323b20752846e6587272d3b03b0343c8ea28a6f819e6e71f" dependencies = [ "async-trait", "axum-core", - "base64 0.21.1", + "base64 0.22.1", "bytes", "futures-util", "http 1.1.0", @@ -746,8 +746,8 @@ dependencies = [ "sha1", "sync_wrapper 1.0.1", "tokio", - "tokio-tungstenite", - "tower", + "tokio-tungstenite 0.24.0", + "tower 0.5.2", "tower-layer", "tower-service", "tracing", @@ -1267,6 +1267,7 @@ dependencies = [ "aws-config", "aws-sdk-kms", "aws-sdk-s3", + "axum", "base64 0.13.1", "bytes", "camino", @@ -1277,7 +1278,7 @@ dependencies = [ "fail", "flate2", "futures", - "hyper 0.14.30", + "http 1.1.0", "metrics", "nix 0.27.1", "notify", @@ -1303,6 +1304,8 @@ dependencies = [ "tokio-postgres", "tokio-stream", "tokio-util", + "tower 0.5.2", + "tower-http", "tracing", "tracing-opentelemetry", "tracing-subscriber", @@ -2720,7 +2723,7 @@ dependencies = [ "pin-project-lite", "socket2", "tokio", - "tower", + "tower 0.4.13", "tower-service", "tracing", ] @@ -3260,9 +3263,9 @@ checksum = "b87248edafb776e59e6ee64a79086f65890d3510f2c656c000bf2a7e8a0aea40" [[package]] name = "matchit" -version = "0.8.2" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "540f1c43aed89909c0cc0cc604e3bb2f7e7a341a3728a9e6cfe760e733cd11ed" +checksum = "47e1ffaa40ddd1f3ed91f717a33c8c0ee23fff369e3aa8772b9605cc1d22f4c3" [[package]] name = "md-5" @@ -4758,7 +4761,7 @@ dependencies = [ "tokio-postgres", "tokio-postgres2", "tokio-rustls 0.26.0", - "tokio-tungstenite", + "tokio-tungstenite 0.21.0", "tokio-util", "tracing", "tracing-subscriber", @@ -5186,7 +5189,7 @@ dependencies = [ "async-trait", "getrandom 0.2.11", "http 1.1.0", - "matchit 0.8.2", + "matchit 0.8.4", "opentelemetry", "reqwest", "reqwest-middleware", @@ -6800,7 +6803,19 @@ dependencies = [ "futures-util", "log", "tokio", - "tungstenite", + "tungstenite 0.21.0", +] + +[[package]] +name = "tokio-tungstenite" +version = "0.24.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edc5f74e248dc973e0dbb7b74c7e0d6fcc301c694ff50049504004ef4d0cdcd9" +dependencies = [ + "futures-util", + "log", + "tokio", + "tungstenite 0.24.0", ] [[package]] @@ -6881,7 +6896,7 @@ dependencies = [ "tokio", "tokio-rustls 0.26.0", "tokio-stream", - "tower", + "tower 0.4.13", "tower-layer", "tower-service", "tracing", @@ -6922,16 +6937,49 @@ dependencies = [ ] [[package]] -name = "tower-layer" -version = "0.3.2" +name = "tower" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c20c8dbed6283a09604c3e69b4b7eeb54e298b8a600d4d5ecb5ad39de609f1d0" +checksum = "d039ad9159c98b70ecfd540b2573b97f7f52c3e8d9f8ad57a24b916a536975f9" +dependencies = [ + "futures-core", + "futures-util", + "pin-project-lite", + "sync_wrapper 1.0.1", + "tokio", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tower-http" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "403fa3b783d4b626a8ad51d766ab03cb6d2dbfc46b1c5d4448395e6628dc9697" +dependencies = [ + "bitflags 2.4.1", + "bytes", + "http 1.1.0", + "http-body 1.0.0", + "pin-project-lite", + "tower-layer", + "tower-service", + "tracing", + "uuid", +] + +[[package]] +name = "tower-layer" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" [[package]] name = "tower-service" -version = "0.3.2" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52" +checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" [[package]] name = "tracing" @@ -7086,6 +7134,24 @@ dependencies = [ "utf-8", ] +[[package]] +name = "tungstenite" +version = "0.24.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "18e5b8366ee7a95b16d32197d0b2604b43a0be89dc5fac9f8e96ccafbaedda8a" +dependencies = [ + "byteorder", + "bytes", + "data-encoding", + "http 1.1.0", + "httparse", + "log", + "rand 0.8.5", + "sha1", + "thiserror", + "utf-8", +] + [[package]] name = "twox-hash" version = "1.6.3" @@ -7867,7 +7933,8 @@ dependencies = [ "tokio-util", "toml_edit", "tonic", - "tower", + "tower 0.4.13", + "tower 0.5.2", "tracing", "tracing-core", "url", diff --git a/Cargo.toml b/Cargo.toml index 197808d5ae..39898e1c8d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -65,7 +65,7 @@ aws-smithy-types = "1.2" aws-credential-types = "1.2.0" aws-sigv4 = { version = "1.2", features = ["sign-http"] } aws-types = "1.3" -axum = { version = "0.7.5", features = ["ws"] } +axum = { version = "0.7.9", features = ["ws"] } base64 = "0.13.0" bincode = "1.3" bindgen = "0.70" @@ -187,7 +187,9 @@ tokio-util = { version = "0.7.10", features = ["io", "rt"] } toml = "0.8" toml_edit = "0.22" tonic = {version = "0.12.3", features = ["tls", "tls-roots"]} -tower-service = "0.3.2" +tower = { version = "0.5.2", default-features = false } +tower-http = { version = "0.6.2", features = ["request-id", "trace"] } +tower-service = "0.3.3" tracing = "0.1" tracing-error = "0.2" tracing-opentelemetry = "0.27" diff --git a/compute_tools/Cargo.toml b/compute_tools/Cargo.toml index 9525b27818..33892813c4 100644 --- a/compute_tools/Cargo.toml +++ b/compute_tools/Cargo.toml @@ -15,6 +15,7 @@ aws-config.workspace = true aws-sdk-s3.workspace = true aws-sdk-kms.workspace = true anyhow.workspace = true +axum = { workspace = true, features = [] } camino.workspace = true chrono.workspace = true cfg-if.workspace = true @@ -22,7 +23,7 @@ clap.workspace = true fail.workspace = true flate2.workspace = true futures.workspace = true -hyper0 = { workspace = true, features = ["full"] } +http.workspace = true metrics.workspace = true nix.workspace = true notify.workspace = true @@ -37,6 +38,8 @@ serde_with.workspace = true serde_json.workspace = true signal-hook.workspace = true tar.workspace = true +tower.workspace = true +tower-http.workspace = true reqwest = { workspace = true, features = ["json"] } tokio = { workspace = true, features = ["rt", "rt-multi-thread"] } tokio-postgres.workspace = true diff --git a/compute_tools/src/bin/compute_ctl.rs b/compute_tools/src/bin/compute_ctl.rs index 6ede5fdceb..04432ad0f3 100644 --- a/compute_tools/src/bin/compute_ctl.rs +++ b/compute_tools/src/bin/compute_ctl.rs @@ -60,7 +60,7 @@ use compute_tools::compute::{ }; use compute_tools::configurator::launch_configurator; use compute_tools::extension_server::get_pg_version_string; -use compute_tools::http::api::launch_http_server; +use compute_tools::http::launch_http_server; use compute_tools::logger::*; use compute_tools::monitor::launch_monitor; use compute_tools::params::*; @@ -493,7 +493,10 @@ fn start_postgres( let mut pg = None; if !prestartup_failed { pg = match compute.start_compute() { - Ok(pg) => Some(pg), + Ok(pg) => { + info!(postmaster_pid = %pg.0.id(), "Postgres was started"); + Some(pg) + } Err(err) => { error!("could not start the compute node: {:#}", err); compute.set_failed_status(err); @@ -591,6 +594,8 @@ fn wait_postgres(pg: Option) -> Result { // propagate to Postgres and it will be shut down as well. let mut exit_code = None; if let Some((mut pg, logs_handle)) = pg { + info!(postmaster_pid = %pg.id(), "Waiting for Postgres to exit"); + let ecode = pg .wait() .expect("failed to start waiting on Postgres process"); diff --git a/compute_tools/src/catalog.rs b/compute_tools/src/catalog.rs index 72198a9479..4a297cfacf 100644 --- a/compute_tools/src/catalog.rs +++ b/compute_tools/src/catalog.rs @@ -36,11 +36,11 @@ pub async fn get_dbs_and_roles(compute: &Arc) -> anyhow::Result ComputeStatusResponse { - ComputeStatusResponse { - start_time: state.start_time, - tenant: state - .pspec - .as_ref() - .map(|pspec| pspec.tenant_id.to_string()), - timeline: state - .pspec - .as_ref() - .map(|pspec| pspec.timeline_id.to_string()), - status: state.status, - last_active: state.last_active, - error: state.error.clone(), - } -} - -// Service function to handle all available routes. -async fn routes(req: Request, compute: &Arc) -> Response { - // - // NOTE: The URI path is currently included in traces. That's OK because - // it doesn't contain any variable parts or sensitive information. But - // please keep that in mind if you change the routing here. - // - match (req.method(), req.uri().path()) { - // Serialized compute state. - (&Method::GET, "/status") => { - debug!("serving /status GET request"); - let state = compute.state.lock().unwrap(); - let status_response = status_response_from_state(&state); - Response::new(Body::from(serde_json::to_string(&status_response).unwrap())) - } - - // Startup metrics in JSON format. Keep /metrics reserved for a possible - // future use for Prometheus metrics format. - (&Method::GET, "/metrics.json") => { - info!("serving /metrics.json GET request"); - let metrics = compute.state.lock().unwrap().metrics.clone(); - Response::new(Body::from(serde_json::to_string(&metrics).unwrap())) - } - - // Prometheus metrics - (&Method::GET, "/metrics") => { - debug!("serving /metrics GET request"); - - // When we call TextEncoder::encode() below, it will immediately - // return an error if a metric family has no metrics, so we need to - // preemptively filter out metric families with no metrics. - let metrics = installed_extensions::collect() - .into_iter() - .filter(|m| !m.get_metric().is_empty()) - .collect::>(); - - let encoder = TextEncoder::new(); - let mut buffer = vec![]; - - if let Err(err) = encoder.encode(&metrics, &mut buffer) { - let msg = format!("error handling /metrics request: {err}"); - error!(msg); - return render_json_error(&msg, StatusCode::INTERNAL_SERVER_ERROR); - } - - match Response::builder() - .status(StatusCode::OK) - .header(CONTENT_TYPE, encoder.format_type()) - .body(Body::from(buffer)) - { - Ok(response) => response, - Err(err) => { - let msg = format!("error handling /metrics request: {err}"); - error!(msg); - render_json_error(&msg, StatusCode::INTERNAL_SERVER_ERROR) - } - } - } - // Collect Postgres current usage insights - (&Method::GET, "/insights") => { - info!("serving /insights GET request"); - let status = compute.get_status(); - if status != ComputeStatus::Running { - let msg = format!("compute is not running, current status: {:?}", status); - error!(msg); - return Response::new(Body::from(msg)); - } - - let insights = compute.collect_insights().await; - Response::new(Body::from(insights)) - } - - (&Method::POST, "/check_writability") => { - info!("serving /check_writability POST request"); - let status = compute.get_status(); - if status != ComputeStatus::Running { - let msg = format!( - "invalid compute status for check_writability request: {:?}", - status - ); - error!(msg); - return Response::new(Body::from(msg)); - } - - let res = crate::checker::check_writability(compute).await; - match res { - Ok(_) => Response::new(Body::from("true")), - Err(e) => { - error!("check_writability failed: {}", e); - Response::new(Body::from(e.to_string())) - } - } - } - - (&Method::POST, "/extensions") => { - info!("serving /extensions POST request"); - let status = compute.get_status(); - if status != ComputeStatus::Running { - let msg = format!( - "invalid compute status for extensions request: {:?}", - status - ); - error!(msg); - return render_json_error(&msg, StatusCode::PRECONDITION_FAILED); - } - - let request = hyper::body::to_bytes(req.into_body()).await.unwrap(); - let request = serde_json::from_slice::(&request).unwrap(); - let res = compute - .install_extension(&request.extension, &request.database, request.version) - .await; - match res { - Ok(version) => render_json(Body::from( - serde_json::to_string(&ExtensionInstallResult { - extension: request.extension, - version, - }) - .unwrap(), - )), - Err(e) => { - error!("install_extension failed: {}", e); - render_json_error(&e.to_string(), StatusCode::INTERNAL_SERVER_ERROR) - } - } - } - - (&Method::GET, "/info") => { - let num_cpus = num_cpus::get_physical(); - info!("serving /info GET request. num_cpus: {}", num_cpus); - Response::new(Body::from( - serde_json::json!({ - "num_cpus": num_cpus, - }) - .to_string(), - )) - } - - // Accept spec in JSON format and request compute configuration. If - // anything goes wrong after we set the compute status to `ConfigurationPending` - // and update compute state with new spec, we basically leave compute - // in the potentially wrong state. That said, it's control-plane's - // responsibility to watch compute state after reconfiguration request - // and to clean restart in case of errors. - (&Method::POST, "/configure") => { - info!("serving /configure POST request"); - match handle_configure_request(req, compute).await { - Ok(msg) => Response::new(Body::from(msg)), - Err((msg, code)) => { - error!("error handling /configure request: {msg}"); - render_json_error(&msg, code) - } - } - } - - (&Method::POST, "/terminate") => { - info!("serving /terminate POST request"); - match handle_terminate_request(compute).await { - Ok(()) => Response::new(Body::empty()), - Err((msg, code)) => { - error!("error handling /terminate request: {msg}"); - render_json_error(&msg, code) - } - } - } - - (&Method::GET, "/dbs_and_roles") => { - info!("serving /dbs_and_roles GET request",); - match get_dbs_and_roles(compute).await { - Ok(res) => render_json(Body::from(serde_json::to_string(&res).unwrap())), - Err(_) => { - render_json_error("can't get dbs and roles", StatusCode::INTERNAL_SERVER_ERROR) - } - } - } - - (&Method::GET, "/database_schema") => { - let database = match must_get_query_param(&req, "database") { - Err(e) => return e.into_response(), - Ok(database) => database, - }; - info!("serving /database_schema GET request with database: {database}",); - match get_database_schema(compute, &database).await { - Ok(res) => render_plain(Body::wrap_stream(res)), - Err(SchemaDumpError::DatabaseDoesNotExist) => { - render_json_error("database does not exist", StatusCode::NOT_FOUND) - } - Err(e) => { - error!("can't get schema dump: {}", e); - render_json_error("can't get schema dump", StatusCode::INTERNAL_SERVER_ERROR) - } - } - } - - (&Method::POST, "/grants") => { - info!("serving /grants POST request"); - let status = compute.get_status(); - if status != ComputeStatus::Running { - let msg = format!( - "invalid compute status for set_role_grants request: {:?}", - status - ); - error!(msg); - return render_json_error(&msg, StatusCode::PRECONDITION_FAILED); - } - - let request = hyper::body::to_bytes(req.into_body()).await.unwrap(); - let request = serde_json::from_slice::(&request).unwrap(); - - let res = compute - .set_role_grants( - &request.database, - &request.schema, - &request.privileges, - &request.role, - ) - .await; - match res { - Ok(()) => render_json(Body::from( - serde_json::to_string(&SetRoleGrantsResponse { - database: request.database, - schema: request.schema, - role: request.role, - privileges: request.privileges, - }) - .unwrap(), - )), - Err(e) => render_json_error( - &format!("could not grant role privileges to the schema: {e}"), - // TODO: can we filter on role/schema not found errors - // and return appropriate error code? - StatusCode::INTERNAL_SERVER_ERROR, - ), - } - } - - // get the list of installed extensions - // currently only used in python tests - // TODO: call it from cplane - (&Method::GET, "/installed_extensions") => { - info!("serving /installed_extensions GET request"); - let status = compute.get_status(); - if status != ComputeStatus::Running { - let msg = format!( - "invalid compute status for extensions request: {:?}", - status - ); - error!(msg); - return Response::new(Body::from(msg)); - } - - let conf = compute.get_conn_conf(None); - let res = - task::spawn_blocking(move || installed_extensions::get_installed_extensions(conf)) - .await - .unwrap(); - - match res { - Ok(res) => render_json(Body::from(serde_json::to_string(&res).unwrap())), - Err(e) => render_json_error( - &format!("could not get list of installed extensions: {}", e), - StatusCode::INTERNAL_SERVER_ERROR, - ), - } - } - - (&Method::POST, "/failpoints") if cfg!(feature = "testing") => { - match failpoints_handler(req, CancellationToken::new()).await { - Ok(r) => r, - Err(ApiError::BadRequest(e)) => { - render_json_error(&e.to_string(), StatusCode::BAD_REQUEST) - } - Err(_) => { - render_json_error("Internal server error", StatusCode::INTERNAL_SERVER_ERROR) - } - } - } - - // download extension files from remote extension storage on demand - (&Method::POST, route) if route.starts_with("/extension_server/") => { - info!("serving {:?} POST request", route); - info!("req.uri {:?}", req.uri()); - - // don't even try to download extensions - // if no remote storage is configured - if compute.ext_remote_storage.is_none() { - info!("no extensions remote storage configured"); - let mut resp = Response::new(Body::from("no remote storage configured")); - *resp.status_mut() = StatusCode::INTERNAL_SERVER_ERROR; - return resp; - } - - let mut is_library = false; - if let Some(params) = req.uri().query() { - info!("serving {:?} POST request with params: {}", route, params); - if params == "is_library=true" { - is_library = true; - } else { - let mut resp = Response::new(Body::from("Wrong request parameters")); - *resp.status_mut() = StatusCode::BAD_REQUEST; - return resp; - } - } - let filename = route.split('/').last().unwrap().to_string(); - info!("serving /extension_server POST request, filename: {filename:?} is_library: {is_library}"); - - // get ext_name and path from spec - // don't lock compute_state for too long - let ext = { - let compute_state = compute.state.lock().unwrap(); - let pspec = compute_state.pspec.as_ref().expect("spec must be set"); - let spec = &pspec.spec; - - // debug only - info!("spec: {:?}", spec); - - let remote_extensions = match spec.remote_extensions.as_ref() { - Some(r) => r, - None => { - info!("no remote extensions spec was provided"); - let mut resp = Response::new(Body::from("no remote storage configured")); - *resp.status_mut() = StatusCode::INTERNAL_SERVER_ERROR; - return resp; - } - }; - - remote_extensions.get_ext( - &filename, - is_library, - &compute.build_tag, - &compute.pgversion, - ) - }; - - match ext { - Ok((ext_name, ext_path)) => { - match compute.download_extension(ext_name, ext_path).await { - Ok(_) => Response::new(Body::from("OK")), - Err(e) => { - error!("extension download failed: {}", e); - let mut resp = Response::new(Body::from(e.to_string())); - *resp.status_mut() = StatusCode::INTERNAL_SERVER_ERROR; - resp - } - } - } - Err(e) => { - warn!("extension download failed to find extension: {}", e); - let mut resp = Response::new(Body::from("failed to find file")); - *resp.status_mut() = StatusCode::INTERNAL_SERVER_ERROR; - resp - } - } - } - - // Return the `404 Not Found` for any other routes. - _ => { - let mut not_found = Response::new(Body::from("404 Not Found")); - *not_found.status_mut() = StatusCode::NOT_FOUND; - not_found - } - } -} - -async fn handle_configure_request( - req: Request, - compute: &Arc, -) -> Result { - if !compute.live_config_allowed { - return Err(( - "live configuration is not allowed for this compute node".to_string(), - StatusCode::PRECONDITION_FAILED, - )); - } - - let body_bytes = hyper::body::to_bytes(req.into_body()).await.unwrap(); - let spec_raw = String::from_utf8(body_bytes.to_vec()).unwrap(); - if let Ok(request) = serde_json::from_str::(&spec_raw) { - let spec = request.spec; - - let parsed_spec = match ParsedSpec::try_from(spec) { - Ok(ps) => ps, - Err(msg) => return Err((msg, StatusCode::BAD_REQUEST)), - }; - - // XXX: wrap state update under lock in code blocks. Otherwise, - // we will try to `Send` `mut state` into the spawned thread - // bellow, which will cause error: - // ``` - // error: future cannot be sent between threads safely - // ``` - { - let mut state = compute.state.lock().unwrap(); - if state.status != ComputeStatus::Empty && state.status != ComputeStatus::Running { - let msg = format!( - "invalid compute status for configuration request: {:?}", - state.status.clone() - ); - return Err((msg, StatusCode::PRECONDITION_FAILED)); - } - state.pspec = Some(parsed_spec); - state.set_status(ComputeStatus::ConfigurationPending, &compute.state_changed); - drop(state); - info!("set new spec and notified waiters"); - } - - // Spawn a blocking thread to wait for compute to become Running. - // This is needed to do not block the main pool of workers and - // be able to serve other requests while some particular request - // is waiting for compute to finish configuration. - let c = compute.clone(); - task::spawn_blocking(move || { - let mut state = c.state.lock().unwrap(); - while state.status != ComputeStatus::Running { - state = c.state_changed.wait(state).unwrap(); - info!( - "waiting for compute to become Running, current status: {:?}", - state.status - ); - - if state.status == ComputeStatus::Failed { - let err = state.error.as_ref().map_or("unknown error", |x| x); - let msg = format!("compute configuration failed: {:?}", err); - return Err((msg, StatusCode::INTERNAL_SERVER_ERROR)); - } - } - - Ok(()) - }) - .await - .unwrap()?; - - // Return current compute state if everything went well. - let state = compute.state.lock().unwrap().clone(); - let status_response = status_response_from_state(&state); - Ok(serde_json::to_string(&status_response).unwrap()) - } else { - Err(("invalid spec".to_string(), StatusCode::BAD_REQUEST)) - } -} - -fn render_json_error(e: &str, status: StatusCode) -> Response { - let error = GenericAPIError { - error: e.to_string(), - }; - Response::builder() - .status(status) - .header(CONTENT_TYPE, "application/json") - .body(Body::from(serde_json::to_string(&error).unwrap())) - .unwrap() -} - -fn render_json(body: Body) -> Response { - Response::builder() - .header(CONTENT_TYPE, "application/json") - .body(body) - .unwrap() -} - -fn render_plain(body: Body) -> Response { - Response::builder() - .header(CONTENT_TYPE, "text/plain") - .body(body) - .unwrap() -} - -async fn handle_terminate_request(compute: &Arc) -> Result<(), (String, StatusCode)> { - { - let mut state = compute.state.lock().unwrap(); - if state.status == ComputeStatus::Terminated { - return Ok(()); - } - if state.status != ComputeStatus::Empty && state.status != ComputeStatus::Running { - let msg = format!( - "invalid compute status for termination request: {}", - state.status - ); - return Err((msg, StatusCode::PRECONDITION_FAILED)); - } - state.set_status(ComputeStatus::TerminationPending, &compute.state_changed); - drop(state); - } - - forward_termination_signal(); - info!("sent signal and notified waiters"); - - // Spawn a blocking thread to wait for compute to become Terminated. - // This is needed to do not block the main pool of workers and - // be able to serve other requests while some particular request - // is waiting for compute to finish configuration. - let c = compute.clone(); - task::spawn_blocking(move || { - let mut state = c.state.lock().unwrap(); - while state.status != ComputeStatus::Terminated { - state = c.state_changed.wait(state).unwrap(); - info!( - "waiting for compute to become {}, current status: {:?}", - ComputeStatus::Terminated, - state.status - ); - } - - Ok(()) - }) - .await - .unwrap()?; - info!("terminated Postgres"); - Ok(()) -} - -// Main Hyper HTTP server function that runs it and blocks waiting on it forever. -#[tokio::main] -async fn serve(port: u16, state: Arc) { - // this usually binds to both IPv4 and IPv6 on linux - // see e.g. https://github.com/rust-lang/rust/pull/34440 - let addr = SocketAddr::new(IpAddr::from(Ipv6Addr::UNSPECIFIED), port); - - let make_service = make_service_fn(move |_conn| { - let state = state.clone(); - async move { - Ok::<_, Infallible>(service_fn(move |req: Request| { - let state = state.clone(); - async move { - Ok::<_, Infallible>( - // NOTE: We include the URI path in the string. It - // doesn't contain any variable parts or sensitive - // information in this API. - tracing_utils::http::tracing_handler( - req, - |req| routes(req, &state), - OtelName::UriPath, - ) - .await, - ) - } - })) - } - }); - - info!("starting HTTP server on {}", addr); - - let server = Server::bind(&addr).serve(make_service); - - // Run this server forever - if let Err(e) = server.await { - error!("server error: {}", e); - } -} - -/// Launch a separate Hyper HTTP API server thread and return its `JoinHandle`. -pub fn launch_http_server(port: u16, state: &Arc) -> Result> { - let state = Arc::clone(state); - - Ok(thread::Builder::new() - .name("http-endpoint".into()) - .spawn(move || serve(port, state))?) -} diff --git a/compute_tools/src/http/extract/json.rs b/compute_tools/src/http/extract/json.rs new file mode 100644 index 0000000000..41f13625ad --- /dev/null +++ b/compute_tools/src/http/extract/json.rs @@ -0,0 +1,48 @@ +use std::ops::{Deref, DerefMut}; + +use axum::{ + async_trait, + extract::{rejection::JsonRejection, FromRequest, Request}, +}; +use compute_api::responses::GenericAPIError; +use http::StatusCode; + +/// Custom `Json` extractor, so that we can format errors into +/// `JsonResponse`. +#[derive(Debug, Clone, Copy, Default)] +pub(crate) struct Json(pub T); + +#[async_trait] +impl FromRequest for Json +where + axum::Json: FromRequest, + S: Send + Sync, +{ + type Rejection = (StatusCode, axum::Json); + + async fn from_request(req: Request, state: &S) -> Result { + match axum::Json::::from_request(req, state).await { + Ok(value) => Ok(Self(value.0)), + Err(rejection) => Err(( + rejection.status(), + axum::Json(GenericAPIError { + error: rejection.body_text().to_lowercase(), + }), + )), + } + } +} + +impl Deref for Json { + type Target = T; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl DerefMut for Json { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.0 + } +} diff --git a/compute_tools/src/http/extract/mod.rs b/compute_tools/src/http/extract/mod.rs new file mode 100644 index 0000000000..1b690e444d --- /dev/null +++ b/compute_tools/src/http/extract/mod.rs @@ -0,0 +1,7 @@ +pub(crate) mod json; +pub(crate) mod path; +pub(crate) mod query; + +pub(crate) use json::Json; +pub(crate) use path::Path; +pub(crate) use query::Query; diff --git a/compute_tools/src/http/extract/path.rs b/compute_tools/src/http/extract/path.rs new file mode 100644 index 0000000000..95edc657f2 --- /dev/null +++ b/compute_tools/src/http/extract/path.rs @@ -0,0 +1,48 @@ +use std::ops::{Deref, DerefMut}; + +use axum::{ + async_trait, + extract::{rejection::PathRejection, FromRequestParts}, +}; +use compute_api::responses::GenericAPIError; +use http::{request::Parts, StatusCode}; + +/// Custom `Path` extractor, so that we can format errors into +/// `JsonResponse`. +#[derive(Debug, Clone, Copy, Default)] +pub(crate) struct Path(pub T); + +#[async_trait] +impl FromRequestParts for Path +where + axum::extract::Path: FromRequestParts, + S: Send + Sync, +{ + type Rejection = (StatusCode, axum::Json); + + async fn from_request_parts(parts: &mut Parts, state: &S) -> Result { + match axum::extract::Path::::from_request_parts(parts, state).await { + Ok(value) => Ok(Self(value.0)), + Err(rejection) => Err(( + rejection.status(), + axum::Json(GenericAPIError { + error: rejection.body_text().to_ascii_lowercase(), + }), + )), + } + } +} + +impl Deref for Path { + type Target = T; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl DerefMut for Path { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.0 + } +} diff --git a/compute_tools/src/http/extract/query.rs b/compute_tools/src/http/extract/query.rs new file mode 100644 index 0000000000..a1f1b0cef0 --- /dev/null +++ b/compute_tools/src/http/extract/query.rs @@ -0,0 +1,48 @@ +use std::ops::{Deref, DerefMut}; + +use axum::{ + async_trait, + extract::{rejection::QueryRejection, FromRequestParts}, +}; +use compute_api::responses::GenericAPIError; +use http::{request::Parts, StatusCode}; + +/// Custom `Query` extractor, so that we can format errors into +/// `JsonResponse`. +#[derive(Debug, Clone, Copy, Default)] +pub(crate) struct Query(pub T); + +#[async_trait] +impl FromRequestParts for Query +where + axum::extract::Query: FromRequestParts, + S: Send + Sync, +{ + type Rejection = (StatusCode, axum::Json); + + async fn from_request_parts(parts: &mut Parts, state: &S) -> Result { + match axum::extract::Query::::from_request_parts(parts, state).await { + Ok(value) => Ok(Self(value.0)), + Err(rejection) => Err(( + rejection.status(), + axum::Json(GenericAPIError { + error: rejection.body_text().to_ascii_lowercase(), + }), + )), + } + } +} + +impl Deref for Query { + type Target = T; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl DerefMut for Query { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.0 + } +} diff --git a/compute_tools/src/http/mod.rs b/compute_tools/src/http/mod.rs index e5fdf85eed..a596bea504 100644 --- a/compute_tools/src/http/mod.rs +++ b/compute_tools/src/http/mod.rs @@ -1 +1,56 @@ -pub mod api; +use axum::{body::Body, response::Response}; +use compute_api::responses::{ComputeStatus, GenericAPIError}; +use http::{header::CONTENT_TYPE, StatusCode}; +use serde::Serialize; +use tracing::error; + +pub use server::launch_http_server; + +mod extract; +mod routes; +mod server; + +/// Convenience response builder for JSON responses +struct JsonResponse; + +impl JsonResponse { + /// Helper for actually creating a response + fn create_response(code: StatusCode, body: impl Serialize) -> Response { + Response::builder() + .status(code) + .header(CONTENT_TYPE.as_str(), "application/json") + .body(Body::from(serde_json::to_string(&body).unwrap())) + .unwrap() + } + + /// Create a successful error response + pub(self) fn success(code: StatusCode, body: impl Serialize) -> Response { + assert!({ + let code = code.as_u16(); + + (200..300).contains(&code) + }); + + Self::create_response(code, body) + } + + /// Create an error response + pub(self) fn error(code: StatusCode, error: impl ToString) -> Response { + assert!(code.as_u16() >= 400); + + let message = error.to_string(); + error!(message); + + Self::create_response(code, &GenericAPIError { error: message }) + } + + /// Create an error response related to the compute being in an invalid state + pub(self) fn invalid_status(status: ComputeStatus) -> Response { + Self::create_response( + StatusCode::PRECONDITION_FAILED, + &GenericAPIError { + error: format!("invalid compute status: {status}"), + }, + ) + } +} diff --git a/compute_tools/src/http/openapi_spec.yaml b/compute_tools/src/http/openapi_spec.yaml index 24a67cac71..50319cdd85 100644 --- a/compute_tools/src/http/openapi_spec.yaml +++ b/compute_tools/src/http/openapi_spec.yaml @@ -37,7 +37,7 @@ paths: schema: $ref: "#/components/schemas/ComputeMetrics" - /metrics + /metrics: get: tags: - Info diff --git a/compute_tools/src/http/routes/check_writability.rs b/compute_tools/src/http/routes/check_writability.rs new file mode 100644 index 0000000000..d7feb055e9 --- /dev/null +++ b/compute_tools/src/http/routes/check_writability.rs @@ -0,0 +1,20 @@ +use std::sync::Arc; + +use axum::{extract::State, response::Response}; +use compute_api::responses::ComputeStatus; +use http::StatusCode; + +use crate::{checker::check_writability, compute::ComputeNode, http::JsonResponse}; + +/// Check that the compute is currently running. +pub(in crate::http) async fn is_writable(State(compute): State>) -> Response { + let status = compute.get_status(); + if status != ComputeStatus::Running { + return JsonResponse::invalid_status(status); + } + + match check_writability(&compute).await { + Ok(_) => JsonResponse::success(StatusCode::OK, true), + Err(e) => JsonResponse::error(StatusCode::INTERNAL_SERVER_ERROR, e), + } +} diff --git a/compute_tools/src/http/routes/configure.rs b/compute_tools/src/http/routes/configure.rs new file mode 100644 index 0000000000..2546cbc344 --- /dev/null +++ b/compute_tools/src/http/routes/configure.rs @@ -0,0 +1,91 @@ +use std::sync::Arc; + +use axum::{extract::State, response::Response}; +use compute_api::{ + requests::ConfigurationRequest, + responses::{ComputeStatus, ComputeStatusResponse}, +}; +use http::StatusCode; +use tokio::task; +use tracing::info; + +use crate::{ + compute::{ComputeNode, ParsedSpec}, + http::{extract::Json, JsonResponse}, +}; + +// Accept spec in JSON format and request compute configuration. If anything +// goes wrong after we set the compute status to `ConfigurationPending` and +// update compute state with new spec, we basically leave compute in the +// potentially wrong state. That said, it's control-plane's responsibility to +// watch compute state after reconfiguration request and to clean restart in +// case of errors. +pub(in crate::http) async fn configure( + State(compute): State>, + request: Json, +) -> Response { + if !compute.live_config_allowed { + return JsonResponse::error( + StatusCode::PRECONDITION_FAILED, + "live configuration is not allowed for this compute node".to_string(), + ); + } + + let pspec = match ParsedSpec::try_from(request.spec.clone()) { + Ok(p) => p, + Err(e) => return JsonResponse::error(StatusCode::BAD_REQUEST, e), + }; + + // XXX: wrap state update under lock in a code block. Otherwise, we will try + // to `Send` `mut state` into the spawned thread bellow, which will cause + // the following rustc error: + // + // error: future cannot be sent between threads safely + { + let mut state = compute.state.lock().unwrap(); + if !matches!(state.status, ComputeStatus::Empty | ComputeStatus::Running) { + return JsonResponse::invalid_status(state.status); + } + + state.pspec = Some(pspec); + state.set_status(ComputeStatus::ConfigurationPending, &compute.state_changed); + drop(state); + } + + // Spawn a blocking thread to wait for compute to become Running. This is + // needed to do not block the main pool of workers and be able to serve + // other requests while some particular request is waiting for compute to + // finish configuration. + let c = compute.clone(); + let completed = task::spawn_blocking(move || { + let mut state = c.state.lock().unwrap(); + while state.status != ComputeStatus::Running { + state = c.state_changed.wait(state).unwrap(); + info!( + "waiting for compute to become {}, current status: {}", + ComputeStatus::Running, + state.status + ); + + if state.status == ComputeStatus::Failed { + let err = state.error.as_ref().map_or("unknown error", |x| x); + let msg = format!("compute configuration failed: {:?}", err); + return Err(msg); + } + } + + Ok(()) + }) + .await + .unwrap(); + + if let Err(e) = completed { + return JsonResponse::error(StatusCode::INTERNAL_SERVER_ERROR, e); + } + + // Return current compute state if everything went well. + let state = compute.state.lock().unwrap().clone(); + let body = ComputeStatusResponse::from(&state); + + JsonResponse::success(StatusCode::OK, body) +} diff --git a/compute_tools/src/http/routes/database_schema.rs b/compute_tools/src/http/routes/database_schema.rs new file mode 100644 index 0000000000..fd716272dc --- /dev/null +++ b/compute_tools/src/http/routes/database_schema.rs @@ -0,0 +1,34 @@ +use std::sync::Arc; + +use axum::{body::Body, extract::State, response::Response}; +use http::{header::CONTENT_TYPE, StatusCode}; +use serde::Deserialize; + +use crate::{ + catalog::{get_database_schema, SchemaDumpError}, + compute::ComputeNode, + http::{extract::Query, JsonResponse}, +}; + +#[derive(Debug, Clone, Deserialize)] +pub(in crate::http) struct DatabaseSchemaParams { + database: String, +} + +/// Get a schema dump of the requested database. +pub(in crate::http) async fn get_schema_dump( + params: Query, + State(compute): State>, +) -> Response { + match get_database_schema(&compute, ¶ms.database).await { + Ok(schema) => Response::builder() + .status(StatusCode::OK) + .header(CONTENT_TYPE.as_str(), "application/json") + .body(Body::from_stream(schema)) + .unwrap(), + Err(SchemaDumpError::DatabaseDoesNotExist) => { + JsonResponse::error(StatusCode::NOT_FOUND, SchemaDumpError::DatabaseDoesNotExist) + } + Err(e) => JsonResponse::error(StatusCode::INTERNAL_SERVER_ERROR, e), + } +} diff --git a/compute_tools/src/http/routes/dbs_and_roles.rs b/compute_tools/src/http/routes/dbs_and_roles.rs new file mode 100644 index 0000000000..4843c3fab4 --- /dev/null +++ b/compute_tools/src/http/routes/dbs_and_roles.rs @@ -0,0 +1,16 @@ +use std::sync::Arc; + +use axum::{extract::State, response::Response}; +use http::StatusCode; + +use crate::{catalog::get_dbs_and_roles, compute::ComputeNode, http::JsonResponse}; + +/// Get the databases and roles from the compute. +pub(in crate::http) async fn get_catalog_objects( + State(compute): State>, +) -> Response { + match get_dbs_and_roles(&compute).await { + Ok(catalog_objects) => JsonResponse::success(StatusCode::OK, catalog_objects), + Err(e) => JsonResponse::error(StatusCode::INTERNAL_SERVER_ERROR, e), + } +} diff --git a/compute_tools/src/http/routes/extension_server.rs b/compute_tools/src/http/routes/extension_server.rs new file mode 100644 index 0000000000..ee5bc675ba --- /dev/null +++ b/compute_tools/src/http/routes/extension_server.rs @@ -0,0 +1,67 @@ +use std::sync::Arc; + +use axum::{ + extract::State, + response::{IntoResponse, Response}, +}; +use http::StatusCode; +use serde::Deserialize; + +use crate::{ + compute::ComputeNode, + http::{ + extract::{Path, Query}, + JsonResponse, + }, +}; + +#[derive(Debug, Clone, Deserialize)] +pub(in crate::http) struct ExtensionServerParams { + is_library: Option, +} + +/// Download a remote extension. +pub(in crate::http) async fn download_extension( + Path(filename): Path, + params: Query, + State(compute): State>, +) -> Response { + // Don't even try to download extensions if no remote storage is configured + if compute.ext_remote_storage.is_none() { + return JsonResponse::error( + StatusCode::PRECONDITION_FAILED, + "remote storage is not configured", + ); + } + + let ext = { + let state = compute.state.lock().unwrap(); + let pspec = state.pspec.as_ref().unwrap(); + let spec = &pspec.spec; + + let remote_extensions = match spec.remote_extensions.as_ref() { + Some(r) => r, + None => { + return JsonResponse::error( + StatusCode::CONFLICT, + "information about remote extensions is unavailable", + ); + } + }; + + remote_extensions.get_ext( + &filename, + params.is_library.unwrap_or(false), + &compute.build_tag, + &compute.pgversion, + ) + }; + + match ext { + Ok((ext_name, ext_path)) => match compute.download_extension(ext_name, ext_path).await { + Ok(_) => StatusCode::OK.into_response(), + Err(e) => JsonResponse::error(StatusCode::INTERNAL_SERVER_ERROR, e), + }, + Err(e) => JsonResponse::error(StatusCode::NOT_FOUND, e), + } +} diff --git a/compute_tools/src/http/routes/extensions.rs b/compute_tools/src/http/routes/extensions.rs new file mode 100644 index 0000000000..1fc03b9109 --- /dev/null +++ b/compute_tools/src/http/routes/extensions.rs @@ -0,0 +1,45 @@ +use std::sync::Arc; + +use axum::{extract::State, response::Response}; +use compute_api::{ + requests::ExtensionInstallRequest, + responses::{ComputeStatus, ExtensionInstallResponse}, +}; +use http::StatusCode; + +use crate::{ + compute::ComputeNode, + http::{extract::Json, JsonResponse}, +}; + +/// Install a extension. +pub(in crate::http) async fn install_extension( + State(compute): State>, + request: Json, +) -> Response { + let status = compute.get_status(); + if status != ComputeStatus::Running { + return JsonResponse::invalid_status(status); + } + + match compute + .install_extension( + &request.extension, + &request.database, + request.version.to_string(), + ) + .await + { + Ok(version) => JsonResponse::success( + StatusCode::CREATED, + Some(ExtensionInstallResponse { + extension: request.extension.clone(), + version, + }), + ), + Err(e) => JsonResponse::error( + StatusCode::INTERNAL_SERVER_ERROR, + format!("failed to install extension: {e}"), + ), + } +} diff --git a/compute_tools/src/http/routes/failpoints.rs b/compute_tools/src/http/routes/failpoints.rs new file mode 100644 index 0000000000..2ec4511676 --- /dev/null +++ b/compute_tools/src/http/routes/failpoints.rs @@ -0,0 +1,35 @@ +use axum::response::{IntoResponse, Response}; +use http::StatusCode; +use tracing::info; +use utils::failpoint_support::{apply_failpoint, ConfigureFailpointsRequest}; + +use crate::http::{extract::Json, JsonResponse}; + +/// Configure failpoints for testing purposes. +pub(in crate::http) async fn configure_failpoints( + failpoints: Json, +) -> Response { + if !fail::has_failpoints() { + return JsonResponse::error( + StatusCode::PRECONDITION_FAILED, + "Cannot manage failpoints because neon was compiled without failpoints support", + ); + } + + for fp in &*failpoints { + info!("cfg failpoint: {} {}", fp.name, fp.actions); + + // We recognize one extra "action" that's not natively recognized + // by the failpoints crate: exit, to immediately kill the process + let cfg_result = apply_failpoint(&fp.name, &fp.actions); + + if let Err(e) = cfg_result { + return JsonResponse::error( + StatusCode::BAD_REQUEST, + format!("failed to configure failpoints: {e}"), + ); + } + } + + StatusCode::OK.into_response() +} diff --git a/compute_tools/src/http/routes/grants.rs b/compute_tools/src/http/routes/grants.rs new file mode 100644 index 0000000000..3f67f011e5 --- /dev/null +++ b/compute_tools/src/http/routes/grants.rs @@ -0,0 +1,48 @@ +use std::sync::Arc; + +use axum::{extract::State, response::Response}; +use compute_api::{ + requests::SetRoleGrantsRequest, + responses::{ComputeStatus, SetRoleGrantsResponse}, +}; +use http::StatusCode; + +use crate::{ + compute::ComputeNode, + http::{extract::Json, JsonResponse}, +}; + +/// Add grants for a role. +pub(in crate::http) async fn add_grant( + State(compute): State>, + request: Json, +) -> Response { + let status = compute.get_status(); + if status != ComputeStatus::Running { + return JsonResponse::invalid_status(status); + } + + match compute + .set_role_grants( + &request.database, + &request.schema, + &request.privileges, + &request.role, + ) + .await + { + Ok(()) => JsonResponse::success( + StatusCode::CREATED, + Some(SetRoleGrantsResponse { + database: request.database.clone(), + schema: request.schema.clone(), + role: request.role.clone(), + privileges: request.privileges.clone(), + }), + ), + Err(e) => JsonResponse::error( + StatusCode::INTERNAL_SERVER_ERROR, + format!("failed to grant role privileges to the schema: {e}"), + ), + } +} diff --git a/compute_tools/src/http/routes/info.rs b/compute_tools/src/http/routes/info.rs new file mode 100644 index 0000000000..32d6fea74c --- /dev/null +++ b/compute_tools/src/http/routes/info.rs @@ -0,0 +1,11 @@ +use axum::response::Response; +use compute_api::responses::InfoResponse; +use http::StatusCode; + +use crate::http::JsonResponse; + +/// Get information about the physical characteristics about the compute. +pub(in crate::http) async fn get_info() -> Response { + let num_cpus = num_cpus::get_physical(); + JsonResponse::success(StatusCode::OK, &InfoResponse { num_cpus }) +} diff --git a/compute_tools/src/http/routes/insights.rs b/compute_tools/src/http/routes/insights.rs new file mode 100644 index 0000000000..6b03a461c3 --- /dev/null +++ b/compute_tools/src/http/routes/insights.rs @@ -0,0 +1,18 @@ +use std::sync::Arc; + +use axum::{extract::State, response::Response}; +use compute_api::responses::ComputeStatus; +use http::StatusCode; + +use crate::{compute::ComputeNode, http::JsonResponse}; + +/// Collect current Postgres usage insights. +pub(in crate::http) async fn get_insights(State(compute): State>) -> Response { + let status = compute.get_status(); + if status != ComputeStatus::Running { + return JsonResponse::invalid_status(status); + } + + let insights = compute.collect_insights().await; + JsonResponse::success(StatusCode::OK, insights) +} diff --git a/compute_tools/src/http/routes/installed_extensions.rs b/compute_tools/src/http/routes/installed_extensions.rs new file mode 100644 index 0000000000..db74a6b195 --- /dev/null +++ b/compute_tools/src/http/routes/installed_extensions.rs @@ -0,0 +1,33 @@ +use std::sync::Arc; + +use axum::{extract::State, response::Response}; +use compute_api::responses::ComputeStatus; +use http::StatusCode; +use tokio::task; + +use crate::{compute::ComputeNode, http::JsonResponse, installed_extensions}; + +/// Get a list of installed extensions. +pub(in crate::http) async fn get_installed_extensions( + State(compute): State>, +) -> Response { + let status = compute.get_status(); + if status != ComputeStatus::Running { + return JsonResponse::invalid_status(status); + } + + let conf = compute.get_conn_conf(None); + let res = task::spawn_blocking(move || installed_extensions::get_installed_extensions(conf)) + .await + .unwrap(); + + match res { + Ok(installed_extensions) => { + JsonResponse::success(StatusCode::OK, Some(installed_extensions)) + } + Err(e) => JsonResponse::error( + StatusCode::INTERNAL_SERVER_ERROR, + format!("failed to get list of installed extensions: {e}"), + ), + } +} diff --git a/compute_tools/src/http/routes/metrics.rs b/compute_tools/src/http/routes/metrics.rs new file mode 100644 index 0000000000..40d71b5de7 --- /dev/null +++ b/compute_tools/src/http/routes/metrics.rs @@ -0,0 +1,32 @@ +use axum::{body::Body, response::Response}; +use http::header::CONTENT_TYPE; +use http::StatusCode; +use metrics::proto::MetricFamily; +use metrics::Encoder; +use metrics::TextEncoder; + +use crate::{http::JsonResponse, installed_extensions}; + +/// Expose Prometheus metrics. +pub(in crate::http) async fn get_metrics() -> Response { + // When we call TextEncoder::encode() below, it will immediately return an + // error if a metric family has no metrics, so we need to preemptively + // filter out metric families with no metrics. + let metrics = installed_extensions::collect() + .into_iter() + .filter(|m| !m.get_metric().is_empty()) + .collect::>(); + + let encoder = TextEncoder::new(); + let mut buffer = vec![]; + + if let Err(e) = encoder.encode(&metrics, &mut buffer) { + return JsonResponse::error(StatusCode::INTERNAL_SERVER_ERROR, e); + } + + Response::builder() + .status(StatusCode::OK) + .header(CONTENT_TYPE, encoder.format_type()) + .body(Body::from(buffer)) + .unwrap() +} diff --git a/compute_tools/src/http/routes/metrics_json.rs b/compute_tools/src/http/routes/metrics_json.rs new file mode 100644 index 0000000000..0709db5011 --- /dev/null +++ b/compute_tools/src/http/routes/metrics_json.rs @@ -0,0 +1,12 @@ +use std::sync::Arc; + +use axum::{extract::State, response::Response}; +use http::StatusCode; + +use crate::{compute::ComputeNode, http::JsonResponse}; + +/// Get startup metrics. +pub(in crate::http) async fn get_metrics(State(compute): State>) -> Response { + let metrics = compute.state.lock().unwrap().metrics.clone(); + JsonResponse::success(StatusCode::OK, metrics) +} diff --git a/compute_tools/src/http/routes/mod.rs b/compute_tools/src/http/routes/mod.rs new file mode 100644 index 0000000000..3efa1153ad --- /dev/null +++ b/compute_tools/src/http/routes/mod.rs @@ -0,0 +1,38 @@ +use compute_api::responses::ComputeStatusResponse; + +use crate::compute::ComputeState; + +pub(in crate::http) mod check_writability; +pub(in crate::http) mod configure; +pub(in crate::http) mod database_schema; +pub(in crate::http) mod dbs_and_roles; +pub(in crate::http) mod extension_server; +pub(in crate::http) mod extensions; +pub(in crate::http) mod failpoints; +pub(in crate::http) mod grants; +pub(in crate::http) mod info; +pub(in crate::http) mod insights; +pub(in crate::http) mod installed_extensions; +pub(in crate::http) mod metrics; +pub(in crate::http) mod metrics_json; +pub(in crate::http) mod status; +pub(in crate::http) mod terminate; + +impl From<&ComputeState> for ComputeStatusResponse { + fn from(state: &ComputeState) -> Self { + ComputeStatusResponse { + start_time: state.start_time, + tenant: state + .pspec + .as_ref() + .map(|pspec| pspec.tenant_id.to_string()), + timeline: state + .pspec + .as_ref() + .map(|pspec| pspec.timeline_id.to_string()), + status: state.status, + last_active: state.last_active, + error: state.error.clone(), + } + } +} diff --git a/compute_tools/src/http/routes/status.rs b/compute_tools/src/http/routes/status.rs new file mode 100644 index 0000000000..d64d53a58f --- /dev/null +++ b/compute_tools/src/http/routes/status.rs @@ -0,0 +1,14 @@ +use std::{ops::Deref, sync::Arc}; + +use axum::{extract::State, http::StatusCode, response::Response}; +use compute_api::responses::ComputeStatusResponse; + +use crate::{compute::ComputeNode, http::JsonResponse}; + +/// Retrieve the state of the comute. +pub(in crate::http) async fn get_status(State(compute): State>) -> Response { + let state = compute.state.lock().unwrap(); + let body = ComputeStatusResponse::from(state.deref()); + + JsonResponse::success(StatusCode::OK, body) +} diff --git a/compute_tools/src/http/routes/terminate.rs b/compute_tools/src/http/routes/terminate.rs new file mode 100644 index 0000000000..7acd84f236 --- /dev/null +++ b/compute_tools/src/http/routes/terminate.rs @@ -0,0 +1,58 @@ +use std::sync::Arc; + +use axum::{ + extract::State, + response::{IntoResponse, Response}, +}; +use compute_api::responses::ComputeStatus; +use http::StatusCode; +use tokio::task; +use tracing::info; + +use crate::{ + compute::{forward_termination_signal, ComputeNode}, + http::JsonResponse, +}; + +/// Terminate the compute. +pub(in crate::http) async fn terminate(State(compute): State>) -> Response { + { + let mut state = compute.state.lock().unwrap(); + if state.status == ComputeStatus::Terminated { + return StatusCode::CREATED.into_response(); + } + + if !matches!(state.status, ComputeStatus::Empty | ComputeStatus::Running) { + return JsonResponse::invalid_status(state.status); + } + + state.set_status(ComputeStatus::TerminationPending, &compute.state_changed); + drop(state); + } + + forward_termination_signal(); + info!("sent signal and notified waiters"); + + // Spawn a blocking thread to wait for compute to become Terminated. + // This is needed to do not block the main pool of workers and + // be able to serve other requests while some particular request + // is waiting for compute to finish configuration. + let c = compute.clone(); + task::spawn_blocking(move || { + let mut state = c.state.lock().unwrap(); + while state.status != ComputeStatus::Terminated { + state = c.state_changed.wait(state).unwrap(); + info!( + "waiting for compute to become {}, current status: {:?}", + ComputeStatus::Terminated, + state.status + ); + } + }) + .await + .unwrap(); + + info!("terminated Postgres"); + + StatusCode::OK.into_response() +} diff --git a/compute_tools/src/http/server.rs b/compute_tools/src/http/server.rs new file mode 100644 index 0000000000..33d4b489a0 --- /dev/null +++ b/compute_tools/src/http/server.rs @@ -0,0 +1,165 @@ +use std::{ + net::{IpAddr, Ipv6Addr, SocketAddr}, + sync::{ + atomic::{AtomicU64, Ordering}, + Arc, + }, + thread, + time::Duration, +}; + +use anyhow::Result; +use axum::{ + response::{IntoResponse, Response}, + routing::{get, post}, + Router, +}; +use http::StatusCode; +use tokio::net::TcpListener; +use tower::ServiceBuilder; +use tower_http::{ + request_id::{MakeRequestId, PropagateRequestIdLayer, RequestId, SetRequestIdLayer}, + trace::TraceLayer, +}; +use tracing::{debug, error, info, Span}; + +use super::routes::{ + check_writability, configure, database_schema, dbs_and_roles, extension_server, extensions, + grants, info as info_route, insights, installed_extensions, metrics, metrics_json, status, + terminate, +}; +use crate::compute::ComputeNode; + +async fn handle_404() -> Response { + StatusCode::NOT_FOUND.into_response() +} + +#[derive(Clone, Default)] +struct ComputeMakeRequestId(Arc); + +impl MakeRequestId for ComputeMakeRequestId { + fn make_request_id( + &mut self, + _request: &http::Request, + ) -> Option { + let request_id = self + .0 + .fetch_add(1, Ordering::SeqCst) + .to_string() + .parse() + .unwrap(); + + Some(RequestId::new(request_id)) + } +} + +/// Run the HTTP server and wait on it forever. +#[tokio::main] +async fn serve(port: u16, compute: Arc) { + const X_REQUEST_ID: &str = "x-request-id"; + + let mut app = Router::new() + .route("/check_writability", post(check_writability::is_writable)) + .route("/configure", post(configure::configure)) + .route("/database_schema", get(database_schema::get_schema_dump)) + .route("/dbs_and_roles", get(dbs_and_roles::get_catalog_objects)) + .route( + "/extension_server/*filename", + post(extension_server::download_extension), + ) + .route("/extensions", post(extensions::install_extension)) + .route("/grants", post(grants::add_grant)) + .route("/info", get(info_route::get_info)) + .route("/insights", get(insights::get_insights)) + .route( + "/installed_extensions", + get(installed_extensions::get_installed_extensions), + ) + .route("/metrics", get(metrics::get_metrics)) + .route("/metrics.json", get(metrics_json::get_metrics)) + .route("/status", get(status::get_status)) + .route("/terminate", post(terminate::terminate)) + .fallback(handle_404) + .layer( + ServiceBuilder::new() + .layer(SetRequestIdLayer::x_request_id( + ComputeMakeRequestId::default(), + )) + .layer( + TraceLayer::new_for_http() + .on_request(|request: &http::Request<_>, _span: &Span| { + let request_id = request + .headers() + .get(X_REQUEST_ID) + .unwrap() + .to_str() + .unwrap(); + + match request.uri().path() { + "/metrics" => { + debug!(%request_id, "{} {}", request.method(), request.uri()) + } + _ => info!(%request_id, "{} {}", request.method(), request.uri()), + }; + }) + .on_response( + |response: &http::Response<_>, latency: Duration, _span: &Span| { + let request_id = response + .headers() + .get(X_REQUEST_ID) + .unwrap() + .to_str() + .unwrap(); + + info!( + %request_id, + code = response.status().as_u16(), + latency = latency.as_millis() + ) + }, + ), + ) + .layer(PropagateRequestIdLayer::x_request_id()), + ) + .with_state(compute); + + // Add in any testing support + if cfg!(feature = "testing") { + use super::routes::failpoints; + + app = app.route("/failpoints", post(failpoints::configure_failpoints)) + } + + // This usually binds to both IPv4 and IPv6 on Linux, see + // https://github.com/rust-lang/rust/pull/34440 for more information + let addr = SocketAddr::new(IpAddr::from(Ipv6Addr::UNSPECIFIED), port); + let listener = match TcpListener::bind(&addr).await { + Ok(listener) => listener, + Err(e) => { + error!( + "failed to bind the compute_ctl HTTP server to port {}: {}", + port, e + ); + return; + } + }; + + if let Ok(local_addr) = listener.local_addr() { + info!("compute_ctl HTTP server listening on {}", local_addr); + } else { + info!("compute_ctl HTTP server listening on port {}", port); + } + + if let Err(e) = axum::serve(listener, app).await { + error!("compute_ctl HTTP server error: {}", e); + } +} + +/// Launch a separate HTTP server thread and return its `JoinHandle`. +pub fn launch_http_server(port: u16, state: &Arc) -> Result> { + let state = Arc::clone(state); + + Ok(thread::Builder::new() + .name("http-server".into()) + .spawn(move || serve(port, state))?) +} diff --git a/compute_tools/src/lib.rs b/compute_tools/src/lib.rs index ee4cf2dfa5..12fea4e61a 100644 --- a/compute_tools/src/lib.rs +++ b/compute_tools/src/lib.rs @@ -3,8 +3,6 @@ #![deny(unsafe_code)] #![deny(clippy::undocumented_unsafe_blocks)] -extern crate hyper0 as hyper; - pub mod checker; pub mod config; pub mod configurator; diff --git a/control_plane/src/endpoint.rs b/control_plane/src/endpoint.rs index 5e47ec4811..b8027abf7c 100644 --- a/control_plane/src/endpoint.rs +++ b/control_plane/src/endpoint.rs @@ -62,7 +62,7 @@ use crate::local_env::LocalEnv; use crate::postgresql_conf::PostgresConf; use crate::storage_controller::StorageController; -use compute_api::responses::{ComputeState, ComputeStatus}; +use compute_api::responses::{ComputeStatus, ComputeStatusResponse}; use compute_api::spec::{Cluster, ComputeFeature, ComputeMode, ComputeSpec}; // contents of a endpoint.json file @@ -739,7 +739,7 @@ impl Endpoint { } // Call the /status HTTP API - pub async fn get_status(&self) -> Result { + pub async fn get_status(&self) -> Result { let client = reqwest::Client::new(); let response = client diff --git a/libs/compute_api/src/responses.rs b/libs/compute_api/src/responses.rs index 0d65f6a38d..9ce605089b 100644 --- a/libs/compute_api/src/responses.rs +++ b/libs/compute_api/src/responses.rs @@ -15,6 +15,17 @@ pub struct GenericAPIError { pub error: String, } +#[derive(Debug, Clone, Serialize)] +pub struct InfoResponse { + pub num_cpus: usize, +} + +#[derive(Debug, Clone, Serialize)] +pub struct ExtensionInstallResponse { + pub extension: PgIdent, + pub version: ExtVersion, +} + /// Response of the /status API #[derive(Serialize, Debug, Deserialize)] #[serde(rename_all = "snake_case")] @@ -28,16 +39,6 @@ pub struct ComputeStatusResponse { pub error: Option, } -#[derive(Deserialize, Serialize)] -#[serde(rename_all = "snake_case")] -pub struct ComputeState { - pub status: ComputeStatus, - /// Timestamp of the last Postgres activity - #[serde(serialize_with = "rfc3339_serialize")] - pub last_active: Option>, - pub error: Option, -} - #[derive(Serialize, Clone, Copy, Debug, Deserialize, PartialEq, Eq)] #[serde(rename_all = "snake_case")] pub enum ComputeStatus { @@ -78,7 +79,7 @@ impl Display for ComputeStatus { } } -fn rfc3339_serialize(x: &Option>, s: S) -> Result +pub fn rfc3339_serialize(x: &Option>, s: S) -> Result where S: Serializer, { diff --git a/workspace_hack/Cargo.toml b/workspace_hack/Cargo.toml index 33bdc25785..0ffeeead18 100644 --- a/workspace_hack/Cargo.toml +++ b/workspace_hack/Cargo.toml @@ -91,7 +91,8 @@ tokio-stream = { version = "0.1", features = ["net"] } tokio-util = { version = "0.7", features = ["codec", "compat", "io", "rt"] } toml_edit = { version = "0.22", features = ["serde"] } tonic = { version = "0.12", features = ["tls-roots"] } -tower = { version = "0.4", default-features = false, features = ["balance", "buffer", "limit", "log", "util"] } +tower-9fbad63c4bcf4a8f = { package = "tower", version = "0.4", default-features = false, features = ["balance", "buffer", "limit", "util"] } +tower-d8f496e17d97b5cb = { package = "tower", version = "0.5", default-features = false, features = ["log", "make", "util"] } tracing = { version = "0.1", features = ["log"] } tracing-core = { version = "0.1" } url = { version = "2", features = ["serde"] } From 6149ac88343c455fc99b6fb8e49c5d8e92e7dec2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arpad=20M=C3=BCller?= Date: Thu, 9 Jan 2025 21:41:49 +0100 Subject: [PATCH 022/111] Handle race between auto-offload and unarchival (#10305) ## Problem Auto-offloading as requested by the compaction task is racy with unarchival, in that the compaction task might attempt to offload an unarchived timeline. By that point it will already have set the timeline to the `Stopping` state however, which makes it unusable for any purpose. For example: 1. compaction task decides to offload timeline 2. timeline gets unarchived 3. `offload_timeline` gets called by compaction task * sets timeline's state to `Stopping` * realizes that the timeline can't be unarchived, errors out 6. endpoint can't be started as the timeline is `Stopping` and thus 'can't be found'. A future iteration of the compaction task can't "heal" this state either as the timeline will still not be archived, same goes for other automatic stuff. The only way to heal this is a tenant detach+attach, or alternatively a pageserver restart. Furthermore, the compaction task is especially amenable for such races as it first stores `can_offload` into a variable, figures out whether compaction is needed (which takes some time), and only then does it attempt an offload operation: the time difference between "check" and "use" is non-trivially small. To make it even worse, we start the compaction task right after attach of a tenant, and it is a common pattern by pageserver users to attach a tenant to then immediately unarchive a timeline, so that an endpoint can be started. ## Solutions not adopted The simplest solution is to move the `can_offload` check to right before attempting of the offload. But this is not a good solution, as no lock is held between that check and timeline shutdown. So races would still be possible, just become less likely. I explored using the timeline state for this, as in adding an additional enum variant. But `Timeline::set_state` is racy (#10297). ## Adopted solution We use the lock on the timeline's upload queue as an arbiter: either unarchival gets to it first and sours the state for auto-offloading, or auto-offloading shuts it down, which stops any parallel unarchival in its tracks. The key part is not releasing the upload queue's lock between the check whether the timeline is archived or not, and shutting it down (the actual implementation only sets `shutting_down` but it has the same effect on `initialized_mut()` as a full shutdown). The rest of the patch is stuff that follows from this. We also move the part where we set the state to `Stopping` to after that arbiter has decided the fate of the timeline. For deletions, we do keep it inside `DeleteTimelineFlow::prepare` however, so that it is called with all of the the timelines locks held that the function allocates (timelines lock most importantly). This is only a precautionary measure however, as I didn't want to analyze deletion related code for possible races. ## Future changes It might make sense to move `can_offload` to right before the offload attempt. Maybe some other properties might have changed as well. Although this will not be perfect either as no lock is held. I want to keep it out of this change to emphasize that this move wasn't the main reason we are race free now. Fixes #10220 --- pageserver/src/tenant.rs | 15 ++- .../src/tenant/remote_timeline_client.rs | 58 ++++++++++ pageserver/src/tenant/timeline/delete.rs | 11 +- pageserver/src/tenant/timeline/offload.rs | 32 +++--- test_runner/regress/test_timeline_archive.py | 100 ++++++++++++++++++ 5 files changed, 195 insertions(+), 21 deletions(-) diff --git a/pageserver/src/tenant.rs b/pageserver/src/tenant.rs index e3dab2fc1d..8e61d09de7 100644 --- a/pageserver/src/tenant.rs +++ b/pageserver/src/tenant.rs @@ -48,6 +48,7 @@ use timeline::compaction::GcCompactJob; use timeline::compaction::ScheduledCompactionTask; use timeline::import_pgdata; use timeline::offload::offload_timeline; +use timeline::offload::OffloadError; use timeline::CompactFlags; use timeline::CompactOptions; use timeline::CompactionError; @@ -2039,7 +2040,7 @@ impl Tenant { ) -> Result, TimelineArchivalError> { info!("unoffloading timeline"); - // We activate the timeline below manually, so this must be called on an active timeline. + // We activate the timeline below manually, so this must be called on an active tenant. // We expect callers of this function to ensure this. match self.current_state() { TenantState::Activating { .. } @@ -3100,9 +3101,17 @@ impl Tenant { }; has_pending_task |= pending_task_left.unwrap_or(false); if pending_task_left == Some(false) && *can_offload { - offload_timeline(self, timeline) + pausable_failpoint!("before-timeline-auto-offload"); + match offload_timeline(self, timeline) .instrument(info_span!("offload_timeline", %timeline_id)) - .await?; + .await + { + Err(OffloadError::NotArchived) => { + // Ignore this, we likely raced with unarchival + Ok(()) + } + other => other, + }?; } } diff --git a/pageserver/src/tenant/remote_timeline_client.rs b/pageserver/src/tenant/remote_timeline_client.rs index b27ac3e933..813111245d 100644 --- a/pageserver/src/tenant/remote_timeline_client.rs +++ b/pageserver/src/tenant/remote_timeline_client.rs @@ -304,6 +304,15 @@ pub enum WaitCompletionError { #[derive(Debug, thiserror::Error)] #[error("Upload queue either in unexpected state or hasn't downloaded manifest yet")] pub struct UploadQueueNotReadyError; + +#[derive(Debug, thiserror::Error)] +pub enum ShutdownIfArchivedError { + #[error(transparent)] + NotInitialized(NotInitialized), + #[error("timeline is not archived")] + NotArchived, +} + /// Behavioral modes that enable seamless live migration. /// /// See docs/rfcs/028-pageserver-migration.md to understand how these fit in. @@ -816,6 +825,55 @@ impl RemoteTimelineClient { Ok(need_wait) } + /// Shuts the timeline client down, but only if the timeline is archived. + /// + /// This function and [`Self::schedule_index_upload_for_timeline_archival_state`] use the + /// same lock to prevent races between unarchival and offloading: unarchival requires the + /// upload queue to be initialized, and leaves behind an upload queue where either dirty + /// or clean has archived_at of `None`. offloading leaves behind an uninitialized upload + /// queue. + pub(crate) async fn shutdown_if_archived( + self: &Arc, + ) -> Result<(), ShutdownIfArchivedError> { + { + let mut guard = self.upload_queue.lock().unwrap(); + let upload_queue = guard + .initialized_mut() + .map_err(ShutdownIfArchivedError::NotInitialized)?; + + match ( + upload_queue.dirty.archived_at.is_none(), + upload_queue.clean.0.archived_at.is_none(), + ) { + // The expected case: the timeline is archived and we don't want to unarchive + (false, false) => {} + (true, false) => { + tracing::info!("can't shut down timeline: timeline slated for unarchival"); + return Err(ShutdownIfArchivedError::NotArchived); + } + (dirty_archived, true) => { + tracing::info!(%dirty_archived, "can't shut down timeline: timeline not archived in remote storage"); + return Err(ShutdownIfArchivedError::NotArchived); + } + } + + // Set the shutting_down flag while the guard from the archival check is held. + // This prevents a race with unarchival, as initialized_mut will not return + // an upload queue from this point. + // Also launch the queued tasks like shutdown() does. + if !upload_queue.shutting_down { + upload_queue.shutting_down = true; + upload_queue.queued_operations.push_back(UploadOp::Shutdown); + // this operation is not counted similar to Barrier + self.launch_queued_tasks(upload_queue); + } + } + + self.shutdown().await; + + Ok(()) + } + /// Launch an index-file upload operation in the background, setting `import_pgdata` field. pub(crate) fn schedule_index_upload_for_import_pgdata_state_update( self: &Arc, diff --git a/pageserver/src/tenant/timeline/delete.rs b/pageserver/src/tenant/timeline/delete.rs index 47a93b19d2..ae44af3fad 100644 --- a/pageserver/src/tenant/timeline/delete.rs +++ b/pageserver/src/tenant/timeline/delete.rs @@ -194,7 +194,9 @@ impl DeleteTimelineFlow { super::debug_assert_current_span_has_tenant_and_timeline_id(); let allow_offloaded_children = false; - let (timeline, mut guard) = Self::prepare(tenant, timeline_id, allow_offloaded_children)?; + let set_stopping = true; + let (timeline, mut guard) = + Self::prepare(tenant, timeline_id, allow_offloaded_children, set_stopping)?; guard.mark_in_progress()?; @@ -334,6 +336,7 @@ impl DeleteTimelineFlow { tenant: &Tenant, timeline_id: TimelineId, allow_offloaded_children: bool, + set_stopping: bool, ) -> Result<(TimelineOrOffloaded, DeletionGuard), DeleteTimelineError> { // Note the interaction between this guard and deletion guard. // Here we attempt to lock deletion guard when we're holding a lock on timelines. @@ -389,8 +392,10 @@ impl DeleteTimelineFlow { } }; - if let TimelineOrOffloaded::Timeline(timeline) = &timeline { - timeline.set_state(TimelineState::Stopping); + if set_stopping { + if let TimelineOrOffloaded::Timeline(timeline) = &timeline { + timeline.set_state(TimelineState::Stopping); + } } Ok((timeline, delete_lock_guard)) diff --git a/pageserver/src/tenant/timeline/offload.rs b/pageserver/src/tenant/timeline/offload.rs index 15628a9645..6c6b19e8b1 100644 --- a/pageserver/src/tenant/timeline/offload.rs +++ b/pageserver/src/tenant/timeline/offload.rs @@ -1,10 +1,11 @@ use std::sync::Arc; -use pageserver_api::models::TenantState; +use pageserver_api::models::{TenantState, TimelineState}; use super::delete::{delete_local_timeline_directory, DeleteTimelineFlow, DeletionGuard}; use super::Timeline; use crate::span::debug_assert_current_span_has_tenant_and_timeline_id; +use crate::tenant::remote_timeline_client::ShutdownIfArchivedError; use crate::tenant::{OffloadedTimeline, Tenant, TenantManifestError, TimelineOrOffloaded}; #[derive(thiserror::Error, Debug)] @@ -36,28 +37,29 @@ pub(crate) async fn offload_timeline( tracing::info!("offloading archived timeline"); let allow_offloaded_children = true; - let (timeline, guard) = - DeleteTimelineFlow::prepare(tenant, timeline.timeline_id, allow_offloaded_children) - .map_err(|e| OffloadError::Other(anyhow::anyhow!(e)))?; + let set_stopping = false; + let (timeline, guard) = DeleteTimelineFlow::prepare( + tenant, + timeline.timeline_id, + allow_offloaded_children, + set_stopping, + ) + .map_err(|e| OffloadError::Other(anyhow::anyhow!(e)))?; let TimelineOrOffloaded::Timeline(timeline) = timeline else { tracing::error!("timeline already offloaded, but given timeline object"); return Ok(()); }; - let is_archived = timeline.is_archived(); - match is_archived { - Some(true) => (), - Some(false) => { - tracing::warn!("tried offloading a non-archived timeline"); - return Err(OffloadError::NotArchived); - } - None => { - // This is legal: calls to this function can race with the timeline shutting down - tracing::info!("tried offloading a timeline whose remote storage is not initialized"); - return Err(OffloadError::Cancelled); + match timeline.remote_client.shutdown_if_archived().await { + Ok(()) => {} + Err(ShutdownIfArchivedError::NotInitialized(_)) => { + // Either the timeline is being deleted, the operation is being retried, or we are shutting down. + // Don't return cancelled here to keep it idempotent. } + Err(ShutdownIfArchivedError::NotArchived) => return Err(OffloadError::NotArchived), } + timeline.set_state(TimelineState::Stopping); // Now that the Timeline is in Stopping state, request all the related tasks to shut down. timeline.shutdown(super::ShutdownMode::Reload).await; diff --git a/test_runner/regress/test_timeline_archive.py b/test_runner/regress/test_timeline_archive.py index 9b3a48add9..bec8270582 100644 --- a/test_runner/regress/test_timeline_archive.py +++ b/test_runner/regress/test_timeline_archive.py @@ -959,3 +959,103 @@ def test_timeline_offload_generations(neon_env_builder: NeonEnvBuilder): assert gc_summary["remote_storage_errors"] == 0 assert gc_summary["indices_deleted"] > 0 assert gc_summary["tenant_manifests_deleted"] > 0 + + +@pytest.mark.parametrize("end_with_offloaded", [False, True]) +def test_timeline_offload_race_unarchive( + neon_env_builder: NeonEnvBuilder, end_with_offloaded: bool +): + """ + Ensure that unarchive and timeline offload don't race each other + """ + # Regression test for issue https://github.com/neondatabase/neon/issues/10220 + # (automatic) timeline offloading defaults to false for now + neon_env_builder.pageserver_config_override = "timeline_offloading = true" + + failpoint = "before-timeline-auto-offload" + + env = neon_env_builder.init_start() + ps_http = env.pageserver.http_client() + + # Turn off gc and compaction loops: we want to issue them manually for better reliability + tenant_id, initial_timeline_id = env.create_tenant( + conf={ + "gc_period": "0s", + "compaction_period": "1s", + } + ) + + # Create a branch + leaf_timeline_id = env.create_branch("test_ancestor_branch_archive", tenant_id) + + # write some stuff to the leaf + with env.endpoints.create_start( + "test_ancestor_branch_archive", tenant_id=tenant_id + ) as endpoint: + endpoint.safe_psql_many( + [ + "CREATE TABLE foo(key serial primary key, t text default 'data_content')", + "INSERT INTO foo SELECT FROM generate_series(1,1000)", + ] + ) + sum = endpoint.safe_psql("SELECT sum(key) from foo where key % 7 = 1") + + ps_http.configure_failpoints((failpoint, "pause")) + + ps_http.timeline_archival_config( + tenant_id, + leaf_timeline_id, + state=TimelineArchivalState.ARCHIVED, + ) + leaf_detail = ps_http.timeline_detail( + tenant_id, + leaf_timeline_id, + ) + assert leaf_detail["is_archived"] is True + + # The actual race: get the compaction task to right before + # offloading the timeline and attempt to unarchive it + wait_until(lambda: env.pageserver.assert_log_contains(f"at failpoint {failpoint}")) + + # This unarchival should go through + ps_http.timeline_archival_config( + tenant_id, + leaf_timeline_id, + state=TimelineArchivalState.UNARCHIVED, + ) + + def timeline_offloaded_api(timeline_id: TimelineId) -> bool: + # TODO add a proper API to check if a timeline has been offloaded or not + return not any( + timeline["timeline_id"] == str(timeline_id) + for timeline in ps_http.timeline_list(tenant_id=tenant_id) + ) + + def leaf_offloaded(): + assert timeline_offloaded_api(leaf_timeline_id) + + # Ensure that we've hit the failed offload attempt + ps_http.configure_failpoints((failpoint, "off")) + wait_until( + lambda: env.pageserver.assert_log_contains( + f".*compaction_loop.*offload_timeline.*{leaf_timeline_id}.*can't shut down timeline.*" + ) + ) + + with env.endpoints.create_start( + "test_ancestor_branch_archive", tenant_id=tenant_id + ) as endpoint: + sum_again = endpoint.safe_psql("SELECT sum(key) from foo where key % 7 = 1") + assert sum == sum_again + + if end_with_offloaded: + # Ensure that offloading still works after all of this + ps_http.timeline_archival_config( + tenant_id, + leaf_timeline_id, + state=TimelineArchivalState.ARCHIVED, + ) + wait_until(leaf_offloaded) + else: + # Test that deletion of leaf timeline works + ps_http.timeline_delete(tenant_id, leaf_timeline_id) From b6205af4a5542935ca97ccd89d205f95bb75ce27 Mon Sep 17 00:00:00 2001 From: Folke Behrens Date: Fri, 10 Jan 2025 09:48:03 +0100 Subject: [PATCH 023/111] Update tracing/otel crates (#10311) Update the tracing(-x) and opentelemetry(-x) crates. Some breaking changes require updating our code: * Initialization is done via builders now https://github.com/open-telemetry/opentelemetry-rust/blob/main/opentelemetry-otlp/CHANGELOG.md#0270 * Errors from OTel SDK are logged via tracing crate as well. https://github.com/open-telemetry/opentelemetry-rust/blob/main/opentelemetry/CHANGELOG.md#0270 --- Cargo.lock | 36 ++++++++++++++-------------- Cargo.toml | 12 +++++----- compute_tools/src/bin/compute_ctl.rs | 5 ---- libs/tracing-utils/src/lib.rs | 24 +++++++++---------- 4 files changed, 35 insertions(+), 42 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 44143fa0da..87a54c78ec 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3693,23 +3693,23 @@ checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" [[package]] name = "opentelemetry" -version = "0.26.0" +version = "0.27.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "570074cc999d1a58184080966e5bd3bf3a9a4af650c3b05047c2621e7405cd17" +checksum = "ab70038c28ed37b97d8ed414b6429d343a8bbf44c9f79ec854f3a643029ba6d7" dependencies = [ "futures-core", "futures-sink", "js-sys", - "once_cell", "pin-project-lite", "thiserror", + "tracing", ] [[package]] name = "opentelemetry-http" -version = "0.26.0" +version = "0.27.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6351496aeaa49d7c267fb480678d85d1cd30c5edb20b497c48c56f62a8c14b99" +checksum = "10a8a7f5f6ba7c1b286c2fbca0454eaba116f63bbe69ed250b642d36fbb04d80" dependencies = [ "async-trait", "bytes", @@ -3720,9 +3720,9 @@ dependencies = [ [[package]] name = "opentelemetry-otlp" -version = "0.26.0" +version = "0.27.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29e1f9c8b032d4f635c730c0efcf731d5e2530ea13fa8bef7939ddc8420696bd" +checksum = "91cf61a1868dacc576bf2b2a1c3e9ab150af7272909e80085c3173384fe11f76" dependencies = [ "async-trait", "futures-core", @@ -3738,9 +3738,9 @@ dependencies = [ [[package]] name = "opentelemetry-proto" -version = "0.26.1" +version = "0.27.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c9d3968ce3aefdcca5c27e3c4ea4391b37547726a70893aab52d3de95d5f8b34" +checksum = "a6e05acbfada5ec79023c85368af14abd0b307c015e9064d249b2a950ef459a6" dependencies = [ "opentelemetry", "opentelemetry_sdk", @@ -3750,22 +3750,21 @@ dependencies = [ [[package]] name = "opentelemetry-semantic-conventions" -version = "0.26.0" +version = "0.27.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db945c1eaea8ac6a9677185357480d215bb6999faa9f691d0c4d4d641eab7a09" +checksum = "bc1b6902ff63b32ef6c489e8048c5e253e2e4a803ea3ea7e783914536eb15c52" [[package]] name = "opentelemetry_sdk" -version = "0.26.0" +version = "0.27.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2c627d9f4c9cdc1f21a29ee4bfbd6028fcb8bcf2a857b43f3abdf72c9c862f3" +checksum = "231e9d6ceef9b0b2546ddf52335785ce41252bc7474ee8ba05bfad277be13ab8" dependencies = [ "async-trait", "futures-channel", "futures-executor", "futures-util", "glob", - "once_cell", "opentelemetry", "percent-encoding", "rand 0.8.5", @@ -3773,6 +3772,7 @@ dependencies = [ "thiserror", "tokio", "tokio-stream", + "tracing", ] [[package]] @@ -5181,9 +5181,9 @@ dependencies = [ [[package]] name = "reqwest-tracing" -version = "0.5.4" +version = "0.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff82cf5730a1311fb9413b0bc2b8e743e0157cd73f010ab4ec374a923873b6a2" +checksum = "73e6153390585f6961341b50e5a1931d6be6dee4292283635903c26ef9d980d2" dependencies = [ "anyhow", "async-trait", @@ -7048,9 +7048,9 @@ dependencies = [ [[package]] name = "tracing-opentelemetry" -version = "0.27.0" +version = "0.28.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc58af5d3f6c5811462cabb3289aec0093f7338e367e5a33d28c0433b3c7360b" +checksum = "97a971f6058498b5c0f1affa23e7ea202057a7301dbff68e968b2d578bcbd053" dependencies = [ "js-sys", "once_cell", diff --git a/Cargo.toml b/Cargo.toml index 39898e1c8d..55bb9bfe11 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -126,10 +126,10 @@ notify = "6.0.0" num_cpus = "1.15" num-traits = "0.2.15" once_cell = "1.13" -opentelemetry = "0.26" -opentelemetry_sdk = "0.26" -opentelemetry-otlp = { version = "0.26", default-features=false, features = ["http-proto", "trace", "http", "reqwest-client"] } -opentelemetry-semantic-conventions = "0.26" +opentelemetry = "0.27" +opentelemetry_sdk = "0.27" +opentelemetry-otlp = { version = "0.27", default-features = false, features = ["http-proto", "trace", "http", "reqwest-client"] } +opentelemetry-semantic-conventions = "0.27" parking_lot = "0.12" parquet = { version = "53", default-features = false, features = ["zstd"] } parquet_derive = "53" @@ -143,7 +143,7 @@ rand = "0.8" redis = { version = "0.25.2", features = ["tokio-rustls-comp", "keep-alive"] } regex = "1.10.2" reqwest = { version = "0.12", default-features = false, features = ["rustls-tls"] } -reqwest-tracing = { version = "0.5", features = ["opentelemetry_0_26"] } +reqwest-tracing = { version = "0.5", features = ["opentelemetry_0_27"] } reqwest-middleware = "0.4" reqwest-retry = "0.7" routerify = "3" @@ -192,7 +192,7 @@ tower-http = { version = "0.6.2", features = ["request-id", "trace"] } tower-service = "0.3.3" tracing = "0.1" tracing-error = "0.2" -tracing-opentelemetry = "0.27" +tracing-opentelemetry = "0.28" tracing-subscriber = { version = "0.3", default-features = false, features = ["smallvec", "fmt", "tracing-log", "std", "env-filter", "json"] } try-lock = "0.2.5" twox-hash = { version = "1.6.3", default-features = false } diff --git a/compute_tools/src/bin/compute_ctl.rs b/compute_tools/src/bin/compute_ctl.rs index 04432ad0f3..b98cf706d3 100644 --- a/compute_tools/src/bin/compute_ctl.rs +++ b/compute_tools/src/bin/compute_ctl.rs @@ -111,11 +111,6 @@ fn main() -> Result<()> { fn init() -> Result<(String, clap::ArgMatches)> { init_tracing_and_logging(DEFAULT_LOG_LEVEL)?; - opentelemetry::global::set_error_handler(|err| { - tracing::info!("OpenTelemetry error: {err}"); - }) - .expect("global error handler lock poisoned"); - let mut signals = Signals::new([SIGINT, SIGTERM, SIGQUIT])?; thread::spawn(move || { for sig in signals.forever() { diff --git a/libs/tracing-utils/src/lib.rs b/libs/tracing-utils/src/lib.rs index c4aad53cdb..818d759eac 100644 --- a/libs/tracing-utils/src/lib.rs +++ b/libs/tracing-utils/src/lib.rs @@ -38,7 +38,6 @@ pub mod http; use opentelemetry::trace::TracerProvider; use opentelemetry::KeyValue; -use opentelemetry_sdk::Resource; use tracing::Subscriber; use tracing_subscriber::registry::LookupSpan; use tracing_subscriber::Layer; @@ -121,7 +120,10 @@ where S: Subscriber + for<'span> LookupSpan<'span>, { // Sets up exporter from the OTEL_EXPORTER_* environment variables. - let exporter = opentelemetry_otlp::new_exporter().http(); + let exporter = opentelemetry_otlp::SpanExporter::builder() + .with_http() + .build() + .expect("could not initialize opentelemetry exporter"); // TODO: opentelemetry::global::set_error_handler() with custom handler that // bypasses default tracing layers, but logs regular looking log @@ -132,17 +134,13 @@ where opentelemetry_sdk::propagation::TraceContextPropagator::new(), ); - let tracer = opentelemetry_otlp::new_pipeline() - .tracing() - .with_exporter(exporter) - .with_trace_config(opentelemetry_sdk::trace::Config::default().with_resource( - Resource::new(vec![KeyValue::new( - opentelemetry_semantic_conventions::resource::SERVICE_NAME, - service_name, - )]), - )) - .install_batch(opentelemetry_sdk::runtime::Tokio) - .expect("could not initialize opentelemetry exporter") + let tracer = opentelemetry_sdk::trace::TracerProvider::builder() + .with_batch_exporter(exporter, opentelemetry_sdk::runtime::Tokio) + .with_resource(opentelemetry_sdk::Resource::new(vec![KeyValue::new( + opentelemetry_semantic_conventions::resource::SERVICE_NAME, + service_name, + )])) + .build() .tracer("global"); tracing_opentelemetry::layer().with_tracer(tracer) From 77660f3d88d27c3a7414eee611a6192900f98fe9 Mon Sep 17 00:00:00 2001 From: Folke Behrens Date: Fri, 10 Jan 2025 10:12:31 +0100 Subject: [PATCH 024/111] proxy: Fix parsing of UnknownTopic with payload (#10339) ## Problem When the proxy receives a `Notification` with an unknown topic it's supposed to use the `UnknownTopic` unit variant. Unfortunately, in adjacently tagged enums serde will not simply ignore the configured content if found and try to deserialize a map/object instead. ## Summary of changes * Use a custom deserialize function to ignore variant content. * Add a little unit test covering both cases. --- proxy/src/redis/notifications.rs | 41 +++++++++++++++++++++++++++++++- 1 file changed, 40 insertions(+), 1 deletion(-) diff --git a/proxy/src/redis/notifications.rs b/proxy/src/redis/notifications.rs index bf9d61ded3..63cdf6176c 100644 --- a/proxy/src/redis/notifications.rs +++ b/proxy/src/redis/notifications.rs @@ -74,7 +74,11 @@ pub(crate) enum Notification { #[serde(rename = "/cancel_session")] Cancel(CancelSession), - #[serde(other, skip_serializing)] + #[serde( + other, + deserialize_with = "deserialize_unknown_topic", + skip_serializing + )] UnknownTopic, } @@ -123,6 +127,15 @@ where serde_json::from_str(&s).map_err(::custom) } +// https://github.com/serde-rs/serde/issues/1714 +fn deserialize_unknown_topic<'de, D>(deserializer: D) -> Result<(), D::Error> +where + D: serde::Deserializer<'de>, +{ + deserializer.deserialize_any(serde::de::IgnoredAny)?; + Ok(()) +} + struct MessageHandler { cache: Arc, cancellation_handler: Arc>, @@ -458,4 +471,30 @@ mod tests { Ok(()) } + + #[test] + fn parse_unknown_topic() -> anyhow::Result<()> { + let with_data = json!({ + "type": "message", + "topic": "/doesnotexist", + "data": { + "payload": "ignored" + }, + "extra_fields": "something" + }) + .to_string(); + let result: Notification = serde_json::from_str(&with_data)?; + assert_eq!(result, Notification::UnknownTopic); + + let without_data = json!({ + "type": "message", + "topic": "/doesnotexist", + "extra_fields": "something" + }) + .to_string(); + let result: Notification = serde_json::from_str(&without_data)?; + assert_eq!(result, Notification::UnknownTopic); + + Ok(()) + } } From 735c66dc65f1163591a2745934f4be766072c88c Mon Sep 17 00:00:00 2001 From: Conrad Ludgate Date: Fri, 10 Jan 2025 09:36:51 +0000 Subject: [PATCH 025/111] fix(proxy): propagate the existing ComputeUserInfo to connect for cancellation (#10322) ## Problem We were incorrectly constructing the ComputeUserInfo, used for cancellation checks, based on the return parameters from postgres. This didn't contain the correct info. ## Summary of changes Propagate down the existing ComputeUserInfo. --- proxy/src/auth/backend/console_redirect.rs | 30 +++++++++++++++++----- proxy/src/compute.rs | 27 ++----------------- proxy/src/console_redirect_proxy.rs | 5 ++-- proxy/src/control_plane/mod.rs | 5 +++- proxy/src/proxy/connect_compute.rs | 6 +++-- proxy/src/proxy/mod.rs | 13 ++++++---- 6 files changed, 45 insertions(+), 41 deletions(-) diff --git a/proxy/src/auth/backend/console_redirect.rs b/proxy/src/auth/backend/console_redirect.rs index dbfda588cc..1cbf91d3ae 100644 --- a/proxy/src/auth/backend/console_redirect.rs +++ b/proxy/src/auth/backend/console_redirect.rs @@ -1,7 +1,8 @@ +use std::fmt; + use async_trait::async_trait; use postgres_client::config::SslMode; use pq_proto::BeMessage as Be; -use std::fmt; use thiserror::Error; use tokio::io::{AsyncRead, AsyncWrite}; use tracing::{info, info_span}; @@ -12,10 +13,13 @@ use crate::auth::IpPattern; use crate::cache::Cached; use crate::config::AuthenticationConfig; use crate::context::RequestContext; -use crate::control_plane::{self, client::cplane_proxy_v1, CachedNodeInfo, NodeInfo}; +use crate::control_plane::client::cplane_proxy_v1; +use crate::control_plane::{self, CachedNodeInfo, NodeInfo}; use crate::error::{ReportableError, UserFacingError}; use crate::proxy::connect_compute::ComputeConnectBackend; +use crate::proxy::NeonOptions; use crate::stream::PqStream; +use crate::types::RoleName; use crate::{auth, compute, waiters}; #[derive(Debug, Error)] @@ -105,10 +109,16 @@ impl ConsoleRedirectBackend { ctx: &RequestContext, auth_config: &'static AuthenticationConfig, client: &mut PqStream, - ) -> auth::Result<(ConsoleRedirectNodeInfo, Option>)> { + ) -> auth::Result<( + ConsoleRedirectNodeInfo, + ComputeUserInfo, + Option>, + )> { authenticate(ctx, auth_config, &self.console_uri, client) .await - .map(|(node_info, ip_allowlist)| (ConsoleRedirectNodeInfo(node_info), ip_allowlist)) + .map(|(node_info, user_info, ip_allowlist)| { + (ConsoleRedirectNodeInfo(node_info), user_info, ip_allowlist) + }) } } @@ -133,7 +143,7 @@ async fn authenticate( auth_config: &'static AuthenticationConfig, link_uri: &reqwest::Url, client: &mut PqStream, -) -> auth::Result<(NodeInfo, Option>)> { +) -> auth::Result<(NodeInfo, ComputeUserInfo, Option>)> { ctx.set_auth_method(crate::context::AuthMethod::ConsoleRedirect); // registering waiter can fail if we get unlucky with rng. @@ -188,8 +198,15 @@ async fn authenticate( let mut config = compute::ConnCfg::new(db_info.host.to_string(), db_info.port); config.dbname(&db_info.dbname).user(&db_info.user); + let user: RoleName = db_info.user.into(); + let user_info = ComputeUserInfo { + endpoint: db_info.aux.endpoint_id.as_str().into(), + user: user.clone(), + options: NeonOptions::default(), + }; + ctx.set_dbname(db_info.dbname.into()); - ctx.set_user(db_info.user.into()); + ctx.set_user(user); ctx.set_project(db_info.aux.clone()); info!("woken up a compute node"); @@ -212,6 +229,7 @@ async fn authenticate( config, aux: db_info.aux, }, + user_info, db_info.allowed_ips, )) } diff --git a/proxy/src/compute.rs b/proxy/src/compute.rs index 788bd63fee..aff796bbab 100644 --- a/proxy/src/compute.rs +++ b/proxy/src/compute.rs @@ -24,10 +24,8 @@ use crate::control_plane::messages::MetricsAuxInfo; use crate::error::{ReportableError, UserFacingError}; use crate::metrics::{Metrics, NumDbConnectionsGuard}; use crate::proxy::neon_option; -use crate::proxy::NeonOptions; use crate::tls::postgres_rustls::MakeRustlsConnect; use crate::types::Host; -use crate::types::{EndpointId, RoleName}; pub const COULD_NOT_CONNECT: &str = "Couldn't connect to compute node"; @@ -253,6 +251,7 @@ impl ConnCfg { ctx: &RequestContext, aux: MetricsAuxInfo, config: &ComputeConfig, + user_info: ComputeUserInfo, ) -> Result { let pause = ctx.latency_timer_pause(crate::metrics::Waiting::Compute); let (socket_addr, stream, host) = self.connect_raw(config.timeout).await?; @@ -287,28 +286,6 @@ impl ConnCfg { self.0.get_ssl_mode() ); - let compute_info = match parameters.get("user") { - Some(user) => { - match parameters.get("database") { - Some(database) => { - ComputeUserInfo { - user: RoleName::from(user), - options: NeonOptions::default(), // just a shim, we don't need options - endpoint: EndpointId::from(database), - } - } - None => { - warn!("compute node didn't return database name"); - ComputeUserInfo::default() - } - } - } - None => { - warn!("compute node didn't return user name"); - ComputeUserInfo::default() - } - }; - // NB: CancelToken is supposed to hold socket_addr, but we use connect_raw. // Yet another reason to rework the connection establishing code. let cancel_closure = CancelClosure::new( @@ -321,7 +298,7 @@ impl ConnCfg { }, vec![], // TODO: deprecated, will be removed host.to_string(), - compute_info, + user_info, ); let connection = PostgresConnection { diff --git a/proxy/src/console_redirect_proxy.rs b/proxy/src/console_redirect_proxy.rs index 846f55f9e1..0c6755063f 100644 --- a/proxy/src/console_redirect_proxy.rs +++ b/proxy/src/console_redirect_proxy.rs @@ -195,7 +195,7 @@ pub(crate) async fn handle_client( ctx.set_db_options(params.clone()); - let (user_info, ip_allowlist) = match backend + let (node_info, user_info, ip_allowlist) = match backend .authenticate(ctx, &config.authentication_config, &mut stream) .await { @@ -208,11 +208,12 @@ pub(crate) async fn handle_client( let mut node = connect_to_compute( ctx, &TcpMechanism { + user_info, params_compat: true, params: ¶ms, locks: &config.connect_compute_locks, }, - &user_info, + &node_info, config.wake_compute_retry_config, &config.connect_to_compute, ) diff --git a/proxy/src/control_plane/mod.rs b/proxy/src/control_plane/mod.rs index c65041df0e..1dca26d686 100644 --- a/proxy/src/control_plane/mod.rs +++ b/proxy/src/control_plane/mod.rs @@ -74,8 +74,11 @@ impl NodeInfo { &self, ctx: &RequestContext, config: &ComputeConfig, + user_info: ComputeUserInfo, ) -> Result { - self.config.connect(ctx, self.aux.clone(), config).await + self.config + .connect(ctx, self.aux.clone(), config, user_info) + .await } pub(crate) fn reuse_settings(&mut self, other: Self) { diff --git a/proxy/src/proxy/connect_compute.rs b/proxy/src/proxy/connect_compute.rs index 8a80494860..dd145e6bb2 100644 --- a/proxy/src/proxy/connect_compute.rs +++ b/proxy/src/proxy/connect_compute.rs @@ -4,7 +4,7 @@ use tokio::time; use tracing::{debug, info, warn}; use super::retry::ShouldRetryWakeCompute; -use crate::auth::backend::ComputeCredentialKeys; +use crate::auth::backend::{ComputeCredentialKeys, ComputeUserInfo}; use crate::compute::{self, PostgresConnection, COULD_NOT_CONNECT}; use crate::config::{ComputeConfig, RetryConfig}; use crate::context::RequestContext; @@ -71,6 +71,8 @@ pub(crate) struct TcpMechanism<'a> { /// connect_to_compute concurrency lock pub(crate) locks: &'static ApiLocks, + + pub(crate) user_info: ComputeUserInfo, } #[async_trait] @@ -88,7 +90,7 @@ impl ConnectMechanism for TcpMechanism<'_> { ) -> Result { let host = node_info.config.get_host(); let permit = self.locks.get_permit(&host).await?; - permit.release_result(node_info.connect(ctx, config).await) + permit.release_result(node_info.connect(ctx, config, self.user_info.clone()).await) } fn update_connect_config(&self, config: &mut compute::ConnCfg) { diff --git a/proxy/src/proxy/mod.rs b/proxy/src/proxy/mod.rs index 1f7dba2f9a..63f93f0a91 100644 --- a/proxy/src/proxy/mod.rs +++ b/proxy/src/proxy/mod.rs @@ -332,16 +332,19 @@ pub(crate) async fn handle_client( } }; - let params_compat = match &user_info { - auth::Backend::ControlPlane(_, info) => { - info.info.options.get(NeonOptions::PARAMS_COMPAT).is_some() - } - auth::Backend::Local(_) => false, + let compute_user_info = match &user_info { + auth::Backend::ControlPlane(_, info) => &info.info, + auth::Backend::Local(_) => unreachable!("local proxy does not run tcp proxy service"), }; + let params_compat = compute_user_info + .options + .get(NeonOptions::PARAMS_COMPAT) + .is_some(); let mut node = connect_to_compute( ctx, &TcpMechanism { + user_info: compute_user_info.clone(), params_compat, params: ¶ms, locks: &config.connect_compute_locks, From db00eb41a1f27ce5d122fbdfbf3f958e5df9b79f Mon Sep 17 00:00:00 2001 From: Christian Schwarz Date: Fri, 10 Jan 2025 12:06:03 +0100 Subject: [PATCH 026/111] fix(spsc_fold): potentially missing wakeup when send()ing in state `SenderWaitsForReceiverToConsume` (#10318) # Problem Before this PR, there were cases where send() in state SenderWaitsForReceiverToConsume would never be woken up by the receiver, because it never registered with `wake_sender`. Example Scenario 1: we stop polling a send() future A that was waiting for the receiver to consume. We drop A and create a new send() future B. B would return Poll::Pending and never regsister a waker. Example Scenario 2: a send() future A transitions from HasData to SenderWaitsForReceiverToConsume. This registers the context X with `wake_sender`. But before the Receiver consumes the data, we poll A from a different context Y. The state is still SenderWaitsForReceiverToConsume, but we wouldn't register the new context with `wake_sender`. When the Receiver comes around to consume and `wake_sender.notify()`s, it wakes the old context X instead of Y. # Fix Register the waker in the case where we're polled in state `SenderWaitsForReceiverToConsume`. # Relation to #10309 I found this bug while investigating #10309. There was never proof that this bug here is the root cause for #10309. In the meantime we found a more probably hypothesis for the root cause than what is being fixed here. Regardless, let's walk through my thought process about how it might have been relevant: There (in page_service), Scenario 1 does not apply because we poll the send() future to completion. Scenario 2 (`tokio::join!`) also does not apply with the current `tokio::join!()` impl, because it will just poll each future every time, each with the same context. Although if we ever used something like a FuturesUnordered anywhere, that will be using a different context, so, in that case, the bug might materialize. Regarding tokio & spurious poll in general: @conradludgate is not aware of any spurious wakeup cases in current tokio, but within a `tokio::join!()`, any wake meant for one future will poll all the futures, so that can appear as a spurious wake up to the N-1 futures of the `tokio::join!()`. --- libs/utils/src/sync/spsc_fold.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/libs/utils/src/sync/spsc_fold.rs b/libs/utils/src/sync/spsc_fold.rs index b44f766ef0..7bad4f1068 100644 --- a/libs/utils/src/sync/spsc_fold.rs +++ b/libs/utils/src/sync/spsc_fold.rs @@ -96,7 +96,11 @@ impl Sender { } } State::SenderWaitsForReceiverToConsume(_data) => { - // Really, we shouldn't be polled until receiver has consumed and wakes us. + // SAFETY: send is single threaded due to `&mut self` requirement, + // therefore register is not concurrent. + unsafe { + self.state.wake_sender.register(cx.waker()); + } Poll::Pending } State::ReceiverGone => Poll::Ready(Err(SendError::ReceiverGone)), From 2b8ea1e76835d60b178ed07911935326085d4ac6 Mon Sep 17 00:00:00 2001 From: Erik Grinaker Date: Fri, 10 Jan 2025 13:14:29 +0100 Subject: [PATCH 027/111] utils: add flamegraph for heap profiles (#10223) ## Problem Unlike CPU profiles, the `/profile/heap` endpoint can't automatically generate SVG flamegraphs. This requires the user to install and use `pprof` tooling, which is unnecessary and annoying. Resolves #10203. ## Summary of changes Add `format=svg` for the `/profile/heap` route, and generate an SVG flamegraph using the `inferno` crate, similarly to what `pprof-rs` already does for CPU profiles. --- Cargo.lock | 77 ++++++++++++++++++++++++++++++--- Cargo.toml | 1 + libs/utils/Cargo.toml | 1 + libs/utils/src/http/endpoint.rs | 54 ++++++++++++++++------- libs/utils/src/pprof.rs | 59 ++++++++++++++++++++++++- 5 files changed, 169 insertions(+), 23 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 87a54c78ec..f727741883 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1653,6 +1653,20 @@ dependencies = [ "parking_lot_core 0.9.8", ] +[[package]] +name = "dashmap" +version = "6.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5041cc499144891f3790297212f32a74fb938e5136a14943f338ef9e0ae276cf" +dependencies = [ + "cfg-if", + "crossbeam-utils", + "hashbrown 0.14.5", + "lock_api", + "once_cell", + "parking_lot_core 0.9.8", +] + [[package]] name = "data-encoding" version = "2.4.0" @@ -1952,6 +1966,15 @@ dependencies = [ "syn 2.0.90", ] +[[package]] +name = "env_filter" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "186e05a59d4c50738528153b83b0b0194d3a29507dfec16eccd4b342903397d0" +dependencies = [ + "log", +] + [[package]] name = "env_logger" version = "0.10.2" @@ -1965,6 +1988,16 @@ dependencies = [ "termcolor", ] +[[package]] +name = "env_logger" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c012a26a7f605efc424dd53697843a72be7dc86ad2d01f7814337794a12231d" +dependencies = [ + "env_filter", + "log", +] + [[package]] name = "equator" version = "0.2.2" @@ -2948,6 +2981,28 @@ dependencies = [ "str_stack", ] +[[package]] +name = "inferno" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75a5d75fee4d36809e6b021e4b96b686e763d365ffdb03af2bd00786353f84fe" +dependencies = [ + "ahash", + "clap", + "crossbeam-channel", + "crossbeam-utils", + "dashmap 6.1.0", + "env_logger 0.11.2", + "indexmap 2.0.1", + "itoa", + "log", + "num-format", + "once_cell", + "quick-xml 0.37.1", + "rgb", + "str_stack", +] + [[package]] name = "inotify" version = "0.9.6" @@ -3155,7 +3210,7 @@ version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4644821e1c3d7a560fe13d842d13f587c07348a1a05d3a797152d41c90c56df2" dependencies = [ - "dashmap", + "dashmap 5.5.0", "hashbrown 0.13.2", ] @@ -4421,7 +4476,7 @@ dependencies = [ "bytes", "crc32c", "criterion", - "env_logger", + "env_logger 0.10.2", "log", "memoffset 0.9.0", "once_cell", @@ -4462,7 +4517,7 @@ dependencies = [ "cfg-if", "criterion", "findshlibs", - "inferno", + "inferno 0.11.21", "libc", "log", "nix 0.26.4", @@ -4688,9 +4743,9 @@ dependencies = [ "clap", "compute_api", "consumption_metrics", - "dashmap", + "dashmap 5.5.0", "ecdsa 0.16.9", - "env_logger", + "env_logger 0.10.2", "fallible-iterator", "flate2", "framed-websockets", @@ -4797,6 +4852,15 @@ dependencies = [ "serde", ] +[[package]] +name = "quick-xml" +version = "0.37.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f22f29bdff3987b4d8632ef95fd6424ec7e4e0a57e2f4fc63e489e75357f6a03" +dependencies = [ + "memchr", +] + [[package]] name = "quote" version = "1.0.37" @@ -7319,6 +7383,7 @@ dependencies = [ "hex-literal", "humantime", "hyper 0.14.30", + "inferno 0.12.0", "itertools 0.10.5", "jemalloc_pprof", "jsonwebtoken", @@ -7422,7 +7487,7 @@ dependencies = [ "anyhow", "camino-tempfile", "clap", - "env_logger", + "env_logger 0.10.2", "log", "postgres", "postgres_ffi", diff --git a/Cargo.toml b/Cargo.toml index 55bb9bfe11..a4e601bb58 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -110,6 +110,7 @@ hyper-util = "0.1" tokio-tungstenite = "0.21.0" indexmap = "2" indoc = "2" +inferno = "0.12.0" ipnet = "2.10.0" itertools = "0.10" itoa = "1.0.11" diff --git a/libs/utils/Cargo.toml b/libs/utils/Cargo.toml index 02bf77760a..edb451a02c 100644 --- a/libs/utils/Cargo.toml +++ b/libs/utils/Cargo.toml @@ -26,6 +26,7 @@ git-version.workspace = true hex = { workspace = true, features = ["serde"] } humantime.workspace = true hyper0 = { workspace = true, features = ["full"] } +inferno.workspace = true itertools.workspace = true fail.workspace = true futures = { workspace = true } diff --git a/libs/utils/src/http/endpoint.rs b/libs/utils/src/http/endpoint.rs index 9b37b69939..4b4aa88d6b 100644 --- a/libs/utils/src/http/endpoint.rs +++ b/libs/utils/src/http/endpoint.rs @@ -417,6 +417,7 @@ pub async fn profile_heap_handler(req: Request) -> Result, enum Format { Jemalloc, Pprof, + Svg, } // Parameters. @@ -424,9 +425,24 @@ pub async fn profile_heap_handler(req: Request) -> Result, None => Format::Pprof, Some("jemalloc") => Format::Jemalloc, Some("pprof") => Format::Pprof, + Some("svg") => Format::Svg, Some(format) => return Err(ApiError::BadRequest(anyhow!("invalid format {format}"))), }; + // Functions and mappings to strip when symbolizing pprof profiles. If true, + // also remove child frames. + static STRIP_FUNCTIONS: Lazy> = Lazy::new(|| { + vec![ + (Regex::new("^__rust").unwrap(), false), + (Regex::new("^_start$").unwrap(), false), + (Regex::new("^irallocx_prof").unwrap(), true), + (Regex::new("^prof_alloc_prep").unwrap(), true), + (Regex::new("^std::rt::lang_start").unwrap(), false), + (Regex::new("^std::sys::backtrace::__rust").unwrap(), false), + ] + }); + const STRIP_MAPPINGS: &[&str] = &["libc", "libgcc", "pthread", "vdso"]; + // Obtain profiler handle. let mut prof_ctl = jemalloc_pprof::PROF_CTL .as_ref() @@ -464,24 +480,9 @@ pub async fn profile_heap_handler(req: Request) -> Result, // Symbolize the profile. // TODO: consider moving this upstream to jemalloc_pprof and avoiding the // serialization roundtrip. - static STRIP_FUNCTIONS: Lazy> = Lazy::new(|| { - // Functions to strip from profiles. If true, also remove child frames. - vec![ - (Regex::new("^__rust").unwrap(), false), - (Regex::new("^_start$").unwrap(), false), - (Regex::new("^irallocx_prof").unwrap(), true), - (Regex::new("^prof_alloc_prep").unwrap(), true), - (Regex::new("^std::rt::lang_start").unwrap(), false), - (Regex::new("^std::sys::backtrace::__rust").unwrap(), false), - ] - }); let profile = pprof::decode(&bytes)?; let profile = pprof::symbolize(profile)?; - let profile = pprof::strip_locations( - profile, - &["libc", "libgcc", "pthread", "vdso"], - &STRIP_FUNCTIONS, - ); + let profile = pprof::strip_locations(profile, STRIP_MAPPINGS, &STRIP_FUNCTIONS); pprof::encode(&profile) }) .await @@ -494,6 +495,27 @@ pub async fn profile_heap_handler(req: Request) -> Result, .body(Body::from(data)) .map_err(|err| ApiError::InternalServerError(err.into())) } + + Format::Svg => { + let body = tokio::task::spawn_blocking(move || { + let bytes = prof_ctl.dump_pprof()?; + let profile = pprof::decode(&bytes)?; + let profile = pprof::symbolize(profile)?; + let profile = pprof::strip_locations(profile, STRIP_MAPPINGS, &STRIP_FUNCTIONS); + let mut opts = inferno::flamegraph::Options::default(); + opts.title = "Heap inuse".to_string(); + opts.count_name = "bytes".to_string(); + pprof::flamegraph(profile, &mut opts) + }) + .await + .map_err(|join_err| ApiError::InternalServerError(join_err.into()))? + .map_err(ApiError::InternalServerError)?; + Response::builder() + .status(200) + .header(CONTENT_TYPE, "image/svg+xml") + .body(Body::from(body)) + .map_err(|err| ApiError::InternalServerError(err.into())) + } } } diff --git a/libs/utils/src/pprof.rs b/libs/utils/src/pprof.rs index 90910897bf..dd57f9ed4b 100644 --- a/libs/utils/src/pprof.rs +++ b/libs/utils/src/pprof.rs @@ -1,8 +1,9 @@ +use anyhow::bail; use flate2::write::{GzDecoder, GzEncoder}; use flate2::Compression; use itertools::Itertools as _; use once_cell::sync::Lazy; -use pprof::protos::{Function, Line, Message as _, Profile}; +use pprof::protos::{Function, Line, Location, Message as _, Profile}; use regex::Regex; use std::borrow::Cow; @@ -188,3 +189,59 @@ pub fn strip_locations( profile } + +/// Generates an SVG flamegraph from a symbolized pprof profile. +pub fn flamegraph( + profile: Profile, + opts: &mut inferno::flamegraph::Options, +) -> anyhow::Result> { + if profile.mapping.iter().any(|m| !m.has_functions) { + bail!("profile not symbolized"); + } + + // Index locations, functions, and strings. + let locations: HashMap = + profile.location.into_iter().map(|l| (l.id, l)).collect(); + let functions: HashMap = + profile.function.into_iter().map(|f| (f.id, f)).collect(); + let strings = profile.string_table; + + // Resolve stacks as function names, and sum sample values per stack. Also reverse the stack, + // since inferno expects it bottom-up. + let mut stacks: HashMap, i64> = HashMap::new(); + for sample in profile.sample { + let mut stack = Vec::with_capacity(sample.location_id.len()); + for location in sample.location_id.into_iter().rev() { + let Some(location) = locations.get(&location) else { + bail!("missing location {location}"); + }; + for line in location.line.iter().rev() { + let Some(function) = functions.get(&line.function_id) else { + bail!("missing function {}", line.function_id); + }; + let Some(name) = strings.get(function.name as usize) else { + bail!("missing string {}", function.name); + }; + stack.push(name.as_str()); + } + } + let Some(&value) = sample.value.first() else { + bail!("missing value"); + }; + *stacks.entry(stack).or_default() += value; + } + + // Construct stack lines for inferno. + let lines = stacks + .into_iter() + .map(|(stack, value)| (stack.into_iter().join(";"), value)) + .map(|(stack, value)| format!("{stack} {value}")) + .sorted() + .collect_vec(); + + // Construct the flamegraph. + let mut bytes = Vec::new(); + let lines = lines.iter().map(|line| line.as_str()); + inferno::flamegraph::from_lines(opts, lines, &mut bytes)?; + Ok(bytes) +} From 0d4fce2d358b3c63b2ac77dac7acd2d42944f3fb Mon Sep 17 00:00:00 2001 From: John Spray Date: Fri, 10 Jan 2025 13:57:23 +0000 Subject: [PATCH 028/111] tests: refine how compat snapshot is generated (#10342) ## Problem I noticed in https://github.com/neondatabase/neon/pull/9537 that tests which work with compat snapshots were writing several hundred MB of data, which isn't really necessary. Also, the snapshots are large but don't have the proper variety of storage format features, e.g. they could just have L0 deltas. ## Summary of changes - Use smaller scale factor and runtime to generate less data - Configure a small layer size and use force image layer generation so that our output contains L1 deltas and image layers, and has a decent number of entries in the layer map --- test_runner/regress/test_compatibility.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/test_runner/regress/test_compatibility.py b/test_runner/regress/test_compatibility.py index ba7305148f..a6eaaf6c4c 100644 --- a/test_runner/regress/test_compatibility.py +++ b/test_runner/regress/test_compatibility.py @@ -141,11 +141,18 @@ def test_create_snapshot( neon_env_builder.num_safekeepers = 3 neon_env_builder.enable_pageserver_remote_storage(RemoteStorageKind.LOCAL_FS) - env = neon_env_builder.init_start() + env = neon_env_builder.init_start( + initial_tenant_conf={ + # Miniature layers to enable generating non-trivial layer map without writing lots of data + "checkpoint_distance": f"{128 * 1024}", + "compaction_threshold": "1", + "compaction_target_size": f"{128 * 1024}", + } + ) endpoint = env.endpoints.create_start("main") - pg_bin.run_capture(["pgbench", "--initialize", "--scale=10", endpoint.connstr()]) - pg_bin.run_capture(["pgbench", "--time=60", "--progress=2", endpoint.connstr()]) + pg_bin.run_capture(["pgbench", "--initialize", "--scale=1", endpoint.connstr()]) + pg_bin.run_capture(["pgbench", "--time=30", "--progress=2", endpoint.connstr()]) pg_bin.run_capture( ["pg_dumpall", f"--dbname={endpoint.connstr()}", f"--file={test_output_dir / 'dump.sql'}"] ) @@ -157,7 +164,9 @@ def test_create_snapshot( pageserver_http = env.pageserver.http_client() flush_ep_to_pageserver(env, endpoint, tenant_id, timeline_id) - pageserver_http.timeline_checkpoint(tenant_id, timeline_id, wait_until_uploaded=True) + pageserver_http.timeline_checkpoint( + tenant_id, timeline_id, wait_until_uploaded=True, force_image_layer_creation=True + ) env.endpoints.stop_all() for sk in env.safekeepers: From 105f66c4ce760477e774382ee3f1f6945fb2a8a3 Mon Sep 17 00:00:00 2001 From: John Spray Date: Fri, 10 Jan 2025 13:57:26 +0000 Subject: [PATCH 029/111] tests: move test_parallel_copy into performance tree (#10343) ## Problem This test writes ~5GB of data. It is not suitable to run in parallel with all the other small tests in test_runner/regress. via #9537 ## Summary of changes - Move test_parallel_copy into the performance directory, so that it does not run in parallel with other tests --- test_runner/{regress => performance}/test_parallel_copy.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename test_runner/{regress => performance}/test_parallel_copy.py (100%) diff --git a/test_runner/regress/test_parallel_copy.py b/test_runner/performance/test_parallel_copy.py similarity index 100% rename from test_runner/regress/test_parallel_copy.py rename to test_runner/performance/test_parallel_copy.py From 71bca6f58045966fd1c630e685e2d6ea3b7bbbd6 Mon Sep 17 00:00:00 2001 From: Folke Behrens Date: Fri, 10 Jan 2025 15:32:26 +0100 Subject: [PATCH 030/111] poetry: Update packaging for poetry v2 (#10344) ## Problem When poetry v2 (released Jan 5) is used it needs `packaging.metadata` module, but we downgrade `packaging` to 23.0. `packaging==23.1` introduced the metadata submodule. ## Summary of changes Update `packaging` to 24.2. --- poetry.lock | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/poetry.lock b/poetry.lock index 5f15223dca..2cd2bc6383 100644 --- a/poetry.lock +++ b/poetry.lock @@ -2028,13 +2028,13 @@ openapi-schema-validator = ">=0.4.2,<0.5.0" [[package]] name = "packaging" -version = "23.0" +version = "24.2" description = "Core utilities for Python packages" optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" files = [ - {file = "packaging-23.0-py3-none-any.whl", hash = "sha256:714ac14496c3e68c99c29b00845f7a2b85f3bb6f1078fd9f72fd20f0570002b2"}, - {file = "packaging-23.0.tar.gz", hash = "sha256:b6ad297f8907de0fa2fe1ccbd26fdaf387f5f47c7275fedf8cce89f99446cf97"}, + {file = "packaging-24.2-py3-none-any.whl", hash = "sha256:09abb1bccd265c01f4a3aa3f7a7db064b36514d2cba19a2f694fe6150451a759"}, + {file = "packaging-24.2.tar.gz", hash = "sha256:c228a6dc5e932d346bc5739379109d49e8853dd8223571c7c5b55260edc0b97f"}, ] [[package]] From aa8da1e6217a771286bff05ebde317edf222af71 Mon Sep 17 00:00:00 2001 From: Christian Schwarz Date: Fri, 10 Jan 2025 15:35:07 +0100 Subject: [PATCH 031/111] move throttle into pagestream_read_message --- pageserver/src/page_service.rs | 113 ++++++++++++++------------------- 1 file changed, 49 insertions(+), 64 deletions(-) diff --git a/pageserver/src/page_service.rs b/pageserver/src/page_service.rs index d00ec11a76..6f680be968 100644 --- a/pageserver/src/page_service.rs +++ b/pageserver/src/page_service.rs @@ -574,44 +574,6 @@ enum BatchedFeMessage { }, } -impl BatchedFeMessage { - async fn throttle_and_record_start_processing( - &mut self, - cancel: &CancellationToken, - ) -> Result<(), QueryError> { - let (shard, tokens, timers) = match self { - BatchedFeMessage::Exists { shard, timer, .. } - | BatchedFeMessage::Nblocks { shard, timer, .. } - | BatchedFeMessage::DbSize { shard, timer, .. } - | BatchedFeMessage::GetSlruSegment { shard, timer, .. } => { - ( - shard, - // 1 token is probably under-estimating because these - // request handlers typically do several Timeline::get calls. - 1, - itertools::Either::Left(std::iter::once(timer)), - ) - } - BatchedFeMessage::GetPage { shard, pages, .. } => ( - shard, - pages.len(), - itertools::Either::Right(pages.iter_mut().map(|(_, _, timer)| timer)), - ), - BatchedFeMessage::RespondError { .. } => return Ok(()), - }; - let throttled = tokio::select! { - throttled = shard.pagestream_throttle.throttle(tokens) => { throttled } - _ = cancel.cancelled() => { - return Err(QueryError::Shutdown); - } - }; - for timer in timers { - timer.observe_throttle_done_execution_starting(&throttled); - } - Ok(()) - } -} - impl PageServerHandler { pub fn new( tenant_manager: Arc, @@ -697,6 +659,18 @@ impl PageServerHandler { // parse request let neon_fe_msg = PagestreamFeMessage::parse(&mut copy_data_bytes.reader())?; + // TODO: turn in to async closure once available to avoid repeating received_at + async fn record_op_start_and_throttle( + shard: &timeline::handle::Handle, + op: metrics::SmgrQueryType, + received_at: Instant, + ) -> SmgrOpTimer { + let mut timer = shard.query_metrics.start_smgr_op(op, received_at); + let throttled = shard.pagestream_throttle.throttle(1).await; + timer.observe_throttle_done_execution_starting(&throttled); + timer + } + let batched_msg = match neon_fe_msg { PagestreamFeMessage::Exists(req) => { let span = tracing::info_span!(parent: parent_span, "handle_get_rel_exists_request", rel = %req.rel, req_lsn = %req.request_lsn); @@ -704,9 +678,12 @@ impl PageServerHandler { .get(tenant_id, timeline_id, ShardSelector::Zero) .instrument(span.clone()) // sets `shard_id` field .await?; - let timer = shard - .query_metrics - .start_smgr_op(metrics::SmgrQueryType::GetRelExists, received_at); + let timer = record_op_start_and_throttle( + &shard, + metrics::SmgrQueryType::GetRelExists, + received_at, + ) + .await; BatchedFeMessage::Exists { span, timer, @@ -720,9 +697,12 @@ impl PageServerHandler { .get(tenant_id, timeline_id, ShardSelector::Zero) .instrument(span.clone()) // sets `shard_id` field .await?; - let timer = shard - .query_metrics - .start_smgr_op(metrics::SmgrQueryType::GetRelSize, received_at); + let timer = record_op_start_and_throttle( + &shard, + metrics::SmgrQueryType::GetRelSize, + received_at, + ) + .await; BatchedFeMessage::Nblocks { span, timer, @@ -736,9 +716,12 @@ impl PageServerHandler { .get(tenant_id, timeline_id, ShardSelector::Zero) .instrument(span.clone()) // sets `shard_id` field .await?; - let timer = shard - .query_metrics - .start_smgr_op(metrics::SmgrQueryType::GetDbSize, received_at); + let timer = record_op_start_and_throttle( + &shard, + metrics::SmgrQueryType::GetDbSize, + received_at, + ) + .await; BatchedFeMessage::DbSize { span, timer, @@ -752,9 +735,12 @@ impl PageServerHandler { .get(tenant_id, timeline_id, ShardSelector::Zero) .instrument(span.clone()) // sets `shard_id` field .await?; - let timer = shard - .query_metrics - .start_smgr_op(metrics::SmgrQueryType::GetSlruSegment, received_at); + let timer = record_op_start_and_throttle( + &shard, + metrics::SmgrQueryType::GetSlruSegment, + received_at, + ) + .await; BatchedFeMessage::GetSlruSegment { span, timer, @@ -805,12 +791,17 @@ impl PageServerHandler { } }; - // It's important to start the timer before waiting for the LSN + // It's important to start the smgr op metric recorder as early as possible // so that the _started counters are incremented before we do - // any serious waiting, e.g., for LSNs. - let timer = shard - .query_metrics - .start_smgr_op(metrics::SmgrQueryType::GetPageAtLsn, received_at); + // any serious waiting, e.g., for throttle or LSNs. + // We feed wait times to the metric recorder so it can deduct them / count + // them seperately. + let timer = record_op_start_and_throttle( + &shard, + metrics::SmgrQueryType::GetPageAtLsn, + received_at, + ) + .await; let effective_request_lsn = match Self::wait_or_get_last_lsn( &shard, @@ -1225,7 +1216,7 @@ impl PageServerHandler { Ok(msg) => msg, Err(e) => break e, }; - let mut msg = match msg { + let msg = match msg { Some(msg) => msg, None => { debug!("pagestream subprotocol end observed"); @@ -1233,10 +1224,6 @@ impl PageServerHandler { } }; - if let Err(cancelled) = msg.throttle_and_record_start_processing(&self.cancel).await { - break cancelled; - } - let err = self .pagesteam_handle_batched_message(pgb_writer, msg, &cancel, ctx) .await; @@ -1394,15 +1381,13 @@ impl PageServerHandler { return Ok(()); } }; - let mut batch = match batch { + let batch = match batch { Ok(batch) => batch, Err(e) => { return Err(e); } }; - batch - .throttle_and_record_start_processing(&self.cancel) - .await?; + self.pagesteam_handle_batched_message(pgb_writer, batch, &cancel, &ctx) .await?; } From 4398051385affb0be87418543b6e3f3f08220b5e Mon Sep 17 00:00:00 2001 From: John Spray Date: Fri, 10 Jan 2025 15:53:23 +0000 Subject: [PATCH 032/111] tests: smaller datasets in LFC tests (#10346) ## Problem These two tests came up in #9537 as doing multi-gigabyte I/O, and from inspection of the tests it doesn't seem like they need that to fulfil their purpose. ## Summary of changes - In test_local_file_cache_unlink, run fewer background threads with a smaller number of rows. These background threads AFAICT exist to make sure some I/O is going on while we unlink the LFC directory, but 5 threads should be enough for "some". - In test_lfc_resize, tweak the test to validate that the cache size is larger than the final size before resizing it, so that we're sure we're writing enough data to really be doing something. Then decrease the pgbench scale. --- test_runner/regress/test_lfc_resize.py | 31 ++++++++++++++------ test_runner/regress/test_local_file_cache.py | 4 +-- 2 files changed, 24 insertions(+), 11 deletions(-) diff --git a/test_runner/regress/test_lfc_resize.py b/test_runner/regress/test_lfc_resize.py index 377b0fb4d4..8762e6525b 100644 --- a/test_runner/regress/test_lfc_resize.py +++ b/test_runner/regress/test_lfc_resize.py @@ -30,7 +30,7 @@ def test_lfc_resize(neon_simple_env: NeonEnv, pg_bin: PgBin): ], ) n_resize = 10 - scale = 100 + scale = 20 def run_pgbench(connstr: str): log.info(f"Start a pgbench workload on pg {connstr}") @@ -46,17 +46,36 @@ def test_lfc_resize(neon_simple_env: NeonEnv, pg_bin: PgBin): conn = endpoint.connect() cur = conn.cursor() + def get_lfc_size() -> tuple[int, int]: + lfc_file_path = endpoint.lfc_path() + lfc_file_size = lfc_file_path.stat().st_size + res = subprocess.run( + ["ls", "-sk", lfc_file_path], check=True, text=True, capture_output=True + ) + lfc_file_blocks = re.findall("([0-9A-F]+)", res.stdout)[0] + log.info(f"Size of LFC file {lfc_file_size}, blocks {lfc_file_blocks}") + + return (lfc_file_size, lfc_file_blocks) + # For as long as pgbench is running, twiddle the LFC size once a second. # Note that we launch this immediately, already while the "pgbench -i" # initialization step is still running. That's quite a different workload # than the actual pgbench benchamark run, so this gives us coverage of both. while thread.is_alive(): - size = random.randint(1, 512) + # Vary the LFC size randomly within a range above what we will later + # decrease it to. This should ensure that the final size decrease + # is really doing something. + size = random.randint(192, 512) cur.execute(f"alter system set neon.file_cache_size_limit='{size}MB'") cur.execute("select pg_reload_conf()") time.sleep(1) + thread.join() + # Before shrinking the cache, check that it really is large now + (lfc_file_size, lfc_file_blocks) = get_lfc_size() + assert int(lfc_file_blocks) > 128 * 1024 + # At the end, set it at 100 MB, and perform a final check that the disk usage # of the file is in that ballbark. # @@ -66,13 +85,7 @@ def test_lfc_resize(neon_simple_env: NeonEnv, pg_bin: PgBin): cur.execute("select pg_reload_conf()") nretries = 10 while True: - lfc_file_path = endpoint.lfc_path() - lfc_file_size = lfc_file_path.stat().st_size - res = subprocess.run( - ["ls", "-sk", lfc_file_path], check=True, text=True, capture_output=True - ) - lfc_file_blocks = re.findall("([0-9A-F]+)", res.stdout)[0] - log.info(f"Size of LFC file {lfc_file_size}, blocks {lfc_file_blocks}") + (lfc_file_size, lfc_file_blocks) = get_lfc_size() assert lfc_file_size <= 512 * 1024 * 1024 if int(lfc_file_blocks) <= 128 * 1024 or nretries == 0: diff --git a/test_runner/regress/test_local_file_cache.py b/test_runner/regress/test_local_file_cache.py index 94c630ffcf..21c9e97a42 100644 --- a/test_runner/regress/test_local_file_cache.py +++ b/test_runner/regress/test_local_file_cache.py @@ -29,8 +29,8 @@ def test_local_file_cache_unlink(neon_env_builder: NeonEnvBuilder): cur = endpoint.connect().cursor() stop = threading.Event() - n_rows = 100000 - n_threads = 20 + n_rows = 10000 + n_threads = 5 n_updates_per_connection = 1000 cur.execute("CREATE TABLE lfctest (id int4 PRIMARY KEY, n int) WITH (fillfactor=10)") From 425b77784045ed20ec1b7a2c58869aa4118e1a44 Mon Sep 17 00:00:00 2001 From: Cheng Chen Date: Fri, 10 Jan 2025 08:38:13 -0800 Subject: [PATCH 033/111] chore(compute): pg_mooncake v0.1.0 (#10337) ## Problem Upgrade pg_mooncake to v0.1.0 ## Summary of changes --- compute/compute-node.Dockerfile | 19 +++++-------------- 1 file changed, 5 insertions(+), 14 deletions(-) diff --git a/compute/compute-node.Dockerfile b/compute/compute-node.Dockerfile index 06aaf9e7f4..303daec240 100644 --- a/compute/compute-node.Dockerfile +++ b/compute/compute-node.Dockerfile @@ -1167,22 +1167,13 @@ FROM rust-extensions-build AS pg-mooncake-build ARG PG_VERSION COPY --from=pg-build /usr/local/pgsql/ /usr/local/pgsql/ -# The topmost commit in the `neon` branch at the time of writing this -# https://github.com/Mooncake-Labs/pg_mooncake/commits/neon/ -# https://github.com/Mooncake-Labs/pg_mooncake/commit/077c92c452bb6896a7b7776ee95f039984f076af -ENV PG_MOONCAKE_VERSION=077c92c452bb6896a7b7776ee95f039984f076af ENV PATH="/usr/local/pgsql/bin/:$PATH" -RUN case "${PG_VERSION}" in \ - 'v14') \ - echo "pg_mooncake is not supported on Postgres ${PG_VERSION}" && exit 0;; \ - esac && \ - git clone --depth 1 --branch neon https://github.com/Mooncake-Labs/pg_mooncake.git pg_mooncake-src && \ - cd pg_mooncake-src && \ - git checkout "${PG_MOONCAKE_VERSION}" && \ - git submodule update --init --depth 1 --recursive && \ - make BUILD_TYPE=release -j $(getconf _NPROCESSORS_ONLN) && \ - make BUILD_TYPE=release -j $(getconf _NPROCESSORS_ONLN) install && \ +RUN wget https://github.com/Mooncake-Labs/pg_mooncake/releases/download/v0.1.0/pg_mooncake-0.1.0.tar.gz -O pg_mooncake.tar.gz && \ + echo "eafd059b77f541f11525eb8affcd66a176968cbd8fe7c0d436e733f2aa4da59f pg_mooncake.tar.gz" | sha256sum --check && \ + mkdir pg_mooncake-src && cd pg_mooncake-src && tar xzf ../pg_mooncake.tar.gz --strip-components=1 -C . && \ + make release -j $(getconf _NPROCESSORS_ONLN) && \ + make install -j $(getconf _NPROCESSORS_ONLN) && \ echo 'trusted = true' >> /usr/local/pgsql/share/extension/pg_mooncake.control ######################################################################################### From 3cd5034eac3677223d8e2dacb772925fd22e0136 Mon Sep 17 00:00:00 2001 From: Vlad Lazar Date: Fri, 10 Jan 2025 18:35:16 +0000 Subject: [PATCH 034/111] storcon: don't assume host:port format in storcon client (#10347) ## Problem https://github.com/neondatabase/infra/pull/2725 updated the scrubber to use a non-host port endpoint for storcon. That breaks when unwrapping the port. ## Summary of changes Support both `host:port` and `host` formats for the storcon api. --- storage_controller/client/src/control_api.rs | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/storage_controller/client/src/control_api.rs b/storage_controller/client/src/control_api.rs index a981b5020e..f8a2790769 100644 --- a/storage_controller/client/src/control_api.rs +++ b/storage_controller/client/src/control_api.rs @@ -1,7 +1,6 @@ use pageserver_client::mgmt_api::{self, ResponseErrorMessageExt}; use reqwest::{Method, Url}; use serde::{de::DeserializeOwned, Serialize}; -use std::str::FromStr; pub struct Client { base_url: Url, @@ -31,16 +30,11 @@ impl Client { RQ: Serialize + Sized, RS: DeserializeOwned + Sized, { - // The configured URL has the /upcall path prefix for pageservers to use: we will strip that out - // for general purpose API access. - let url = Url::from_str(&format!( - "http://{}:{}/{path}", - self.base_url.host_str().unwrap(), - self.base_url.port().unwrap() - )) - .unwrap(); - - let mut builder = self.client.request(method, url); + let request_path = self + .base_url + .join(&path) + .expect("Failed to build request path"); + let mut builder = self.client.request(method, request_path); if let Some(body) = body { builder = builder.json(&body) } From cdd34dfc1270b275646be524421b38ecf5d36665 Mon Sep 17 00:00:00 2001 From: Christian Schwarz Date: Fri, 10 Jan 2025 20:19:48 +0100 Subject: [PATCH 035/111] impr(utils/spsc_fold): add another test case (#10319) Wondered about the case covered here while investigating #10309. --- libs/utils/src/sync/spsc_fold.rs | 34 ++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/libs/utils/src/sync/spsc_fold.rs b/libs/utils/src/sync/spsc_fold.rs index 7bad4f1068..0cab291d51 100644 --- a/libs/utils/src/sync/spsc_fold.rs +++ b/libs/utils/src/sync/spsc_fold.rs @@ -453,4 +453,38 @@ mod tests { let err = recv_task.await.unwrap().expect_err("should error"); assert!(matches!(err, RecvError::SenderGone)); } + + #[tokio::test(start_paused = true)] + async fn test_receiver_drop_while_waiting_for_receiver_to_consume_unblocks_sender() { + let (mut sender, receiver) = channel(); + + let state = receiver.state.clone(); + + sender.send((), |_, _| unreachable!()).await.unwrap(); + + assert!(matches!(&*state.value.lock().unwrap(), &State::HasData(_))); + + let unmergeable = sender.send((), |_, _| Err(())); + let mut unmergeable = std::pin::pin!(unmergeable); + tokio::select! { + _ = tokio::time::sleep(FOREVER) => {}, + _ = &mut unmergeable => { + panic!("unmergeable should not complete"); + }, + } + + assert!(matches!( + &*state.value.lock().unwrap(), + &State::SenderWaitsForReceiverToConsume(_) + )); + + drop(receiver); + + assert!(matches!( + &*state.value.lock().unwrap(), + &State::ReceiverGone + )); + + unmergeable.await.unwrap_err(); + } } From 9b43204893900a890ef29f571a1e1a77e49d2e22 Mon Sep 17 00:00:00 2001 From: Christian Schwarz Date: Fri, 10 Jan 2025 20:21:01 +0100 Subject: [PATCH 036/111] fix(page_service): Timeline::gate held open while throttling (#10314) When we moved throttling up from Timeline::get into page_service, we stopped being sensitive to `Timeline::cancel`, even though we're holding a Handle and thus a guard on the `Timeline::gate` open. This PR rectifies the situation. Refs - Found while investigating #10309 (hung detach because gate kept open), but not expected to be the root cause of that issue because the affected tenants are not being throttled according to their metrics. --- pageserver/src/page_service.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pageserver/src/page_service.rs b/pageserver/src/page_service.rs index 0c4a1b18f5..f6504bd3b5 100644 --- a/pageserver/src/page_service.rs +++ b/pageserver/src/page_service.rs @@ -618,6 +618,9 @@ impl BatchedFeMessage { }; let throttled = tokio::select! { throttled = shard.pagestream_throttle.throttle(tokens) => { throttled } + _ = shard.cancel.cancelled() => { + return Err(QueryError::Shutdown); + } _ = cancel.cancelled() => { return Err(QueryError::Shutdown); } From 8793e28ccb76640f995463a1584fc3ad36e7476e Mon Sep 17 00:00:00 2001 From: Christian Schwarz Date: Fri, 10 Jan 2025 20:23:17 +0100 Subject: [PATCH 037/111] throttling cancel-sensitivity --- pageserver/src/page_service.rs | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/pageserver/src/page_service.rs b/pageserver/src/page_service.rs index 6f680be968..a050509373 100644 --- a/pageserver/src/page_service.rs +++ b/pageserver/src/page_service.rs @@ -664,11 +664,17 @@ impl PageServerHandler { shard: &timeline::handle::Handle, op: metrics::SmgrQueryType, received_at: Instant, - ) -> SmgrOpTimer { + ) -> Result { + // It's important to start the smgr op metric recorder as early as possible + // so that the _started counters are incremented before we do + // any serious waiting, e.g., for throttle, batching, or actual request handling. let mut timer = shard.query_metrics.start_smgr_op(op, received_at); - let throttled = shard.pagestream_throttle.throttle(1).await; + let throttled = tokio::select! { + res = shard.pagestream_throttle.throttle(1) => res, + _ = shard.cancel.cancelled() => return Err(QueryError::Shutdown), + }; timer.observe_throttle_done_execution_starting(&throttled); - timer + Ok(timer) } let batched_msg = match neon_fe_msg { @@ -683,7 +689,7 @@ impl PageServerHandler { metrics::SmgrQueryType::GetRelExists, received_at, ) - .await; + .await?; BatchedFeMessage::Exists { span, timer, @@ -702,7 +708,7 @@ impl PageServerHandler { metrics::SmgrQueryType::GetRelSize, received_at, ) - .await; + .await?; BatchedFeMessage::Nblocks { span, timer, @@ -721,7 +727,7 @@ impl PageServerHandler { metrics::SmgrQueryType::GetDbSize, received_at, ) - .await; + .await?; BatchedFeMessage::DbSize { span, timer, @@ -740,7 +746,7 @@ impl PageServerHandler { metrics::SmgrQueryType::GetSlruSegment, received_at, ) - .await; + .await?; BatchedFeMessage::GetSlruSegment { span, timer, @@ -801,7 +807,7 @@ impl PageServerHandler { metrics::SmgrQueryType::GetPageAtLsn, received_at, ) - .await; + .await?; let effective_request_lsn = match Self::wait_or_get_last_lsn( &shard, From 4d496a29c2c4fc4f0330caf8168022cca65fb8d2 Mon Sep 17 00:00:00 2001 From: Christian Schwarz Date: Fri, 10 Jan 2025 17:12:45 +0100 Subject: [PATCH 038/111] following up to the last commit, the observation points that we use to calculate the various latency metrics are different, adjust for that --- pageserver/src/metrics.rs | 99 ++++++++++++++++++++++++---------- pageserver/src/page_service.rs | 32 +++++++++-- 2 files changed, 99 insertions(+), 32 deletions(-) diff --git a/pageserver/src/metrics.rs b/pageserver/src/metrics.rs index b4e20cb8b9..fe019cc678 100644 --- a/pageserver/src/metrics.rs +++ b/pageserver/src/metrics.rs @@ -1239,12 +1239,20 @@ pub(crate) struct SmgrOpTimerInner { #[derive(Debug)] enum SmgrOpTimerState { Received { + // In the future, we may want to track the full time the request spent + // inside pageserver process (time spent in kernel buffers can't be tracked). + // `received_at` would be used for that. + #[allow(dead_code)] received_at: Instant, }, - ThrottleDoneExecutionStarting { - received_at: Instant, + ParsedRoutedThrottledNowBatching { throttle_started_at: Instant, - started_execution_at: Instant, + throttle_done_at: Instant, + }, + BatchedNowExecuting { + throttle_started_at: Instant, + throttle_done_at: Instant, + execution_started_at: Instant, }, } @@ -1255,22 +1263,20 @@ pub(crate) struct SmgrOpFlushInProgress { } impl SmgrOpTimer { - pub(crate) fn observe_throttle_done_execution_starting(&mut self, throttle: &ThrottleResult) { + pub(crate) fn observe_throttle_done(&mut self, throttle: &ThrottleResult) { let inner = self.0.as_mut().expect("other public methods consume self"); match (&mut inner.timings, throttle) { - (SmgrOpTimerState::Received { received_at }, throttle) => match throttle { + (SmgrOpTimerState::Received { received_at: _ }, throttle) => match throttle { ThrottleResult::NotThrottled { start } => { - inner.timings = SmgrOpTimerState::ThrottleDoneExecutionStarting { - received_at: *received_at, + inner.timings = SmgrOpTimerState::ParsedRoutedThrottledNowBatching { throttle_started_at: *start, - started_execution_at: *start, + throttle_done_at: *start, }; } ThrottleResult::Throttled { start, end } => { - inner.timings = SmgrOpTimerState::ThrottleDoneExecutionStarting { - received_at: *start, + inner.timings = SmgrOpTimerState::ParsedRoutedThrottledNowBatching { throttle_started_at: *start, - started_execution_at: *end, + throttle_done_at: *end, }; } }, @@ -1278,7 +1284,32 @@ impl SmgrOpTimer { } } - pub(crate) fn observe_smgr_op_completion_and_start_flushing(mut self) -> SmgrOpFlushInProgress { + pub(crate) fn observe_execution_start(&mut self, at: Instant) { + let inner = self.0.as_mut().expect("other public methods consume self"); + match &mut inner.timings { + SmgrOpTimerState::ParsedRoutedThrottledNowBatching { + throttle_started_at, + throttle_done_at, + } => { + inner.timings = SmgrOpTimerState::BatchedNowExecuting { + throttle_started_at: *throttle_started_at, + throttle_done_at: *throttle_done_at, + execution_started_at: at, + }; + } + x => panic!("called in unexpected state: {x:?}"), + } + } + + pub(crate) fn observe_execution_end_flush_start(mut self) -> SmgrOpFlushInProgress { + assert!( + matches!( + self.0.as_ref().unwrap().timings, + SmgrOpTimerState::BatchedNowExecuting { .. } + ), + "called in unexpected state: {:?}", + self.0.as_ref().unwrap().timings, + ); let (flush_start, inner) = self .smgr_op_end() .expect("this method consume self, and the only other caller is drop handler"); @@ -1300,41 +1331,53 @@ impl SmgrOpTimer { let now = Instant::now(); + // TODO: use label for unfinished requests instead of Duration::ZERO. + // This is quite rare in practice, only during tenant/pageservers shutdown. + let throttle; let batch; let execution; - let throttle; match inner.timings { - SmgrOpTimerState::Received { received_at } => { - batch = (now - received_at).as_secs_f64(); - // TODO: use label for dropped requests. - // This is quite rare in practice, only during tenant/pageservers shutdown. + SmgrOpTimerState::Received { received_at: _ } => { throttle = Duration::ZERO; - execution = Duration::ZERO.as_secs_f64(); + batch = Duration::ZERO; + execution = Duration::ZERO; } - SmgrOpTimerState::ThrottleDoneExecutionStarting { - received_at, + SmgrOpTimerState::ParsedRoutedThrottledNowBatching { throttle_started_at, - started_execution_at, + throttle_done_at, } => { - batch = (throttle_started_at - received_at).as_secs_f64(); - throttle = started_execution_at - throttle_started_at; - execution = (now - started_execution_at).as_secs_f64(); + throttle = throttle_done_at - throttle_started_at; + batch = Duration::ZERO; + execution = Duration::ZERO; + } + SmgrOpTimerState::BatchedNowExecuting { + throttle_started_at, + throttle_done_at, + execution_started_at, + } => { + throttle = throttle_done_at - throttle_started_at; + batch = throttle_done_at - execution_started_at; + execution = now - execution_started_at; } } // update time spent in batching - inner.global_batch_wait_time.observe(batch); - inner.per_timeline_batch_wait_time.observe(batch); + inner.global_batch_wait_time.observe(batch.as_secs_f64()); + inner + .per_timeline_batch_wait_time + .observe(batch.as_secs_f64()); // time spent in throttle metric is updated by throttle impl let _ = throttle; // update metrics for execution latency - inner.global_execution_latency_histo.observe(execution); + inner + .global_execution_latency_histo + .observe(execution.as_secs_f64()); if let Some(per_timeline_execution_latency_histo) = &inner.per_timeline_execution_latency_histo { - per_timeline_execution_latency_histo.observe(execution); + per_timeline_execution_latency_histo.observe(execution.as_secs_f64()); } Some((now, inner)) diff --git a/pageserver/src/page_service.rs b/pageserver/src/page_service.rs index a050509373..f72f7dab11 100644 --- a/pageserver/src/page_service.rs +++ b/pageserver/src/page_service.rs @@ -574,6 +574,25 @@ enum BatchedFeMessage { }, } +impl BatchedFeMessage { + fn observe_execution_start(&mut self, at: Instant) { + match self { + BatchedFeMessage::Exists { timer, .. } + | BatchedFeMessage::Nblocks { timer, .. } + | BatchedFeMessage::DbSize { timer, .. } + | BatchedFeMessage::GetSlruSegment { timer, .. } => { + timer.observe_execution_start(at); + } + BatchedFeMessage::GetPage { pages, .. } => { + for (_, _, timer) in pages { + timer.observe_execution_start(at); + } + } + BatchedFeMessage::RespondError { .. } => {} + } + } +} + impl PageServerHandler { pub fn new( tenant_manager: Arc, @@ -673,7 +692,7 @@ impl PageServerHandler { res = shard.pagestream_throttle.throttle(1) => res, _ = shard.cancel.cancelled() => return Err(QueryError::Shutdown), }; - timer.observe_throttle_done_execution_starting(&throttled); + timer.observe_throttle_done(&throttled); Ok(timer) } @@ -912,6 +931,13 @@ impl PageServerHandler { where IO: AsyncRead + AsyncWrite + Send + Sync + Unpin, { + let started_at = Instant::now(); + let batch = { + let mut batch = batch; + batch.observe_execution_start(started_at); + batch + }; + // invoke handler function let (handler_results, span): ( Vec>, @@ -1071,8 +1097,7 @@ impl PageServerHandler { // The timer's underlying metric is used for a storage-internal latency SLO and // we don't want to include latency in it that we can't control. // And as pointed out above, in this case, we don't control the time that flush will take. - let flushing_timer = - timer.map(|timer| timer.observe_smgr_op_completion_and_start_flushing()); + let flushing_timer = timer.map(|timer| timer.observe_execution_end_flush_start()); // what we want to do let flush_fut = pgb_writer.flush(); @@ -1393,7 +1418,6 @@ impl PageServerHandler { return Err(e); } }; - self.pagesteam_handle_batched_message(pgb_writer, batch, &cancel, &ctx) .await?; } From ad5120197c3a07b1a9ce945bb9f8fb1e3081e19c Mon Sep 17 00:00:00 2001 From: Christian Schwarz Date: Fri, 10 Jan 2025 17:20:36 +0100 Subject: [PATCH 039/111] self-review --- pageserver/src/page_service.rs | 5 ----- 1 file changed, 5 deletions(-) diff --git a/pageserver/src/page_service.rs b/pageserver/src/page_service.rs index f72f7dab11..17ac4c660b 100644 --- a/pageserver/src/page_service.rs +++ b/pageserver/src/page_service.rs @@ -816,11 +816,6 @@ impl PageServerHandler { } }; - // It's important to start the smgr op metric recorder as early as possible - // so that the _started counters are incremented before we do - // any serious waiting, e.g., for throttle or LSNs. - // We feed wait times to the metric recorder so it can deduct them / count - // them seperately. let timer = record_op_start_and_throttle( &shard, metrics::SmgrQueryType::GetPageAtLsn, From d6a2b62cfbfb08ed8ecdd71701e78e8fd1c41348 Mon Sep 17 00:00:00 2001 From: Christian Schwarz Date: Fri, 10 Jan 2025 19:47:27 +0100 Subject: [PATCH 040/111] grand refactor of SmgrOpTimer states --- pageserver/src/metrics.rs | 340 +++++++++-------------- pageserver/src/page_service.rs | 12 +- pageserver/src/tenant.rs | 14 +- pageserver/src/tenant/throttle.rs | 45 +-- pageserver/src/tenant/timeline.rs | 8 +- pageserver/src/tenant/timeline/delete.rs | 1 + 6 files changed, 171 insertions(+), 249 deletions(-) diff --git a/pageserver/src/metrics.rs b/pageserver/src/metrics.rs index fe019cc678..b9b4f44630 100644 --- a/pageserver/src/metrics.rs +++ b/pageserver/src/metrics.rs @@ -1233,6 +1233,8 @@ pub(crate) struct SmgrOpTimerInner { global_flush_in_progress_micros: IntCounter, per_timeline_flush_in_progress_micros: IntCounter, + throttling: Arc, + timings: SmgrOpTimerState, } @@ -1245,132 +1247,105 @@ enum SmgrOpTimerState { #[allow(dead_code)] received_at: Instant, }, - ParsedRoutedThrottledNowBatching { + ParsedRoutedNowThrottling { throttle_started_at: Instant, + }, + ParsedRoutedThrottledNowBatching { throttle_done_at: Instant, }, BatchedNowExecuting { - throttle_started_at: Instant, - throttle_done_at: Instant, execution_started_at: Instant, }, + Flushing, + // NB: when adding observation points, remember to update the Drop impl. } -pub(crate) struct SmgrOpFlushInProgress { - flush_started_at: Instant, - global_micros: IntCounter, - per_timeline_micros: IntCounter, -} - +// NB: when adding observation points, remember to update the Drop impl. impl SmgrOpTimer { - pub(crate) fn observe_throttle_done(&mut self, throttle: &ThrottleResult) { - let inner = self.0.as_mut().expect("other public methods consume self"); - match (&mut inner.timings, throttle) { - (SmgrOpTimerState::Received { received_at: _ }, throttle) => match throttle { - ThrottleResult::NotThrottled { start } => { - inner.timings = SmgrOpTimerState::ParsedRoutedThrottledNowBatching { - throttle_started_at: *start, - throttle_done_at: *start, - }; - } - ThrottleResult::Throttled { start, end } => { - inner.timings = SmgrOpTimerState::ParsedRoutedThrottledNowBatching { - throttle_started_at: *start, - throttle_done_at: *end, - }; - } - }, - (x, _) => panic!("called in unexpected state: {x:?}"), + pub(crate) fn observe_throttle_start(&mut self, at: Instant) { + let Some(inner) = self.0.as_mut() else { + return; + }; + let SmgrOpTimerState::Received { received_at: _ } = &mut inner.timings else { + return; + }; + inner.throttling.count_accounted_start.inc(); + inner.timings = SmgrOpTimerState::ParsedRoutedNowThrottling { + throttle_started_at: at, + }; + } + pub(crate) fn observe_throttle_done(&mut self, throttle: ThrottleResult) { + let Some(inner) = self.0.as_mut() else { + return; + }; + let SmgrOpTimerState::ParsedRoutedNowThrottling { + throttle_started_at, + } = &mut inner.timings + else { + return; + }; + inner.throttling.count_accounted_finish.inc(); + match throttle { + ThrottleResult::NotThrottled { end } => { + inner.timings = SmgrOpTimerState::ParsedRoutedThrottledNowBatching { + throttle_done_at: end, + }; + } + ThrottleResult::Throttled { end } => { + // update metrics + inner.throttling.count_throttled.inc(); + inner + .throttling + .wait_time + .inc_by((end - *throttle_started_at).as_micros().try_into().unwrap()); + // state transition + inner.timings = SmgrOpTimerState::ParsedRoutedThrottledNowBatching { + throttle_done_at: end, + }; + } } } pub(crate) fn observe_execution_start(&mut self, at: Instant) { - let inner = self.0.as_mut().expect("other public methods consume self"); - match &mut inner.timings { - SmgrOpTimerState::ParsedRoutedThrottledNowBatching { - throttle_started_at, - throttle_done_at, - } => { - inner.timings = SmgrOpTimerState::BatchedNowExecuting { - throttle_started_at: *throttle_started_at, - throttle_done_at: *throttle_done_at, - execution_started_at: at, - }; - } - x => panic!("called in unexpected state: {x:?}"), - } - } - - pub(crate) fn observe_execution_end_flush_start(mut self) -> SmgrOpFlushInProgress { - assert!( - matches!( - self.0.as_ref().unwrap().timings, - SmgrOpTimerState::BatchedNowExecuting { .. } - ), - "called in unexpected state: {:?}", - self.0.as_ref().unwrap().timings, - ); - let (flush_start, inner) = self - .smgr_op_end() - .expect("this method consume self, and the only other caller is drop handler"); - let SmgrOpTimerInner { - global_flush_in_progress_micros, - per_timeline_flush_in_progress_micros, - .. - } = inner; - SmgrOpFlushInProgress { - flush_started_at: flush_start, - global_micros: global_flush_in_progress_micros, - per_timeline_micros: per_timeline_flush_in_progress_micros, - } - } - - /// Returns `None`` if this method has already been called, `Some` otherwise. - fn smgr_op_end(&mut self) -> Option<(Instant, SmgrOpTimerInner)> { - let inner = self.0.take()?; - - let now = Instant::now(); - - // TODO: use label for unfinished requests instead of Duration::ZERO. - // This is quite rare in practice, only during tenant/pageservers shutdown. - let throttle; - let batch; - let execution; - match inner.timings { - SmgrOpTimerState::Received { received_at: _ } => { - throttle = Duration::ZERO; - batch = Duration::ZERO; - execution = Duration::ZERO; - } - SmgrOpTimerState::ParsedRoutedThrottledNowBatching { - throttle_started_at, - throttle_done_at, - } => { - throttle = throttle_done_at - throttle_started_at; - batch = Duration::ZERO; - execution = Duration::ZERO; - } - SmgrOpTimerState::BatchedNowExecuting { - throttle_started_at, - throttle_done_at, - execution_started_at, - } => { - throttle = throttle_done_at - throttle_started_at; - batch = throttle_done_at - execution_started_at; - execution = now - execution_started_at; - } - } - - // update time spent in batching + let Some(inner) = self.0.as_mut() else { + return; + }; + let SmgrOpTimerState::ParsedRoutedThrottledNowBatching { throttle_done_at } = + &mut inner.timings + else { + return; + }; + // update metrics + let batch = at - *throttle_done_at; inner.global_batch_wait_time.observe(batch.as_secs_f64()); inner .per_timeline_batch_wait_time .observe(batch.as_secs_f64()); + // state transition + inner.timings = SmgrOpTimerState::BatchedNowExecuting { + execution_started_at: at, + } + } - // time spent in throttle metric is updated by throttle impl - let _ = throttle; - - // update metrics for execution latency + /// For all but the first caller, this is a no-op. + /// The first callers receives Some, subsequent ones None. + pub(crate) fn observe_execution_end_flush_start( + &mut self, + at: Instant, + ) -> Option { + // NB: unlike the other observe_* methods, this one take()s. + #[allow(clippy::question_mark)] // maintain similar code pattern. + let Some(mut inner) = self.0.take() else { + return None; + }; + let SmgrOpTimerState::BatchedNowExecuting { + execution_started_at, + } = &mut inner.timings + else { + return None; + }; + // update metrics + let execution = at - *execution_started_at; inner .global_execution_latency_histo .observe(execution.as_secs_f64()); @@ -1380,13 +1355,45 @@ impl SmgrOpTimer { per_timeline_execution_latency_histo.observe(execution.as_secs_f64()); } - Some((now, inner)) + // state transition + inner.timings = SmgrOpTimerState::Flushing; + + // return the flush in progress object which + // will do the remaining metrics updates + let SmgrOpTimerInner { + global_flush_in_progress_micros, + per_timeline_flush_in_progress_micros, + .. + } = inner; + Some(SmgrOpFlushInProgress { + flush_started_at: at, + global_micros: global_flush_in_progress_micros, + per_timeline_micros: per_timeline_flush_in_progress_micros, + }) } } +pub(crate) struct SmgrOpFlushInProgress { + flush_started_at: Instant, + global_micros: IntCounter, + per_timeline_micros: IntCounter, +} + impl Drop for SmgrOpTimer { fn drop(&mut self) { - self.smgr_op_end(); + // In case of early drop, update any of the remaining metrics with + // observations so that (started,finished) counter pairs balance out + // and all counters on the latency path have the the same number of + // observations. + // It's technically lying and it would be better if each metric had + // a separate label or similar for cancelled requests. + // But we don't have that right now and counter pairs balancing + // out is useful when using the metrics in panels and whatnot. + let now = Instant::now(); + self.observe_throttle_start(now); + self.observe_throttle_done(ThrottleResult::NotThrottled { end: now }); + self.observe_execution_start(now); + self.observe_execution_end_flush_start(now); } } @@ -1397,12 +1404,12 @@ impl SmgrOpFlushInProgress { { let mut fut = std::pin::pin!(fut); - let now = Instant::now(); // Whenever observe_guard gets called, or dropped, // it adds the time elapsed since its last call to metrics. // Last call is tracked in `now`. let mut observe_guard = scopeguard::guard( || { + let now = Instant::now(); let elapsed = now - self.flush_started_at; self.global_micros .inc_by(u64::try_from(elapsed.as_micros()).unwrap()); @@ -1445,7 +1452,6 @@ pub enum SmgrQueryType { GetSlruSegment, } -#[derive(Debug)] pub(crate) struct SmgrQueryTimePerTimeline { global_started: [IntCounter; SmgrQueryType::COUNT], global_latency: [Histogram; SmgrQueryType::COUNT], @@ -1457,6 +1463,7 @@ pub(crate) struct SmgrQueryTimePerTimeline { per_timeline_flush_in_progress_micros: IntCounter, global_batch_wait_time: Histogram, per_timeline_batch_wait_time: Histogram, + throttling: Arc, } static SMGR_QUERY_STARTED_GLOBAL: Lazy = Lazy::new(|| { @@ -1662,7 +1669,11 @@ static PAGE_SERVICE_SMGR_BATCH_WAIT_TIME_GLOBAL: Lazy = Lazy::new(|| }); impl SmgrQueryTimePerTimeline { - pub(crate) fn new(tenant_shard_id: &TenantShardId, timeline_id: &TimelineId) -> Self { + pub(crate) fn new( + tenant_shard_id: &TenantShardId, + timeline_id: &TimelineId, + pagestream_throttle_metrics: Arc, + ) -> Self { let tenant_id = tenant_shard_id.tenant_id.to_string(); let shard_slug = format!("{}", tenant_shard_id.shard_slug()); let timeline_id = timeline_id.to_string(); @@ -1723,6 +1734,7 @@ impl SmgrQueryTimePerTimeline { per_timeline_flush_in_progress_micros, global_batch_wait_time, per_timeline_batch_wait_time, + throttling: pagestream_throttle_metrics, } } pub(crate) fn start_smgr_op(&self, op: SmgrQueryType, received_at: Instant) -> SmgrOpTimer { @@ -1738,88 +1750,24 @@ impl SmgrQueryTimePerTimeline { SmgrOpTimer(Some(SmgrOpTimerInner { global_execution_latency_histo: self.global_latency[op as usize].clone(), per_timeline_execution_latency_histo: per_timeline_latency_histo, - timings: SmgrOpTimerState::Received { received_at }, global_flush_in_progress_micros: self.global_flush_in_progress_micros.clone(), per_timeline_flush_in_progress_micros: self .per_timeline_flush_in_progress_micros .clone(), global_batch_wait_time: self.global_batch_wait_time.clone(), per_timeline_batch_wait_time: self.per_timeline_batch_wait_time.clone(), + throttling: self.throttling.clone(), + timings: SmgrOpTimerState::Received { received_at }, })) } + /// TODO: do something about this? seems odd, we have a similar call on SmgrOpTimer pub(crate) fn observe_getpage_batch_start(&self, batch_size: usize) { self.global_batch_size.observe(batch_size as f64); self.per_timeline_batch_size.observe(batch_size as f64); } } -#[cfg(test)] -mod smgr_query_time_tests { - use std::time::Instant; - - use pageserver_api::shard::TenantShardId; - use strum::IntoEnumIterator; - use utils::id::{TenantId, TimelineId}; - - // Regression test, we used hard-coded string constants before using an enum. - #[test] - fn op_label_name() { - use super::SmgrQueryType::*; - let expect: [(super::SmgrQueryType, &'static str); 5] = [ - (GetRelExists, "get_rel_exists"), - (GetRelSize, "get_rel_size"), - (GetPageAtLsn, "get_page_at_lsn"), - (GetDbSize, "get_db_size"), - (GetSlruSegment, "get_slru_segment"), - ]; - for (op, expect) in expect { - let actual: &'static str = op.into(); - assert_eq!(actual, expect); - } - } - - #[test] - fn basic() { - let ops: Vec<_> = super::SmgrQueryType::iter().collect(); - - for op in &ops { - let tenant_id = TenantId::generate(); - let timeline_id = TimelineId::generate(); - let metrics = super::SmgrQueryTimePerTimeline::new( - &TenantShardId::unsharded(tenant_id), - &timeline_id, - ); - - let get_counts = || { - let global: u64 = ops - .iter() - .map(|op| metrics.global_latency[*op as usize].get_sample_count()) - .sum(); - ( - global, - metrics.per_timeline_getpage_latency.get_sample_count(), - ) - }; - - let (pre_global, pre_per_tenant_timeline) = get_counts(); - assert_eq!(pre_per_tenant_timeline, 0); - - let timer = metrics.start_smgr_op(*op, Instant::now()); - drop(timer); - - let (post_global, post_per_tenant_timeline) = get_counts(); - if matches!(op, super::SmgrQueryType::GetPageAtLsn) { - // getpage ops are tracked per-timeline, others aren't - assert_eq!(post_per_tenant_timeline, 1); - } else { - assert_eq!(post_per_tenant_timeline, 0); - } - assert!(post_global > pre_global); - } - } -} - // keep in sync with control plane Go code so that we can validate // compute's basebackup_ms metric with our perspective in the context of SLI/SLO. static COMPUTE_STARTUP_BUCKETS: Lazy<[f64; 28]> = Lazy::new(|| { @@ -3606,9 +3554,7 @@ pub(crate) mod tenant_throttling { use once_cell::sync::Lazy; use utils::shard::TenantShardId; - use crate::tenant::{self}; - - struct GlobalAndPerTenantIntCounter { + pub(crate) struct GlobalAndPerTenantIntCounter { global: IntCounter, per_tenant: IntCounter, } @@ -3626,10 +3572,10 @@ pub(crate) mod tenant_throttling { } pub(crate) struct Metrics { - count_accounted_start: GlobalAndPerTenantIntCounter, - count_accounted_finish: GlobalAndPerTenantIntCounter, - wait_time: GlobalAndPerTenantIntCounter, - count_throttled: GlobalAndPerTenantIntCounter, + pub(super) count_accounted_start: GlobalAndPerTenantIntCounter, + pub(super) count_accounted_finish: GlobalAndPerTenantIntCounter, + pub(super) wait_time: GlobalAndPerTenantIntCounter, + pub(super) count_throttled: GlobalAndPerTenantIntCounter, } static COUNT_ACCOUNTED_START: Lazy = Lazy::new(|| { @@ -3764,26 +3710,6 @@ pub(crate) mod tenant_throttling { } } } - - impl tenant::throttle::Metric for Metrics { - #[inline(always)] - fn accounting_start(&self) { - self.count_accounted_start.inc(); - } - #[inline(always)] - fn accounting_finish(&self) { - self.count_accounted_finish.inc(); - } - #[inline(always)] - fn observe_throttling( - &self, - tenant::throttle::Observation { wait_time }: &tenant::throttle::Observation, - ) { - let val = u64::try_from(wait_time.as_micros()).unwrap(); - self.wait_time.inc_by(val); - self.count_throttled.inc(); - } - } } pub(crate) mod disk_usage_based_eviction { diff --git a/pageserver/src/page_service.rs b/pageserver/src/page_service.rs index 17ac4c660b..ff510382d4 100644 --- a/pageserver/src/page_service.rs +++ b/pageserver/src/page_service.rs @@ -688,11 +688,13 @@ impl PageServerHandler { // so that the _started counters are incremented before we do // any serious waiting, e.g., for throttle, batching, or actual request handling. let mut timer = shard.query_metrics.start_smgr_op(op, received_at); + let now = Instant::now(); + timer.observe_throttle_start(now); let throttled = tokio::select! { - res = shard.pagestream_throttle.throttle(1) => res, + res = shard.pagestream_throttle.throttle(1, now) => res, _ = shard.cancel.cancelled() => return Err(QueryError::Shutdown), }; - timer.observe_throttle_done(&throttled); + timer.observe_throttle_done(throttled); Ok(timer) } @@ -1092,7 +1094,11 @@ impl PageServerHandler { // The timer's underlying metric is used for a storage-internal latency SLO and // we don't want to include latency in it that we can't control. // And as pointed out above, in this case, we don't control the time that flush will take. - let flushing_timer = timer.map(|timer| timer.observe_execution_end_flush_start()); + let flushing_timer = timer.map(|mut timer| { + timer + .observe_execution_end_flush_start(Instant::now()) + .expect("we are the first caller") + }); // what we want to do let flush_fut = pgb_writer.flush(); diff --git a/pageserver/src/tenant.rs b/pageserver/src/tenant.rs index e3dab2fc1d..6da6a97b8b 100644 --- a/pageserver/src/tenant.rs +++ b/pageserver/src/tenant.rs @@ -369,8 +369,9 @@ pub struct Tenant { /// Throttle applied at the top of [`Timeline::get`]. /// All [`Tenant::timelines`] of a given [`Tenant`] instance share the same [`throttle::Throttle`] instance. - pub(crate) pagestream_throttle: - Arc>, + pub(crate) pagestream_throttle: Arc, + + pub(crate) pagestream_throttle_metrics: Arc, /// An ongoing timeline detach concurrency limiter. /// @@ -1691,6 +1692,7 @@ impl Tenant { TimelineResources { remote_client, pagestream_throttle: self.pagestream_throttle.clone(), + pagestream_throttle_metrics: self.pagestream_throttle_metrics.clone(), l0_flush_global_state: self.l0_flush_global_state.clone(), }, LoadTimelineCause::Attach, @@ -4084,6 +4086,9 @@ impl Tenant { Ok(timeline) } + /// [`Tenant::shutdown`] must be called before dropping the returned [`Tenant`] object + /// to ensure proper cleanup of background tasks and metrics. + // // Allow too_many_arguments because a constructor's argument list naturally grows with the // number of attributes in the struct: breaking these out into a builder wouldn't be helpful. #[allow(clippy::too_many_arguments)] @@ -4192,8 +4197,10 @@ impl Tenant { gate: Gate::default(), pagestream_throttle: Arc::new(throttle::Throttle::new( Tenant::get_pagestream_throttle_config(conf, &attached_conf.tenant_conf), - crate::metrics::tenant_throttling::Metrics::new(&tenant_shard_id), )), + pagestream_throttle_metrics: Arc::new( + crate::metrics::tenant_throttling::Pagestream::new(&tenant_shard_id), + ), tenant_conf: Arc::new(ArcSwap::from_pointee(attached_conf)), ongoing_timeline_detach: std::sync::Mutex::default(), gc_block: Default::default(), @@ -5100,6 +5107,7 @@ impl Tenant { TimelineResources { remote_client: self.build_timeline_remote_client(timeline_id), pagestream_throttle: self.pagestream_throttle.clone(), + pagestream_throttle_metrics: self.pagestream_throttle_metrics.clone(), l0_flush_global_state: self.l0_flush_global_state.clone(), } } diff --git a/pageserver/src/tenant/throttle.rs b/pageserver/src/tenant/throttle.rs index 8ab6a0e060..300d779125 100644 --- a/pageserver/src/tenant/throttle.rs +++ b/pageserver/src/tenant/throttle.rs @@ -3,7 +3,7 @@ use std::{ atomic::{AtomicU64, Ordering}, Arc, }, - time::{Duration, Instant}, + time::Instant, }; use arc_swap::ArcSwap; @@ -16,9 +16,8 @@ use utils::leaky_bucket::{LeakyBucketConfig, RateLimiter}; /// To share a throttle among multiple entities, wrap it in an [`Arc`]. /// /// The intial use case for this is tenant-wide throttling of getpage@lsn requests. -pub struct Throttle { +pub struct Throttle { inner: ArcSwap, - metric: M, /// will be turned into [`Stats::count_accounted_start`] count_accounted_start: AtomicU64, /// will be turned into [`Stats::count_accounted_finish`] @@ -36,15 +35,6 @@ pub struct Inner { pub type Config = pageserver_api::models::ThrottleConfig; -pub struct Observation { - pub wait_time: Duration, -} -pub trait Metric { - fn accounting_start(&self); - fn accounting_finish(&self); - fn observe_throttling(&self, observation: &Observation); -} - /// See [`Throttle::reset_stats`]. pub struct Stats { /// Number of requests that started [`Throttle::throttle`] calls. @@ -59,18 +49,14 @@ pub struct Stats { } pub enum ThrottleResult { - NotThrottled { start: Instant }, - Throttled { start: Instant, end: Instant }, + NotThrottled { end: Instant }, + Throttled { end: Instant }, } -impl Throttle -where - M: Metric, -{ - pub fn new(config: Config, metric: M) -> Self { +impl Throttle { + pub fn new(config: Config) -> Self { Self { inner: ArcSwap::new(Arc::new(Self::new_inner(config))), - metric, count_accounted_start: AtomicU64::new(0), count_accounted_finish: AtomicU64::new(0), count_throttled: AtomicU64::new(0), @@ -127,32 +113,27 @@ where self.inner.load().rate_limiter.steady_rps() } - pub async fn throttle(&self, key_count: usize) -> ThrottleResult { + /// `start` must be [`Instant::now`] or earlier. + pub async fn throttle(&self, key_count: usize, start: Instant) -> ThrottleResult { let inner = self.inner.load_full(); // clones the `Inner` Arc - let start = std::time::Instant::now(); - if !inner.enabled { - return ThrottleResult::NotThrottled { start }; + return ThrottleResult::NotThrottled { end: start }; } - self.metric.accounting_start(); self.count_accounted_start.fetch_add(1, Ordering::Relaxed); let did_throttle = inner.rate_limiter.acquire(key_count).await; self.count_accounted_finish.fetch_add(1, Ordering::Relaxed); - self.metric.accounting_finish(); if did_throttle { self.count_throttled.fetch_add(1, Ordering::Relaxed); - let now = Instant::now(); - let wait_time = now - start; + let end = Instant::now(); + let wait_time = end - start; self.sum_throttled_usecs .fetch_add(wait_time.as_micros() as u64, Ordering::Relaxed); - let observation = Observation { wait_time }; - self.metric.observe_throttling(&observation); - ThrottleResult::Throttled { start, end: now } + ThrottleResult::Throttled { end } } else { - ThrottleResult::NotThrottled { start } + ThrottleResult::NotThrottled { end: start } } } } diff --git a/pageserver/src/tenant/timeline.rs b/pageserver/src/tenant/timeline.rs index c1b71262e0..e222a624de 100644 --- a/pageserver/src/tenant/timeline.rs +++ b/pageserver/src/tenant/timeline.rs @@ -208,8 +208,8 @@ fn drop_wlock(rlock: tokio::sync::RwLockWriteGuard<'_, T>) { /// The outward-facing resources required to build a Timeline pub struct TimelineResources { pub remote_client: RemoteTimelineClient, - pub pagestream_throttle: - Arc>, + pub pagestream_throttle: Arc, + pub pagestream_throttle_metrics: Arc, pub l0_flush_global_state: l0_flush::L0FlushGlobalState, } @@ -412,8 +412,7 @@ pub struct Timeline { gc_lock: tokio::sync::Mutex<()>, /// Cloned from [`super::Tenant::pagestream_throttle`] on construction. - pub(crate) pagestream_throttle: - Arc>, + pub(crate) pagestream_throttle: Arc, /// Size estimator for aux file v2 pub(crate) aux_file_size_estimator: AuxFileSizeEstimator, @@ -2310,6 +2309,7 @@ impl Timeline { query_metrics: crate::metrics::SmgrQueryTimePerTimeline::new( &tenant_shard_id, &timeline_id, + resources.pagestream_throttle_metrics, ), directory_metrics: array::from_fn(|_| AtomicU64::new(0)), diff --git a/pageserver/src/tenant/timeline/delete.rs b/pageserver/src/tenant/timeline/delete.rs index 47a93b19d2..593eca7385 100644 --- a/pageserver/src/tenant/timeline/delete.rs +++ b/pageserver/src/tenant/timeline/delete.rs @@ -299,6 +299,7 @@ impl DeleteTimelineFlow { TimelineResources { remote_client, pagestream_throttle: tenant.pagestream_throttle.clone(), + pagestream_throttle_metrics: tenant.pagestream_throttle_metrics.clone(), l0_flush_global_state: tenant.l0_flush_global_state.clone(), }, // Important. We dont pass ancestor above because it can be missing. From 58332cb3615954bedb317bbbf311df47310d4a03 Mon Sep 17 00:00:00 2001 From: Christian Schwarz Date: Fri, 10 Jan 2025 21:35:50 +0100 Subject: [PATCH 041/111] pageserver: remove unused metric `pageserver_layers_visited_per_read_global` (#10141) As of commit "pageserver: remove legacy read path" (#8601) we always use vectored get, which has a separate metric. --- pageserver/src/metrics.rs | 10 ---------- test_runner/fixtures/metrics.py | 1 - test_runner/regress/test_compaction.py | 12 +----------- 3 files changed, 1 insertion(+), 22 deletions(-) diff --git a/pageserver/src/metrics.rs b/pageserver/src/metrics.rs index a313a64080..5b8419fda9 100644 --- a/pageserver/src/metrics.rs +++ b/pageserver/src/metrics.rs @@ -91,15 +91,6 @@ pub(crate) static STORAGE_TIME_GLOBAL: Lazy = Lazy::new(|| { .expect("failed to define a metric") }); -pub(crate) static READ_NUM_LAYERS_VISITED: Lazy = Lazy::new(|| { - register_histogram!( - "pageserver_layers_visited_per_read_global", - "Number of layers visited to reconstruct one key", - vec![1.0, 4.0, 8.0, 16.0, 32.0, 64.0, 128.0, 256.0, 512.0, 1024.0], - ) - .expect("failed to define a metric") -}); - pub(crate) static VEC_READ_NUM_LAYERS_VISITED: Lazy = Lazy::new(|| { register_histogram!( "pageserver_layers_visited_per_vectored_read_global", @@ -3894,7 +3885,6 @@ pub fn preinitialize_metrics(conf: &'static PageServerConf) { // histograms [ - &READ_NUM_LAYERS_VISITED, &VEC_READ_NUM_LAYERS_VISITED, &WAIT_LSN_TIME, &WAL_REDO_TIME, diff --git a/test_runner/fixtures/metrics.py b/test_runner/fixtures/metrics.py index c5295360c3..fa541bad17 100644 --- a/test_runner/fixtures/metrics.py +++ b/test_runner/fixtures/metrics.py @@ -131,7 +131,6 @@ PAGESERVER_GLOBAL_METRICS: tuple[str, ...] = ( "pageserver_getpage_reconstruct_seconds_sum", *[f"pageserver_basebackup_query_seconds_{x}" for x in ["bucket", "count", "sum"]], *histogram("pageserver_smgr_query_seconds_global"), - *histogram("pageserver_layers_visited_per_read_global"), *histogram("pageserver_getpage_get_reconstruct_data_seconds"), *histogram("pageserver_wait_lsn_seconds"), *histogram("pageserver_remote_operation_seconds"), diff --git a/test_runner/regress/test_compaction.py b/test_runner/regress/test_compaction.py index ae48a8fc27..fe0422088a 100644 --- a/test_runner/regress/test_compaction.py +++ b/test_runner/regress/test_compaction.py @@ -84,9 +84,6 @@ page_cache_size=10 log.info("Checking layer access metrics ...") layer_access_metric_names = [ - "pageserver_layers_visited_per_read_global_sum", - "pageserver_layers_visited_per_read_global_count", - "pageserver_layers_visited_per_read_global_bucket", "pageserver_layers_visited_per_vectored_read_global_sum", "pageserver_layers_visited_per_vectored_read_global_count", "pageserver_layers_visited_per_vectored_read_global_bucket", @@ -97,12 +94,6 @@ page_cache_size=10 layer_access_metrics = metrics.query_all(name) log.info(f"Got metrics: {layer_access_metrics}") - non_vectored_sum = metrics.query_one("pageserver_layers_visited_per_read_global_sum") - non_vectored_count = metrics.query_one("pageserver_layers_visited_per_read_global_count") - if non_vectored_count.value != 0: - non_vectored_average = non_vectored_sum.value / non_vectored_count.value - else: - non_vectored_average = 0 vectored_sum = metrics.query_one("pageserver_layers_visited_per_vectored_read_global_sum") vectored_count = metrics.query_one("pageserver_layers_visited_per_vectored_read_global_count") if vectored_count.value > 0: @@ -113,11 +104,10 @@ page_cache_size=10 assert vectored_sum.value == 0 vectored_average = 0 - log.info(f"{non_vectored_average=} {vectored_average=}") + log.info(f"{vectored_average=}") # The upper bound for average number of layer visits below (8) # was chosen empirically for this workload. - assert non_vectored_average < 8 assert vectored_average < 8 From b5d54ba52a92f263c087eff56ab7ced9499c1ae6 Mon Sep 17 00:00:00 2001 From: "Alex Chi Z." <4198311+skyzh@users.noreply.github.com> Date: Fri, 10 Jan 2025 15:53:00 -0500 Subject: [PATCH 042/111] refactor(pageserver): move queue logic to compaction.rs (#10330) ## Problem close https://github.com/neondatabase/neon/issues/10031, part of https://github.com/neondatabase/neon/issues/9114 ## Summary of changes Move the compaction job generation to `compaction.rs`, thus making the code more readable and debuggable. We now also return running job through the get compaction job API, versus before we only return scheduled jobs. --------- Signed-off-by: Alex Chi Z --- libs/pageserver_api/src/models.rs | 2 + pageserver/src/http/routes.rs | 14 +- pageserver/src/tenant.rs | 175 +++-------- pageserver/src/tenant/timeline/compaction.rs | 293 ++++++++++++++++++- 4 files changed, 324 insertions(+), 160 deletions(-) diff --git a/libs/pageserver_api/src/models.rs b/libs/pageserver_api/src/models.rs index 39390d7647..9af6c4021d 100644 --- a/libs/pageserver_api/src/models.rs +++ b/libs/pageserver_api/src/models.rs @@ -272,6 +272,8 @@ pub struct CompactInfoResponse { pub compact_key_range: Option, pub compact_lsn_range: Option, pub sub_compaction: bool, + pub running: bool, + pub job_id: usize, } #[derive(Serialize, Deserialize, Clone)] diff --git a/pageserver/src/http/routes.rs b/pageserver/src/http/routes.rs index 60ef4c3702..94e0b101bd 100644 --- a/pageserver/src/http/routes.rs +++ b/pageserver/src/http/routes.rs @@ -97,8 +97,8 @@ use crate::tenant::{LogicalSizeCalculationCause, PageReconstructError}; use crate::DEFAULT_PG_VERSION; use crate::{disk_usage_eviction_task, tenant}; use pageserver_api::models::{ - CompactInfoResponse, StatusResponse, TenantConfigRequest, TenantInfo, TimelineCreateRequest, - TimelineGcRequest, TimelineInfo, + StatusResponse, TenantConfigRequest, TenantInfo, TimelineCreateRequest, TimelineGcRequest, + TimelineInfo, }; use utils::{ auth::SwappableJwtAuth, @@ -2052,15 +2052,7 @@ async fn timeline_compact_info_handler( let tenant = state .tenant_manager .get_attached_tenant_shard(tenant_shard_id)?; - let res = tenant.get_scheduled_compaction_tasks(timeline_id); - let mut resp = Vec::new(); - for item in res { - resp.push(CompactInfoResponse { - compact_key_range: item.compact_key_range, - compact_lsn_range: item.compact_lsn_range, - sub_compaction: item.sub_compaction, - }); - } + let resp = tenant.get_scheduled_compaction_tasks(timeline_id); json_response(StatusCode::OK, resp) } .instrument(info_span!("timeline_compact_info", tenant_id = %tenant_shard_id.tenant_id, shard_id = %tenant_shard_id.shard_slug(), %timeline_id)) diff --git a/pageserver/src/tenant.rs b/pageserver/src/tenant.rs index 8e61d09de7..2928c435cb 100644 --- a/pageserver/src/tenant.rs +++ b/pageserver/src/tenant.rs @@ -21,6 +21,7 @@ use enumset::EnumSet; use futures::stream::FuturesUnordered; use futures::StreamExt; use pageserver_api::models; +use pageserver_api::models::CompactInfoResponse; use pageserver_api::models::LsnLease; use pageserver_api::models::TimelineArchivalState; use pageserver_api::models::TimelineState; @@ -37,21 +38,17 @@ use remote_timeline_client::manifest::{ }; use remote_timeline_client::UploadQueueNotReadyError; use std::collections::BTreeMap; -use std::collections::VecDeque; use std::fmt; use std::future::Future; use std::sync::atomic::AtomicBool; use std::sync::Weak; use std::time::SystemTime; use storage_broker::BrokerClientChannel; -use timeline::compaction::GcCompactJob; -use timeline::compaction::ScheduledCompactionTask; +use timeline::compaction::GcCompactionQueue; use timeline::import_pgdata; use timeline::offload::offload_timeline; use timeline::offload::OffloadError; -use timeline::CompactFlags; use timeline::CompactOptions; -use timeline::CompactionError; use timeline::ShutdownMode; use tokio::io::BufReader; use tokio::sync::watch; @@ -347,10 +344,8 @@ pub struct Tenant { /// Overhead of mutex is acceptable because compaction is done with a multi-second period. compaction_circuit_breaker: std::sync::Mutex, - /// Scheduled compaction tasks. Currently, this can only be populated by triggering - /// a manual gc-compaction from the manual compaction API. - scheduled_compaction_tasks: - std::sync::Mutex>>, + /// Scheduled gc-compaction tasks. + scheduled_compaction_tasks: std::sync::Mutex>>, /// If the tenant is in Activating state, notify this to encourage it /// to proceed to Active as soon as possible, rather than waiting for lazy @@ -2997,104 +2992,18 @@ impl Tenant { if has_pending_l0_compaction_task { Some(true) } else { - let mut has_pending_scheduled_compaction_task; - let next_scheduled_compaction_task = { - let mut guard = self.scheduled_compaction_tasks.lock().unwrap(); - if let Some(tline_pending_tasks) = guard.get_mut(timeline_id) { - if !tline_pending_tasks.is_empty() { - info!( - "{} tasks left in the compaction schedule queue", - tline_pending_tasks.len() - ); - } - let next_task = tline_pending_tasks.pop_front(); - has_pending_scheduled_compaction_task = !tline_pending_tasks.is_empty(); - next_task - } else { - has_pending_scheduled_compaction_task = false; - None - } + let queue = { + let guard = self.scheduled_compaction_tasks.lock().unwrap(); + guard.get(timeline_id).cloned() }; - if let Some(mut next_scheduled_compaction_task) = next_scheduled_compaction_task - { - if !next_scheduled_compaction_task - .options - .flags - .contains(CompactFlags::EnhancedGcBottomMostCompaction) - { - warn!("ignoring scheduled compaction task: scheduled task must be gc compaction: {:?}", next_scheduled_compaction_task.options); - } else if next_scheduled_compaction_task.options.sub_compaction { - info!("running scheduled enhanced gc bottom-most compaction with sub-compaction, splitting compaction jobs"); - let jobs: Vec = timeline - .gc_compaction_split_jobs( - GcCompactJob::from_compact_options( - next_scheduled_compaction_task.options.clone(), - ), - next_scheduled_compaction_task - .options - .sub_compaction_max_job_size_mb, - ) - .await - .map_err(CompactionError::Other)?; - if jobs.is_empty() { - info!("no jobs to run, skipping scheduled compaction task"); - } else { - has_pending_scheduled_compaction_task = true; - let jobs_len = jobs.len(); - let mut guard = self.scheduled_compaction_tasks.lock().unwrap(); - let tline_pending_tasks = guard.entry(*timeline_id).or_default(); - for (idx, job) in jobs.into_iter().enumerate() { - // Unfortunately we need to convert the `GcCompactJob` back to `CompactionOptions` - // until we do further refactors to allow directly call `compact_with_gc`. - let mut flags: EnumSet = EnumSet::default(); - flags |= CompactFlags::EnhancedGcBottomMostCompaction; - if job.dry_run { - flags |= CompactFlags::DryRun; - } - let options = CompactOptions { - flags, - sub_compaction: false, - compact_key_range: Some(job.compact_key_range.into()), - compact_lsn_range: Some(job.compact_lsn_range.into()), - sub_compaction_max_job_size_mb: None, - }; - tline_pending_tasks.push_back(if idx == jobs_len - 1 { - ScheduledCompactionTask { - options, - // The last job in the queue sends the signal and releases the gc guard - result_tx: next_scheduled_compaction_task - .result_tx - .take(), - gc_block: next_scheduled_compaction_task - .gc_block - .take(), - } - } else { - ScheduledCompactionTask { - options, - result_tx: None, - gc_block: None, - } - }); - } - info!("scheduled enhanced gc bottom-most compaction with sub-compaction, split into {} jobs", jobs_len); - } - } else { - let _ = timeline - .compact_with_options( - cancel, - next_scheduled_compaction_task.options, - ctx, - ) - .instrument(info_span!("scheduled_compact_timeline", %timeline_id)) - .await?; - if let Some(tx) = next_scheduled_compaction_task.result_tx.take() { - // TODO: we can send compaction statistics in the future - tx.send(()).ok(); - } - } + if let Some(queue) = queue { + let has_pending_tasks = queue + .iteration(cancel, ctx, &self.gc_block, timeline) + .await?; + Some(has_pending_tasks) + } else { + Some(false) } - Some(has_pending_scheduled_compaction_task) } } else { None @@ -3124,34 +3033,32 @@ impl Tenant { } /// Cancel scheduled compaction tasks - pub(crate) fn cancel_scheduled_compaction( - &self, - timeline_id: TimelineId, - ) -> Vec { + pub(crate) fn cancel_scheduled_compaction(&self, timeline_id: TimelineId) { let mut guard = self.scheduled_compaction_tasks.lock().unwrap(); - if let Some(tline_pending_tasks) = guard.get_mut(&timeline_id) { - let current_tline_pending_tasks = std::mem::take(tline_pending_tasks); - current_tline_pending_tasks.into_iter().collect() - } else { - Vec::new() + if let Some(q) = guard.get_mut(&timeline_id) { + q.cancel_scheduled(); } } pub(crate) fn get_scheduled_compaction_tasks( &self, timeline_id: TimelineId, - ) -> Vec { - use itertools::Itertools; - let guard = self.scheduled_compaction_tasks.lock().unwrap(); - guard - .get(&timeline_id) - .map(|tline_pending_tasks| { - tline_pending_tasks - .iter() - .map(|x| x.options.clone()) - .collect_vec() - }) - .unwrap_or_default() + ) -> Vec { + let res = { + let guard = self.scheduled_compaction_tasks.lock().unwrap(); + guard.get(&timeline_id).map(|q| q.remaining_jobs()) + }; + let Some((running, remaining)) = res else { + return Vec::new(); + }; + let mut result = Vec::new(); + if let Some((id, running)) = running { + result.extend(running.into_compact_info_resp(id, true)); + } + for (id, job) in remaining { + result.extend(job.into_compact_info_resp(id, false)); + } + result } /// Schedule a compaction task for a timeline. @@ -3160,20 +3067,12 @@ impl Tenant { timeline_id: TimelineId, options: CompactOptions, ) -> anyhow::Result> { - let gc_guard = match self.gc_block.start().await { - Ok(guard) => guard, - Err(e) => { - bail!("cannot run gc-compaction because gc is blocked: {}", e); - } - }; let (tx, rx) = tokio::sync::oneshot::channel(); let mut guard = self.scheduled_compaction_tasks.lock().unwrap(); - let tline_pending_tasks = guard.entry(timeline_id).or_default(); - tline_pending_tasks.push_back(ScheduledCompactionTask { - options, - result_tx: Some(tx), - gc_block: Some(gc_guard), - }); + let q = guard + .entry(timeline_id) + .or_insert_with(|| Arc::new(GcCompactionQueue::new())); + q.schedule_manual_compaction(options, Some(tx)); Ok(rx) } diff --git a/pageserver/src/tenant/timeline/compaction.rs b/pageserver/src/tenant/timeline/compaction.rs index 55cde8603e..05f8d476f9 100644 --- a/pageserver/src/tenant/timeline/compaction.rs +++ b/pageserver/src/tenant/timeline/compaction.rs @@ -4,7 +4,7 @@ //! //! The old legacy algorithm is implemented directly in `timeline.rs`. -use std::collections::{BinaryHeap, HashMap, HashSet}; +use std::collections::{BinaryHeap, HashMap, HashSet, VecDeque}; use std::ops::{Deref, Range}; use std::sync::Arc; @@ -16,10 +16,12 @@ use super::{ use anyhow::{anyhow, bail, Context}; use bytes::Bytes; +use enumset::EnumSet; use fail::fail_point; use itertools::Itertools; use pageserver_api::key::KEY_SIZE; use pageserver_api::keyspace::ShardedRange; +use pageserver_api::models::CompactInfoResponse; use pageserver_api::shard::{ShardCount, ShardIdentity, TenantShardId}; use serde::Serialize; use tokio_util::sync::CancellationToken; @@ -30,6 +32,7 @@ use crate::context::{AccessStatsBehavior, RequestContext, RequestContextBuilder} use crate::page_cache; use crate::statvfs::Statvfs; use crate::tenant::checks::check_valid_layermap; +use crate::tenant::gc_block::GcBlock; use crate::tenant::remote_timeline_client::WaitCompletionError; use crate::tenant::storage_layer::batch_split_writer::{ BatchWriterResult, SplitDeltaLayerWriter, SplitImageLayerWriter, @@ -63,16 +66,284 @@ use super::CompactionError; /// Maximum number of deltas before generating an image layer in bottom-most compaction. const COMPACTION_DELTA_THRESHOLD: usize = 5; -/// A scheduled compaction task. -pub(crate) struct ScheduledCompactionTask { - /// It's unfortunate that we need to store a compact options struct here because the only outer - /// API we can call here is `compact_with_options` which does a few setup calls before starting the - /// actual compaction job... We should refactor this to store `GcCompactionJob` in the future. - pub options: CompactOptions, - /// The channel to send the compaction result. If this is a subcompaction, the last compaction job holds the sender. - pub result_tx: Option>, - /// Hold the GC block. If this is a subcompaction, the last compaction job holds the gc block guard. - pub gc_block: Option, +#[derive(Debug, Clone, Copy, Hash, PartialEq, Eq)] +pub struct GcCompactionJobId(pub usize); + +impl std::fmt::Display for GcCompactionJobId { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", self.0) + } +} + +#[derive(Debug, Clone)] +pub enum GcCompactionQueueItem { + Manual(CompactOptions), + SubCompactionJob(CompactOptions), + #[allow(dead_code)] + UpdateL2Lsn(Lsn), + Notify(GcCompactionJobId), +} + +impl GcCompactionQueueItem { + pub fn into_compact_info_resp( + self, + id: GcCompactionJobId, + running: bool, + ) -> Option { + match self { + GcCompactionQueueItem::Manual(options) => Some(CompactInfoResponse { + compact_key_range: options.compact_key_range, + compact_lsn_range: options.compact_lsn_range, + sub_compaction: options.sub_compaction, + running, + job_id: id.0, + }), + GcCompactionQueueItem::SubCompactionJob(options) => Some(CompactInfoResponse { + compact_key_range: options.compact_key_range, + compact_lsn_range: options.compact_lsn_range, + sub_compaction: options.sub_compaction, + running, + job_id: id.0, + }), + GcCompactionQueueItem::UpdateL2Lsn(_) => None, + GcCompactionQueueItem::Notify(_) => None, + } + } +} + +struct GcCompactionQueueInner { + running: Option<(GcCompactionJobId, GcCompactionQueueItem)>, + queued: VecDeque<(GcCompactionJobId, GcCompactionQueueItem)>, + notify: HashMap>, + gc_guards: HashMap, + last_id: GcCompactionJobId, +} + +impl GcCompactionQueueInner { + fn next_id(&mut self) -> GcCompactionJobId { + let id = self.last_id; + self.last_id = GcCompactionJobId(id.0 + 1); + id + } +} + +/// A structure to store gc_compaction jobs. +pub struct GcCompactionQueue { + /// All items in the queue, and the currently-running job. + inner: std::sync::Mutex, + /// Ensure only one thread is consuming the queue. + consumer_lock: tokio::sync::Mutex<()>, +} + +impl GcCompactionQueue { + pub fn new() -> Self { + GcCompactionQueue { + inner: std::sync::Mutex::new(GcCompactionQueueInner { + running: None, + queued: VecDeque::new(), + notify: HashMap::new(), + gc_guards: HashMap::new(), + last_id: GcCompactionJobId(0), + }), + consumer_lock: tokio::sync::Mutex::new(()), + } + } + + pub fn cancel_scheduled(&self) { + let mut guard = self.inner.lock().unwrap(); + guard.queued.clear(); + guard.notify.clear(); + guard.gc_guards.clear(); + } + + /// Schedule a manual compaction job. + pub fn schedule_manual_compaction( + &self, + options: CompactOptions, + notify: Option>, + ) -> GcCompactionJobId { + let mut guard = self.inner.lock().unwrap(); + let id = guard.next_id(); + guard + .queued + .push_back((id, GcCompactionQueueItem::Manual(options))); + if let Some(notify) = notify { + guard.notify.insert(id, notify); + } + info!("scheduled compaction job id={}", id); + id + } + + /// Trigger an auto compaction. + #[allow(dead_code)] + pub fn trigger_auto_compaction(&self, _: &Arc) {} + + /// Notify the caller the job has finished and unblock GC. + fn notify_and_unblock(&self, id: GcCompactionJobId) { + info!("compaction job id={} finished", id); + let mut guard = self.inner.lock().unwrap(); + if let Some(blocking) = guard.gc_guards.remove(&id) { + drop(blocking) + } + if let Some(tx) = guard.notify.remove(&id) { + let _ = tx.send(()); + } + } + + async fn handle_sub_compaction( + &self, + id: GcCompactionJobId, + options: CompactOptions, + timeline: &Arc, + gc_block: &GcBlock, + ) -> Result<(), CompactionError> { + info!("running scheduled enhanced gc bottom-most compaction with sub-compaction, splitting compaction jobs"); + let jobs: Vec = timeline + .gc_compaction_split_jobs( + GcCompactJob::from_compact_options(options.clone()), + options.sub_compaction_max_job_size_mb, + ) + .await + .map_err(CompactionError::Other)?; + if jobs.is_empty() { + info!("no jobs to run, skipping scheduled compaction task"); + self.notify_and_unblock(id); + } else { + let gc_guard = match gc_block.start().await { + Ok(guard) => guard, + Err(e) => { + return Err(CompactionError::Other(anyhow!( + "cannot run gc-compaction because gc is blocked: {}", + e + ))); + } + }; + + let jobs_len = jobs.len(); + let mut pending_tasks = Vec::new(); + for job in jobs { + // Unfortunately we need to convert the `GcCompactJob` back to `CompactionOptions` + // until we do further refactors to allow directly call `compact_with_gc`. + let mut flags: EnumSet = EnumSet::default(); + flags |= CompactFlags::EnhancedGcBottomMostCompaction; + if job.dry_run { + flags |= CompactFlags::DryRun; + } + let options = CompactOptions { + flags, + sub_compaction: false, + compact_key_range: Some(job.compact_key_range.into()), + compact_lsn_range: Some(job.compact_lsn_range.into()), + sub_compaction_max_job_size_mb: None, + }; + pending_tasks.push(GcCompactionQueueItem::SubCompactionJob(options)); + } + pending_tasks.push(GcCompactionQueueItem::Notify(id)); + { + let mut guard = self.inner.lock().unwrap(); + guard.gc_guards.insert(id, gc_guard); + let mut tasks = Vec::new(); + for task in pending_tasks { + let id = guard.next_id(); + tasks.push((id, task)); + } + tasks.reverse(); + for item in tasks { + guard.queued.push_front(item); + } + } + info!("scheduled enhanced gc bottom-most compaction with sub-compaction, split into {} jobs", jobs_len); + } + Ok(()) + } + + /// Take a job from the queue and process it. Returns if there are still pending tasks. + pub async fn iteration( + &self, + cancel: &CancellationToken, + ctx: &RequestContext, + gc_block: &GcBlock, + timeline: &Arc, + ) -> Result { + let _one_op_at_a_time_guard = self.consumer_lock.lock().await; + let has_pending_tasks; + let (id, item) = { + let mut guard = self.inner.lock().unwrap(); + let Some((id, item)) = guard.queued.pop_front() else { + return Ok(false); + }; + guard.running = Some((id, item.clone())); + has_pending_tasks = !guard.queued.is_empty(); + (id, item) + }; + + match item { + GcCompactionQueueItem::Manual(options) => { + if !options + .flags + .contains(CompactFlags::EnhancedGcBottomMostCompaction) + { + warn!("ignoring scheduled compaction task: scheduled task must be gc compaction: {:?}", options); + } else if options.sub_compaction { + self.handle_sub_compaction(id, options, timeline, gc_block) + .await?; + } else { + let gc_guard = match gc_block.start().await { + Ok(guard) => guard, + Err(e) => { + return Err(CompactionError::Other(anyhow!( + "cannot run gc-compaction because gc is blocked: {}", + e + ))); + } + }; + { + let mut guard = self.inner.lock().unwrap(); + guard.gc_guards.insert(id, gc_guard); + } + let _ = timeline + .compact_with_options(cancel, options, ctx) + .instrument(info_span!("scheduled_compact_timeline", %timeline.timeline_id)) + .await?; + self.notify_and_unblock(id); + } + } + GcCompactionQueueItem::SubCompactionJob(options) => { + let _ = timeline + .compact_with_options(cancel, options, ctx) + .instrument(info_span!("scheduled_compact_timeline", %timeline.timeline_id)) + .await?; + } + GcCompactionQueueItem::Notify(id) => { + self.notify_and_unblock(id); + } + GcCompactionQueueItem::UpdateL2Lsn(_) => { + unreachable!() + } + } + { + let mut guard = self.inner.lock().unwrap(); + guard.running = None; + } + Ok(has_pending_tasks) + } + + #[allow(clippy::type_complexity)] + pub fn remaining_jobs( + &self, + ) -> ( + Option<(GcCompactionJobId, GcCompactionQueueItem)>, + VecDeque<(GcCompactionJobId, GcCompactionQueueItem)>, + ) { + let guard = self.inner.lock().unwrap(); + (guard.running.clone(), guard.queued.clone()) + } + + #[allow(dead_code)] + pub fn remaining_jobs_num(&self) -> usize { + let guard = self.inner.lock().unwrap(); + guard.queued.len() + if guard.running.is_some() { 1 } else { 0 } + } } /// A job description for the gc-compaction job. This structure describes the rectangle range that the job will From 23c0748cdd3d07c67f805d29281b1e0aea967a4f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arpad=20M=C3=BCller?= Date: Sat, 11 Jan 2025 03:52:45 +0100 Subject: [PATCH 043/111] Remove active column (#10335) We don't need or want the `active` column. Remove it. Vlad pointed out that this is safe. Thanks to the separation of the schemata in earlier PRs, this is easy. follow-up of #10205 Part of https://github.com/neondatabase/neon/issues/9981 --- .../2025-01-09-160454_safekeepers_remove_active/down.sql | 4 ++++ .../2025-01-09-160454_safekeepers_remove_active/up.sql | 1 + storage_controller/src/persistence.rs | 7 ++----- storage_controller/src/schema.rs | 1 - 4 files changed, 7 insertions(+), 6 deletions(-) create mode 100644 storage_controller/migrations/2025-01-09-160454_safekeepers_remove_active/down.sql create mode 100644 storage_controller/migrations/2025-01-09-160454_safekeepers_remove_active/up.sql diff --git a/storage_controller/migrations/2025-01-09-160454_safekeepers_remove_active/down.sql b/storage_controller/migrations/2025-01-09-160454_safekeepers_remove_active/down.sql new file mode 100644 index 0000000000..c2624f858b --- /dev/null +++ b/storage_controller/migrations/2025-01-09-160454_safekeepers_remove_active/down.sql @@ -0,0 +1,4 @@ +-- this sadly isn't a "true" revert of the migration, as the column is now at the end of the table. +-- But preserving order is not a trivial operation. +-- https://wiki.postgresql.org/wiki/Alter_column_position +ALTER TABLE safekeepers ADD active BOOLEAN NOT NULL DEFAULT false; diff --git a/storage_controller/migrations/2025-01-09-160454_safekeepers_remove_active/up.sql b/storage_controller/migrations/2025-01-09-160454_safekeepers_remove_active/up.sql new file mode 100644 index 0000000000..d76f044eda --- /dev/null +++ b/storage_controller/migrations/2025-01-09-160454_safekeepers_remove_active/up.sql @@ -0,0 +1 @@ +ALTER TABLE safekeepers DROP active; diff --git a/storage_controller/src/persistence.rs b/storage_controller/src/persistence.rs index cebf3e9594..beb014f0a8 100644 --- a/storage_controller/src/persistence.rs +++ b/storage_controller/src/persistence.rs @@ -1258,7 +1258,6 @@ pub(crate) struct SafekeeperPersistence { pub(crate) version: i64, pub(crate) host: String, pub(crate) port: i32, - pub(crate) active: bool, pub(crate) http_port: i32, pub(crate) availability_zone_id: String, pub(crate) scheduling_policy: String, @@ -1270,7 +1269,6 @@ impl SafekeeperPersistence { SkSchedulingPolicy::from_str(&self.scheduling_policy).map_err(|e| { DatabaseError::Logical(format!("can't construct SkSchedulingPolicy: {e:?}")) })?; - // omit the `active` flag on purpose: it is deprecated. Ok(SafekeeperDescribeResponse { id: NodeId(self.id as u64), region_id: self.region_id.clone(), @@ -1295,7 +1293,8 @@ pub(crate) struct SafekeeperUpsert { pub(crate) version: i64, pub(crate) host: String, pub(crate) port: i32, - pub(crate) active: bool, + /// The active flag will not be stored in the database and will be ignored. + pub(crate) active: Option, pub(crate) http_port: i32, pub(crate) availability_zone_id: String, } @@ -1311,7 +1310,6 @@ impl SafekeeperUpsert { version: self.version, host: &self.host, port: self.port, - active: self.active, http_port: self.http_port, availability_zone_id: &self.availability_zone_id, // None means a wish to not update this column. We expose abilities to update it via other means. @@ -1328,7 +1326,6 @@ struct InsertUpdateSafekeeper<'a> { version: i64, host: &'a str, port: i32, - active: bool, http_port: i32, availability_zone_id: &'a str, scheduling_policy: Option<&'a str>, diff --git a/storage_controller/src/schema.rs b/storage_controller/src/schema.rs index 44c91619ab..14c30c296d 100644 --- a/storage_controller/src/schema.rs +++ b/storage_controller/src/schema.rs @@ -36,7 +36,6 @@ diesel::table! { version -> Int8, host -> Text, port -> Int4, - active -> Bool, http_port -> Int4, availability_zone_id -> Text, scheduling_policy -> Varchar, From 70a3bf37a0d2df87ba4bab0f5d73466ecbabfb90 Mon Sep 17 00:00:00 2001 From: Heikki Linnakangas Date: Sat, 11 Jan 2025 15:09:55 +0200 Subject: [PATCH 044/111] Stop building 'compute-tools' image (#10333) It's been unused from time immemorial. --------- Co-authored-by: Matthias van de Meent --- .github/workflows/build_and_test.yml | 47 ++-------------------------- compute/compute-node.Dockerfile | 11 ------- compute_tools/src/bin/fast_import.rs | 2 +- docs/docker.md | 6 +--- 4 files changed, 5 insertions(+), 61 deletions(-) diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index 01f5c3ede9..cd95a5b16d 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -728,30 +728,6 @@ jobs: tags: | neondatabase/neon-test-extensions-${{ matrix.version.pg }}:${{needs.tag.outputs.build-tag}}-${{ matrix.version.debian }}-${{ matrix.arch }} - - name: Build compute-tools image - # compute-tools are Postgres independent, so build it only once - # We pick 16, because that builds on debian 11 with older glibc (and is - # thus compatible with newer glibc), rather than 17 on Debian 12, as - # that isn't guaranteed to be compatible with Debian 11 - if: matrix.version.pg == 'v16' - uses: docker/build-push-action@v6 - with: - target: compute-tools-image - context: . - build-args: | - GIT_VERSION=${{ github.event.pull_request.head.sha || github.sha }} - BUILD_TAG=${{ needs.tag.outputs.build-tag }} - TAG=${{ needs.build-build-tools-image.outputs.image-tag }}-${{ matrix.version.debian }} - DEBIAN_VERSION=${{ matrix.version.debian }} - provenance: false - push: true - pull: true - file: compute/compute-node.Dockerfile - cache-from: type=registry,ref=cache.neon.build/compute-node-${{ matrix.version.pg }}:cache-${{ matrix.version.debian }}-${{ matrix.arch }} - cache-to: ${{ github.ref_name == 'main' && format('type=registry,ref=cache.neon.build/compute-tools-{0}:cache-{1}-{2},mode=max', matrix.version.pg, matrix.version.debian, matrix.arch) || '' }} - tags: | - neondatabase/compute-tools:${{ needs.tag.outputs.build-tag }}-${{ matrix.version.debian }}-${{ matrix.arch }} - compute-node-image: needs: [ compute-node-image-arch, tag ] permissions: @@ -794,14 +770,6 @@ jobs: neondatabase/neon-test-extensions-${{ matrix.version.pg }}:${{ needs.tag.outputs.build-tag }}-${{ matrix.version.debian }}-x64 \ neondatabase/neon-test-extensions-${{ matrix.version.pg }}:${{ needs.tag.outputs.build-tag }}-${{ matrix.version.debian }}-arm64 - - name: Create multi-arch compute-tools image - if: matrix.version.pg == 'v16' - run: | - docker buildx imagetools create -t neondatabase/compute-tools:${{ needs.tag.outputs.build-tag }} \ - -t neondatabase/compute-tools:${{ needs.tag.outputs.build-tag }}-${{ matrix.version.debian }} \ - neondatabase/compute-tools:${{ needs.tag.outputs.build-tag }}-${{ matrix.version.debian }}-x64 \ - neondatabase/compute-tools:${{ needs.tag.outputs.build-tag }}-${{ matrix.version.debian }}-arm64 - - name: Configure AWS credentials uses: aws-actions/configure-aws-credentials@v4 with: @@ -817,12 +785,6 @@ jobs: docker buildx imagetools create -t 369495373322.dkr.ecr.eu-central-1.amazonaws.com/compute-node-${{ matrix.version.pg }}:${{ needs.tag.outputs.build-tag }} \ neondatabase/compute-node-${{ matrix.version.pg }}:${{ needs.tag.outputs.build-tag }} - - name: Push multi-arch compute-tools image to ECR - if: matrix.version.pg == 'v16' - run: | - docker buildx imagetools create -t 369495373322.dkr.ecr.eu-central-1.amazonaws.com/compute-tools:${{ needs.tag.outputs.build-tag }} \ - neondatabase/compute-tools:${{ needs.tag.outputs.build-tag }} - vm-compute-node-image: needs: [ check-permissions, tag, compute-node-image ] runs-on: [ self-hosted, large ] @@ -1001,9 +963,6 @@ jobs: docker buildx imagetools create -t $repo/neon:latest \ $repo/neon:${{ needs.tag.outputs.build-tag }} - docker buildx imagetools create -t $repo/compute-tools:latest \ - $repo/compute-tools:${{ needs.tag.outputs.build-tag }} - for version in ${VERSIONS}; do docker buildx imagetools create -t $repo/compute-node-${version}:latest \ $repo/compute-node-${version}:${{ needs.tag.outputs.build-tag }} @@ -1032,7 +991,7 @@ jobs: - name: Copy all images to prod ECR if: github.ref_name == 'release' || github.ref_name == 'release-proxy' || github.ref_name == 'release-compute' run: | - for image in neon compute-tools {vm-,}compute-node-{v14,v15,v16,v17}; do + for image in neon {vm-,}compute-node-{v14,v15,v16,v17}; do docker buildx imagetools create -t 093970136003.dkr.ecr.eu-central-1.amazonaws.com/${image}:${{ needs.tag.outputs.build-tag }} \ 369495373322.dkr.ecr.eu-central-1.amazonaws.com/${image}:${{ needs.tag.outputs.build-tag }} done @@ -1044,7 +1003,7 @@ jobs: with: client_id: ${{ vars.AZURE_DEV_CLIENT_ID }} image_tag: ${{ needs.tag.outputs.build-tag }} - images: neon compute-tools vm-compute-node-v14 vm-compute-node-v15 vm-compute-node-v16 vm-compute-node-v17 compute-node-v14 compute-node-v15 compute-node-v16 compute-node-v17 + images: neon vm-compute-node-v14 vm-compute-node-v15 vm-compute-node-v16 vm-compute-node-v17 compute-node-v14 compute-node-v15 compute-node-v16 compute-node-v17 registry_name: ${{ vars.AZURE_DEV_REGISTRY_NAME }} subscription_id: ${{ vars.AZURE_DEV_SUBSCRIPTION_ID }} tenant_id: ${{ vars.AZURE_TENANT_ID }} @@ -1056,7 +1015,7 @@ jobs: with: client_id: ${{ vars.AZURE_PROD_CLIENT_ID }} image_tag: ${{ needs.tag.outputs.build-tag }} - images: neon compute-tools vm-compute-node-v14 vm-compute-node-v15 vm-compute-node-v16 vm-compute-node-v17 compute-node-v14 compute-node-v15 compute-node-v16 compute-node-v17 + images: neon vm-compute-node-v14 vm-compute-node-v15 vm-compute-node-v16 vm-compute-node-v17 compute-node-v14 compute-node-v15 compute-node-v16 compute-node-v17 registry_name: ${{ vars.AZURE_PROD_REGISTRY_NAME }} subscription_id: ${{ vars.AZURE_PROD_SUBSCRIPTION_ID }} tenant_id: ${{ vars.AZURE_TENANT_ID }} diff --git a/compute/compute-node.Dockerfile b/compute/compute-node.Dockerfile index 303daec240..255dafa401 100644 --- a/compute/compute-node.Dockerfile +++ b/compute/compute-node.Dockerfile @@ -1288,17 +1288,6 @@ USER nonroot COPY --chown=nonroot . . RUN mold -run cargo build --locked --profile release-line-debug-size-lto --bin compute_ctl --bin fast_import --bin local_proxy -######################################################################################### -# -# Final compute-tools image -# -######################################################################################### - -FROM debian:$DEBIAN_FLAVOR AS compute-tools-image - -COPY --from=compute-tools /home/nonroot/target/release-line-debug-size-lto/compute_ctl /usr/local/bin/compute_ctl -COPY --from=compute-tools /home/nonroot/target/release-line-debug-size-lto/fast_import /usr/local/bin/fast_import - ######################################################################################### # # Layer "pgbouncer" diff --git a/compute_tools/src/bin/fast_import.rs b/compute_tools/src/bin/fast_import.rs index 793ec4cf10..f554362751 100644 --- a/compute_tools/src/bin/fast_import.rs +++ b/compute_tools/src/bin/fast_import.rs @@ -17,7 +17,7 @@ //! //! # Local Testing //! -//! - Comment out most of the pgxns in The Dockerfile.compute-tools to speed up the build. +//! - Comment out most of the pgxns in compute-node.Dockerfile to speed up the build. //! - Build the image with the following command: //! //! ```bash diff --git a/docs/docker.md b/docs/docker.md index 0914a00082..ae74c2b2ab 100644 --- a/docs/docker.md +++ b/docs/docker.md @@ -7,15 +7,11 @@ Currently we build two main images: - [neondatabase/neon](https://hub.docker.com/repository/docker/neondatabase/neon) — image with pre-built `pageserver`, `safekeeper` and `proxy` binaries and all the required runtime dependencies. Built from [/Dockerfile](/Dockerfile). - [neondatabase/compute-node-v16](https://hub.docker.com/repository/docker/neondatabase/compute-node-v16) — compute node image with pre-built Postgres binaries from [neondatabase/postgres](https://github.com/neondatabase/postgres). Similar images exist for v15 and v14. Built from [/compute-node/Dockerfile](/compute/compute-node.Dockerfile). -And additional intermediate image: - -- [neondatabase/compute-tools](https://hub.docker.com/repository/docker/neondatabase/compute-tools) — compute node configuration management tools. - ## Build pipeline We build all images after a successful `release` tests run and push automatically to Docker Hub with two parallel CI jobs -1. `neondatabase/compute-tools` and `neondatabase/compute-node-v16` (and -v15 and -v14) +1. `neondatabase/compute-node-v17` (and -16, -v15, -v14) 2. `neondatabase/neon` From 846e8fdce4a9e6a06941c865575350343007c2a4 Mon Sep 17 00:00:00 2001 From: Heikki Linnakangas Date: Sat, 11 Jan 2025 16:20:50 +0200 Subject: [PATCH 045/111] Remove obsolete hnsw extension (#8008) This has been deprecated and disabled for new installations for a long time. Let's remove it for good. --- compute/compute-node.Dockerfile | 14 - pgxn/hnsw/Makefile | 26 -- pgxn/hnsw/README.md | 25 -- pgxn/hnsw/hnsw--0.1.0.sql | 29 -- pgxn/hnsw/hnsw.c | 590 -------------------------------- pgxn/hnsw/hnsw.control | 4 - pgxn/hnsw/hnsw.h | 15 - pgxn/hnsw/hnswalg.cpp | 379 -------------------- pgxn/hnsw/hnswalg.h | 69 ---- pgxn/hnsw/test/expected/knn.out | 28 -- pgxn/hnsw/test/sql/knn.sql | 13 - 11 files changed, 1192 deletions(-) delete mode 100644 pgxn/hnsw/Makefile delete mode 100644 pgxn/hnsw/README.md delete mode 100644 pgxn/hnsw/hnsw--0.1.0.sql delete mode 100644 pgxn/hnsw/hnsw.c delete mode 100644 pgxn/hnsw/hnsw.control delete mode 100644 pgxn/hnsw/hnsw.h delete mode 100644 pgxn/hnsw/hnswalg.cpp delete mode 100644 pgxn/hnsw/hnswalg.h delete mode 100644 pgxn/hnsw/test/expected/knn.out delete mode 100644 pgxn/hnsw/test/sql/knn.sql diff --git a/compute/compute-node.Dockerfile b/compute/compute-node.Dockerfile index 255dafa401..f4df507b74 100644 --- a/compute/compute-node.Dockerfile +++ b/compute/compute-node.Dockerfile @@ -1258,20 +1258,6 @@ RUN make -j $(getconf _NPROCESSORS_ONLN) \ make -j $(getconf _NPROCESSORS_ONLN) \ PG_CONFIG=/usr/local/pgsql/bin/pg_config \ -C pgxn/neon_rmgr \ - -s install && \ - case "${PG_VERSION}" in \ - "v14" | "v15") \ - ;; \ - "v16" | "v17") \ - echo "Skipping HNSW for PostgreSQL ${PG_VERSION}" && exit 0 \ - ;; \ - *) \ - echo "unexpected PostgreSQL version" && exit 1 \ - ;; \ - esac && \ - make -j $(getconf _NPROCESSORS_ONLN) \ - PG_CONFIG=/usr/local/pgsql/bin/pg_config \ - -C pgxn/hnsw \ -s install ######################################################################################### diff --git a/pgxn/hnsw/Makefile b/pgxn/hnsw/Makefile deleted file mode 100644 index 66436b5920..0000000000 --- a/pgxn/hnsw/Makefile +++ /dev/null @@ -1,26 +0,0 @@ -EXTENSION = hnsw -EXTVERSION = 0.1.0 - -MODULE_big = hnsw -DATA = $(wildcard *--*.sql) -OBJS = hnsw.o hnswalg.o - -TESTS = $(wildcard test/sql/*.sql) -REGRESS = $(patsubst test/sql/%.sql,%,$(TESTS)) -REGRESS_OPTS = --inputdir=test --load-extension=hnsw - -# For auto-vectorization: -# - GCC (needs -ftree-vectorize OR -O3) - https://gcc.gnu.org/projects/tree-ssa/vectorization.html -PG_CFLAGS += -O3 -PG_CXXFLAGS += -O3 -std=c++11 -PG_LDFLAGS += -lstdc++ - -all: $(EXTENSION)--$(EXTVERSION).sql - -PG_CONFIG ?= pg_config -PGXS := $(shell $(PG_CONFIG) --pgxs) -include $(PGXS) - -dist: - mkdir -p dist - git archive --format zip --prefix=$(EXTENSION)-$(EXTVERSION)/ --output dist/$(EXTENSION)-$(EXTVERSION).zip master diff --git a/pgxn/hnsw/README.md b/pgxn/hnsw/README.md deleted file mode 100644 index bc9c8d571c..0000000000 --- a/pgxn/hnsw/README.md +++ /dev/null @@ -1,25 +0,0 @@ -# Revisiting the Inverted Indices for Billion-Scale Approximate Nearest Neighbors - -This ANN extension of Postgres is based -on [ivf-hnsw](https://github.com/dbaranchuk/ivf-hnsw.git) implementation of [HNSW](https://www.pinecone.io/learn/hnsw), -the code for the current state-of-the-art billion-scale nearest neighbor search system presented in the paper: - -[Revisiting the Inverted Indices for Billion-Scale Approximate Nearest Neighbors](http://openaccess.thecvf.com/content_ECCV_2018/html/Dmitry_Baranchuk_Revisiting_the_Inverted_ECCV_2018_paper.html), -
-Dmitry Baranchuk, Artem Babenko, Yury Malkov - -# Postgres extension - -HNSW index is hold in memory (built on demand) and it's maxial size is limited -by `maxelements` index parameter. Another required parameter is nubmer of dimensions (if it is not specified in column type). -Optional parameter `ef` specifies number of neighbors which are considered during index construction and search (corresponds `efConstruction` and `efSearch` parameters -described in the article). - -# Example of usage: - -``` -create extension hnsw; -create table embeddings(id integer primary key, payload real[]); -create index on embeddings using hnsw(payload) with (maxelements=1000000, dims=100, m=32); -select id from embeddings order by payload <-> array[1.0, 2.0,...] limit 100; -``` \ No newline at end of file diff --git a/pgxn/hnsw/hnsw--0.1.0.sql b/pgxn/hnsw/hnsw--0.1.0.sql deleted file mode 100644 index ebf424326d..0000000000 --- a/pgxn/hnsw/hnsw--0.1.0.sql +++ /dev/null @@ -1,29 +0,0 @@ --- complain if script is sourced in psql, rather than via CREATE EXTENSION -\echo Use "CREATE EXTENSION hnsw" to load this file. \quit - --- functions - -CREATE FUNCTION l2_distance(real[], real[]) RETURNS real - AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE; - --- operators - -CREATE OPERATOR <-> ( - LEFTARG = real[], RIGHTARG = real[], PROCEDURE = l2_distance, - COMMUTATOR = '<->' -); - --- access method - -CREATE FUNCTION hnsw_handler(internal) RETURNS index_am_handler - AS 'MODULE_PATHNAME' LANGUAGE C; - -CREATE ACCESS METHOD hnsw TYPE INDEX HANDLER hnsw_handler; - -COMMENT ON ACCESS METHOD hnsw IS 'hnsw index access method'; - --- opclasses - -CREATE OPERATOR CLASS knn_ops - DEFAULT FOR TYPE real[] USING hnsw AS - OPERATOR 1 <-> (real[], real[]) FOR ORDER BY float_ops; diff --git a/pgxn/hnsw/hnsw.c b/pgxn/hnsw/hnsw.c deleted file mode 100644 index e624cb831f..0000000000 --- a/pgxn/hnsw/hnsw.c +++ /dev/null @@ -1,590 +0,0 @@ -#include "postgres.h" - -#include "access/amapi.h" -#include "access/generic_xlog.h" -#include "access/relation.h" -#include "access/reloptions.h" -#include "access/tableam.h" -#include "catalog/index.h" -#include "commands/vacuum.h" -#include "nodes/execnodes.h" -#include "storage/bufmgr.h" -#include "utils/guc.h" -#include "utils/selfuncs.h" - -#include -#include - -#include "hnsw.h" - -PG_MODULE_MAGIC; - -typedef struct { - int32 vl_len_; /* varlena header (do not touch directly!) */ - int dims; - int maxelements; - int efConstruction; - int efSearch; - int M; -} HnswOptions; - -static relopt_kind hnsw_relopt_kind; - -typedef struct { - HierarchicalNSW* hnsw; - size_t curr; - size_t n_results; - ItemPointer results; -} HnswScanOpaqueData; - -typedef HnswScanOpaqueData* HnswScanOpaque; - -typedef struct { - Oid relid; - uint32 status; - HierarchicalNSW* hnsw; -} HnswHashEntry; - - -#define SH_PREFIX hnsw_index -#define SH_ELEMENT_TYPE HnswHashEntry -#define SH_KEY_TYPE Oid -#define SH_KEY relid -#define SH_STORE_HASH -#define SH_GET_HASH(tb, a) ((a)->relid) -#define SH_HASH_KEY(tb, key) (key) -#define SH_EQUAL(tb, a, b) ((a) == (b)) -#define SH_SCOPE static inline -#define SH_DEFINE -#define SH_DECLARE -#include "lib/simplehash.h" - -#define INDEX_HASH_SIZE 11 - -#define DEFAULT_EF_SEARCH 64 - -PGDLLEXPORT void _PG_init(void); - -static hnsw_index_hash *hnsw_indexes; - -/* - * Initialize index options and variables - */ -void -_PG_init(void) -{ - hnsw_relopt_kind = add_reloption_kind(); - add_int_reloption(hnsw_relopt_kind, "dims", "Number of dimensions", - 0, 0, INT_MAX, AccessExclusiveLock); - add_int_reloption(hnsw_relopt_kind, "maxelements", "Maximal number of elements", - 0, 0, INT_MAX, AccessExclusiveLock); - add_int_reloption(hnsw_relopt_kind, "m", "Number of neighbors of each vertex", - 100, 0, INT_MAX, AccessExclusiveLock); - add_int_reloption(hnsw_relopt_kind, "efconstruction", "Number of inspected neighbors during index construction", - 16, 1, INT_MAX, AccessExclusiveLock); - add_int_reloption(hnsw_relopt_kind, "efsearch", "Number of inspected neighbors during index search", - 64, 1, INT_MAX, AccessExclusiveLock); - hnsw_indexes = hnsw_index_create(TopMemoryContext, INDEX_HASH_SIZE, NULL); -} - - -static void -hnsw_build_callback(Relation index, ItemPointer tid, Datum *values, - bool *isnull, bool tupleIsAlive, void *state) -{ - HierarchicalNSW* hnsw = (HierarchicalNSW*) state; - ArrayType* array; - int n_items; - label_t label = 0; - - /* Skip nulls */ - if (isnull[0]) - return; - - array = DatumGetArrayTypeP(values[0]); - n_items = ArrayGetNItems(ARR_NDIM(array), ARR_DIMS(array)); - if (n_items != hnsw_dimensions(hnsw)) - { - elog(ERROR, "Wrong number of dimensions: %d instead of %d expected", - n_items, hnsw_dimensions(hnsw)); - } - - memcpy(&label, tid, sizeof(*tid)); - hnsw_add_point(hnsw, (coord_t*)ARR_DATA_PTR(array), label); -} - -static void -hnsw_populate(HierarchicalNSW* hnsw, Relation indexRel, Relation heapRel) -{ - IndexInfo* indexInfo = BuildIndexInfo(indexRel); - Assert(indexInfo->ii_NumIndexAttrs == 1); - table_index_build_scan(heapRel, indexRel, indexInfo, - true, true, hnsw_build_callback, (void *) hnsw, NULL); -} - -#ifdef __APPLE__ - -#include -#include - -static void -hnsw_check_available_memory(Size requested) -{ - size_t total; - if (sysctlbyname("hw.memsize", NULL, &total, NULL, 0) < 0) - elog(ERROR, "Failed to get amount of RAM: %m"); - - if ((Size)NBuffers*BLCKSZ + requested >= total) - elog(ERROR, "HNSW index requeries %ld bytes while only %ld are available", - requested, total - (Size)NBuffers*BLCKSZ); -} - -#else - -#include - -static void -hnsw_check_available_memory(Size requested) -{ - struct sysinfo si; - Size total; - if (sysinfo(&si) < 0) - elog(ERROR, "Failed to get amount of RAM: %m"); - - total = si.totalram*si.mem_unit; - if ((Size)NBuffers*BLCKSZ + requested >= total) - elog(ERROR, "HNSW index requeries %ld bytes while only %ld are available", - requested, total - (Size)NBuffers*BLCKSZ); -} - -#endif - -static HierarchicalNSW* -hnsw_get_index(Relation indexRel, Relation heapRel) -{ - HierarchicalNSW* hnsw; - Oid indexoid = RelationGetRelid(indexRel); - HnswHashEntry* entry = hnsw_index_lookup(hnsw_indexes, indexoid); - if (entry == NULL) - { - size_t dims, maxelements; - size_t M; - size_t maxM; - size_t size_links_level0; - size_t size_data_per_element; - size_t data_size; - dsm_handle handle = indexoid << 1; /* make it even */ - void* impl_private = NULL; - void* mapped_address = NULL; - Size mapped_size = 0; - Size shmem_size; - bool exists = true; - bool found; - HnswOptions *opts = (HnswOptions *) indexRel->rd_options; - if (opts == NULL || opts->maxelements == 0 || opts->dims == 0) { - elog(ERROR, "HNSW index requires 'maxelements' and 'dims' to be specified"); - } - dims = opts->dims; - maxelements = opts->maxelements; - M = opts->M; - maxM = M * 2; - data_size = dims * sizeof(coord_t); - size_links_level0 = (maxM + 1) * sizeof(idx_t); - size_data_per_element = size_links_level0 + data_size + sizeof(label_t); - shmem_size = hnsw_sizeof() + maxelements * size_data_per_element; - - hnsw_check_available_memory(shmem_size); - - /* first try to attach to existed index */ - if (!dsm_impl_op(DSM_OP_ATTACH, handle, 0, &impl_private, - &mapped_address, &mapped_size, DEBUG1)) - { - /* index doesn't exists: try to create it */ - if (!dsm_impl_op(DSM_OP_CREATE, handle, shmem_size, &impl_private, - &mapped_address, &mapped_size, DEBUG1)) - { - /* We can do it under shared lock, so some other backend may - * try to initialize index. If create is failed because index already - * created by somebody else, then try to attach to it once again - */ - if (!dsm_impl_op(DSM_OP_ATTACH, handle, 0, &impl_private, - &mapped_address, &mapped_size, ERROR)) - { - return NULL; - } - } - else - { - exists = false; - } - } - Assert(mapped_size == shmem_size); - hnsw = (HierarchicalNSW*)mapped_address; - - if (!exists) - { - hnsw_init(hnsw, dims, maxelements, M, maxM, opts->efConstruction); - hnsw_populate(hnsw, indexRel, heapRel); - } - entry = hnsw_index_insert(hnsw_indexes, indexoid, &found); - Assert(!found); - entry->hnsw = hnsw; - } - else - { - hnsw = entry->hnsw; - } - return hnsw; -} - -/* - * Start or restart an index scan - */ -static IndexScanDesc -hnsw_beginscan(Relation index, int nkeys, int norderbys) -{ - IndexScanDesc scan = RelationGetIndexScan(index, nkeys, norderbys); - HnswScanOpaque so = (HnswScanOpaque) palloc(sizeof(HnswScanOpaqueData)); - Relation heap = relation_open(index->rd_index->indrelid, NoLock); - so->hnsw = hnsw_get_index(index, heap); - relation_close(heap, NoLock); - so->curr = 0; - so->n_results = 0; - so->results = NULL; - scan->opaque = so; - return scan; -} - -/* - * Start or restart an index scan - */ -static void -hnsw_rescan(IndexScanDesc scan, ScanKey keys, int nkeys, ScanKey orderbys, int norderbys) -{ - HnswScanOpaque so = (HnswScanOpaque) scan->opaque; - if (so->results) - { - pfree(so->results); - so->results = NULL; - } - so->curr = 0; - if (orderbys && scan->numberOfOrderBys > 0) - memmove(scan->orderByData, orderbys, scan->numberOfOrderBys * sizeof(ScanKeyData)); -} - -/* - * Fetch the next tuple in the given scan - */ -static bool -hnsw_gettuple(IndexScanDesc scan, ScanDirection dir) -{ - HnswScanOpaque so = (HnswScanOpaque) scan->opaque; - - /* - * Index can be used to scan backward, but Postgres doesn't support - * backward scan on operators - */ - Assert(ScanDirectionIsForward(dir)); - - if (so->curr == 0) - { - Datum value; - ArrayType* array; - int n_items; - size_t n_results; - label_t* results; - HnswOptions *opts = (HnswOptions *) scan->indexRelation->rd_options; - size_t efSearch = opts ? opts->efSearch : DEFAULT_EF_SEARCH; - - /* Safety check */ - if (scan->orderByData == NULL) - elog(ERROR, "cannot scan HNSW index without order"); - - /* No items will match if null */ - if (scan->orderByData->sk_flags & SK_ISNULL) - return false; - - value = scan->orderByData->sk_argument; - array = DatumGetArrayTypeP(value); - n_items = ArrayGetNItems(ARR_NDIM(array), ARR_DIMS(array)); - if (n_items != hnsw_dimensions(so->hnsw)) - { - elog(ERROR, "Wrong number of dimensions: %d instead of %d expected", - n_items, hnsw_dimensions(so->hnsw)); - } - - if (!hnsw_search(so->hnsw, (coord_t*)ARR_DATA_PTR(array), efSearch, &n_results, &results)) - elog(ERROR, "HNSW index search failed"); - so->results = (ItemPointer)palloc(n_results*sizeof(ItemPointerData)); - so->n_results = n_results; - for (size_t i = 0; i < n_results; i++) - { - memcpy(&so->results[i], &results[i], sizeof(so->results[i])); - } - free(results); - } - if (so->curr >= so->n_results) - { - return false; - } - else - { - scan->xs_heaptid = so->results[so->curr++]; - scan->xs_recheckorderby = false; - return true; - } -} - -/* - * End a scan and release resources - */ -static void -hnsw_endscan(IndexScanDesc scan) -{ - HnswScanOpaque so = (HnswScanOpaque) scan->opaque; - if (so->results) - pfree(so->results); - pfree(so); - scan->opaque = NULL; -} - - -/* - * Estimate the cost of an index scan - */ -static void -hnsw_costestimate(PlannerInfo *root, IndexPath *path, double loop_count, - Cost *indexStartupCost, Cost *indexTotalCost, - Selectivity *indexSelectivity, double *indexCorrelation - ,double *indexPages -) -{ - GenericCosts costs; - - /* Never use index without order */ - if (path->indexorderbys == NULL) - { - *indexStartupCost = DBL_MAX; - *indexTotalCost = DBL_MAX; - *indexSelectivity = 0; - *indexCorrelation = 0; - *indexPages = 0; - return; - } - - MemSet(&costs, 0, sizeof(costs)); - - genericcostestimate(root, path, loop_count, &costs); - - /* Startup cost and total cost are same */ - *indexStartupCost = costs.indexTotalCost; - *indexTotalCost = costs.indexTotalCost; - *indexSelectivity = costs.indexSelectivity; - *indexCorrelation = costs.indexCorrelation; - *indexPages = costs.numIndexPages; -} - -/* - * Parse and validate the reloptions - */ -static bytea * -hnsw_options(Datum reloptions, bool validate) -{ - static const relopt_parse_elt tab[] = { - {"dims", RELOPT_TYPE_INT, offsetof(HnswOptions, dims)}, - {"maxelements", RELOPT_TYPE_INT, offsetof(HnswOptions, maxelements)}, - {"efconstruction", RELOPT_TYPE_INT, offsetof(HnswOptions, efConstruction)}, - {"efsearch", RELOPT_TYPE_INT, offsetof(HnswOptions, efSearch)}, - {"m", RELOPT_TYPE_INT, offsetof(HnswOptions, M)} - }; - - return (bytea *) build_reloptions(reloptions, validate, - hnsw_relopt_kind, - sizeof(HnswOptions), - tab, lengthof(tab)); -} - -/* - * Validate catalog entries for the specified operator class - */ -static bool -hnsw_validate(Oid opclassoid) -{ - return true; -} - -/* - * Build the index for a logged table - */ -static IndexBuildResult * -hnsw_build(Relation heap, Relation index, IndexInfo *indexInfo) -{ - HierarchicalNSW* hnsw = hnsw_get_index(index, heap); - IndexBuildResult* result = (IndexBuildResult *) palloc(sizeof(IndexBuildResult)); - result->heap_tuples = result->index_tuples = hnsw_count(hnsw); - - return result; -} - -/* - * Insert a tuple into the index - */ -static bool -hnsw_insert(Relation index, Datum *values, bool *isnull, ItemPointer heap_tid, - Relation heap, IndexUniqueCheck checkUnique, - bool indexUnchanged, - IndexInfo *indexInfo) -{ - HierarchicalNSW* hnsw = hnsw_get_index(index, heap); - Datum value; - ArrayType* array; - int n_items; - label_t label = 0; - - /* Skip nulls */ - if (isnull[0]) - return false; - - /* Detoast value */ - value = PointerGetDatum(PG_DETOAST_DATUM(values[0])); - array = DatumGetArrayTypeP(value); - n_items = ArrayGetNItems(ARR_NDIM(array), ARR_DIMS(array)); - if (n_items != hnsw_dimensions(hnsw)) - { - elog(ERROR, "Wrong number of dimensions: %d instead of %d expected", - n_items, hnsw_dimensions(hnsw)); - } - memcpy(&label, heap_tid, sizeof(*heap_tid)); - if (!hnsw_add_point(hnsw, (coord_t*)ARR_DATA_PTR(array), label)) - elog(ERROR, "HNSW index insert failed"); - return true; -} - -/* - * Build the index for an unlogged table - */ -static void -hnsw_buildempty(Relation index) -{ - /* index will be constructed on dema nd when accessed */ -} - -/* - * Clean up after a VACUUM operation - */ -static IndexBulkDeleteResult * -hnsw_vacuumcleanup(IndexVacuumInfo *info, IndexBulkDeleteResult *stats) -{ - Relation rel = info->index; - - if (stats == NULL) - return NULL; - - stats->num_pages = RelationGetNumberOfBlocks(rel); - - return stats; -} - -/* - * Bulk delete tuples from the index - */ -static IndexBulkDeleteResult * -hnsw_bulkdelete(IndexVacuumInfo *info, IndexBulkDeleteResult *stats, - IndexBulkDeleteCallback callback, void *callback_state) -{ - if (stats == NULL) - stats = (IndexBulkDeleteResult *) palloc0(sizeof(IndexBulkDeleteResult)); - return stats; -} - -/* - * Define index handler - * - * See https://www.postgresql.org/docs/current/index-api.html - */ -PGDLLEXPORT PG_FUNCTION_INFO_V1(hnsw_handler); -Datum -hnsw_handler(PG_FUNCTION_ARGS) -{ - IndexAmRoutine *amroutine = makeNode(IndexAmRoutine); - - amroutine->amstrategies = 0; - amroutine->amsupport = 0; - amroutine->amoptsprocnum = 0; - amroutine->amcanorder = false; - amroutine->amcanorderbyop = true; - amroutine->amcanbackward = false; /* can change direction mid-scan */ - amroutine->amcanunique = false; - amroutine->amcanmulticol = false; - amroutine->amoptionalkey = true; - amroutine->amsearcharray = false; - amroutine->amsearchnulls = false; - amroutine->amstorage = false; - amroutine->amclusterable = false; - amroutine->ampredlocks = false; - amroutine->amcanparallel = false; - amroutine->amcaninclude = false; - amroutine->amusemaintenanceworkmem = false; /* not used during VACUUM */ - amroutine->amparallelvacuumoptions = VACUUM_OPTION_PARALLEL_BULKDEL; - amroutine->amkeytype = InvalidOid; - - /* Interface functions */ - amroutine->ambuild = hnsw_build; - amroutine->ambuildempty = hnsw_buildempty; - amroutine->aminsert = hnsw_insert; - amroutine->ambulkdelete = hnsw_bulkdelete; - amroutine->amvacuumcleanup = hnsw_vacuumcleanup; - amroutine->amcanreturn = NULL; /* tuple not included in heapsort */ - amroutine->amcostestimate = hnsw_costestimate; - amroutine->amoptions = hnsw_options; - amroutine->amproperty = NULL; /* TODO AMPROP_DISTANCE_ORDERABLE */ - amroutine->ambuildphasename = NULL; - amroutine->amvalidate = hnsw_validate; - amroutine->amadjustmembers = NULL; - amroutine->ambeginscan = hnsw_beginscan; - amroutine->amrescan = hnsw_rescan; - amroutine->amgettuple = hnsw_gettuple; - amroutine->amgetbitmap = NULL; - amroutine->amendscan = hnsw_endscan; - amroutine->ammarkpos = NULL; - amroutine->amrestrpos = NULL; - - /* Interface functions to support parallel index scans */ - amroutine->amestimateparallelscan = NULL; - amroutine->aminitparallelscan = NULL; - amroutine->amparallelrescan = NULL; - - PG_RETURN_POINTER(amroutine); -} - -/* - * Get the L2 distance between vectors - */ -PGDLLEXPORT PG_FUNCTION_INFO_V1(l2_distance); -Datum -l2_distance(PG_FUNCTION_ARGS) -{ - ArrayType *a = PG_GETARG_ARRAYTYPE_P(0); - ArrayType *b = PG_GETARG_ARRAYTYPE_P(1); - int a_dim = ArrayGetNItems(ARR_NDIM(a), ARR_DIMS(a)); - int b_dim = ArrayGetNItems(ARR_NDIM(b), ARR_DIMS(b)); - dist_t distance = 0.0; - dist_t diff; - coord_t *ax = (coord_t*)ARR_DATA_PTR(a); - coord_t *bx = (coord_t*)ARR_DATA_PTR(b); - - if (a_dim != b_dim) - { - ereport(ERROR, - (errcode(ERRCODE_DATA_EXCEPTION), - errmsg("different array dimensions %d and %d", a_dim, b_dim))); - } - - for (int i = 0; i < a_dim; i++) - { - diff = ax[i] - bx[i]; - distance += diff * diff; - } - - PG_RETURN_FLOAT4((dist_t)sqrt(distance)); -} diff --git a/pgxn/hnsw/hnsw.control b/pgxn/hnsw/hnsw.control deleted file mode 100644 index fbfa1a5b47..0000000000 --- a/pgxn/hnsw/hnsw.control +++ /dev/null @@ -1,4 +0,0 @@ -comment = '** Deprecated ** Please use pg_embedding instead' -default_version = '0.1.0' -module_pathname = '$libdir/hnsw' -relocatable = true diff --git a/pgxn/hnsw/hnsw.h b/pgxn/hnsw/hnsw.h deleted file mode 100644 index d4065ab8fe..0000000000 --- a/pgxn/hnsw/hnsw.h +++ /dev/null @@ -1,15 +0,0 @@ -#pragma once - -typedef float coord_t; -typedef float dist_t; -typedef uint32_t idx_t; -typedef uint64_t label_t; - -typedef struct HierarchicalNSW HierarchicalNSW; - -bool hnsw_search(HierarchicalNSW* hnsw, const coord_t *point, size_t efSearch, size_t* n_results, label_t** results); -bool hnsw_add_point(HierarchicalNSW* hnsw, const coord_t *point, label_t label); -void hnsw_init(HierarchicalNSW* hnsw, size_t dim, size_t maxelements, size_t M, size_t maxM, size_t efConstruction); -int hnsw_dimensions(HierarchicalNSW* hnsw); -size_t hnsw_count(HierarchicalNSW* hnsw); -size_t hnsw_sizeof(void); diff --git a/pgxn/hnsw/hnswalg.cpp b/pgxn/hnsw/hnswalg.cpp deleted file mode 100644 index f6de3b8314..0000000000 --- a/pgxn/hnsw/hnswalg.cpp +++ /dev/null @@ -1,379 +0,0 @@ -#include "hnswalg.h" - -#if defined(__GNUC__) -#define PORTABLE_ALIGN32 __attribute__((aligned(32))) -#define PREFETCH(addr,hint) __builtin_prefetch(addr, 0, hint) -#else -#define PORTABLE_ALIGN32 __declspec(align(32)) -#define PREFETCH(addr,hint) -#endif - -HierarchicalNSW::HierarchicalNSW(size_t dim_, size_t maxelements_, size_t M_, size_t maxM_, size_t efConstruction_) -{ - dim = dim_; - data_size = dim * sizeof(coord_t); - - efConstruction = efConstruction_; - - maxelements = maxelements_; - M = M_; - maxM = maxM_; - size_links_level0 = (maxM + 1) * sizeof(idx_t); - size_data_per_element = size_links_level0 + data_size + sizeof(label_t); - offset_data = size_links_level0; - offset_label = offset_data + data_size; - - enterpoint_node = 0; - cur_element_count = 0; -#ifdef __x86_64__ - use_avx2 = __builtin_cpu_supports("avx2"); -#endif -} - -std::priority_queue> HierarchicalNSW::searchBaseLayer(const coord_t *point, size_t ef) -{ - std::vector visited; - visited.resize((cur_element_count + 31) >> 5); - - std::priority_queue> topResults; - std::priority_queue> candidateSet; - - dist_t dist = fstdistfunc(point, getDataByInternalId(enterpoint_node)); - - topResults.emplace(dist, enterpoint_node); - candidateSet.emplace(-dist, enterpoint_node); - visited[enterpoint_node >> 5] = 1 << (enterpoint_node & 31); - dist_t lowerBound = dist; - - while (!candidateSet.empty()) - { - std::pair curr_el_pair = candidateSet.top(); - if (-curr_el_pair.first > lowerBound) - break; - - candidateSet.pop(); - idx_t curNodeNum = curr_el_pair.second; - - idx_t* data = get_linklist0(curNodeNum); - size_t size = *data++; - - PREFETCH(getDataByInternalId(*data), 0); - - for (size_t j = 0; j < size; ++j) { - size_t tnum = *(data + j); - - PREFETCH(getDataByInternalId(*(data + j + 1)), 0); - - if (!(visited[tnum >> 5] & (1 << (tnum & 31)))) { - visited[tnum >> 5] |= 1 << (tnum & 31); - - dist = fstdistfunc(point, getDataByInternalId(tnum)); - - if (topResults.top().first > dist || topResults.size() < ef) { - candidateSet.emplace(-dist, tnum); - - PREFETCH(get_linklist0(candidateSet.top().second), 0); - topResults.emplace(dist, tnum); - - if (topResults.size() > ef) - topResults.pop(); - - lowerBound = topResults.top().first; - } - } - } - } - return topResults; -} - - -void HierarchicalNSW::getNeighborsByHeuristic(std::priority_queue> &topResults, size_t NN) -{ - if (topResults.size() < NN) - return; - - std::priority_queue> resultSet; - std::vector> returnlist; - - while (topResults.size() > 0) { - resultSet.emplace(-topResults.top().first, topResults.top().second); - topResults.pop(); - } - - while (resultSet.size()) { - if (returnlist.size() >= NN) - break; - std::pair curen = resultSet.top(); - dist_t dist_to_query = -curen.first; - resultSet.pop(); - bool good = true; - for (std::pair curen2 : returnlist) { - dist_t curdist = fstdistfunc(getDataByInternalId(curen2.second), - getDataByInternalId(curen.second)); - if (curdist < dist_to_query) { - good = false; - break; - } - } - if (good) returnlist.push_back(curen); - } - for (std::pair elem : returnlist) - topResults.emplace(-elem.first, elem.second); -} - -void HierarchicalNSW::mutuallyConnectNewElement(const coord_t *point, idx_t cur_c, - std::priority_queue> topResults) -{ - getNeighborsByHeuristic(topResults, M); - - std::vector res; - res.reserve(M); - while (topResults.size() > 0) { - res.push_back(topResults.top().second); - topResults.pop(); - } - { - idx_t* data = get_linklist0(cur_c); - if (*data) - throw std::runtime_error("Should be blank"); - - *data++ = res.size(); - - for (size_t idx = 0; idx < res.size(); idx++) { - if (data[idx]) - throw std::runtime_error("Should be blank"); - data[idx] = res[idx]; - } - } - for (size_t idx = 0; idx < res.size(); idx++) { - if (res[idx] == cur_c) - throw std::runtime_error("Connection to the same element"); - - size_t resMmax = maxM; - idx_t *ll_other = get_linklist0(res[idx]); - idx_t sz_link_list_other = *ll_other; - - if (sz_link_list_other > resMmax || sz_link_list_other < 0) - throw std::runtime_error("Bad sz_link_list_other"); - - if (sz_link_list_other < resMmax) { - idx_t *data = ll_other + 1; - data[sz_link_list_other] = cur_c; - *ll_other = sz_link_list_other + 1; - } else { - // finding the "weakest" element to replace it with the new one - idx_t *data = ll_other + 1; - dist_t d_max = fstdistfunc(getDataByInternalId(cur_c), getDataByInternalId(res[idx])); - // Heuristic: - std::priority_queue> candidates; - candidates.emplace(d_max, cur_c); - - for (size_t j = 0; j < sz_link_list_other; j++) - candidates.emplace(fstdistfunc(getDataByInternalId(data[j]), getDataByInternalId(res[idx])), data[j]); - - getNeighborsByHeuristic(candidates, resMmax); - - size_t indx = 0; - while (!candidates.empty()) { - data[indx] = candidates.top().second; - candidates.pop(); - indx++; - } - *ll_other = indx; - } - } -} - -void HierarchicalNSW::addPoint(const coord_t *point, label_t label) -{ - if (cur_element_count >= maxelements) { - throw std::runtime_error("The number of elements exceeds the specified limit"); - } - idx_t cur_c = cur_element_count++; - memset((char *) get_linklist0(cur_c), 0, size_data_per_element); - memcpy(getDataByInternalId(cur_c), point, data_size); - memcpy(getExternalLabel(cur_c), &label, sizeof label); - - // Do nothing for the first element - if (cur_c != 0) { - std::priority_queue > topResults = searchBaseLayer(point, efConstruction); - mutuallyConnectNewElement(point, cur_c, topResults); - } -}; - -std::priority_queue> HierarchicalNSW::searchKnn(const coord_t *query, size_t k) -{ - std::priority_queue> topResults; - auto topCandidates = searchBaseLayer(query, k); - while (topCandidates.size() > k) { - topCandidates.pop(); - } - while (!topCandidates.empty()) { - std::pair rez = topCandidates.top(); - label_t label; - memcpy(&label, getExternalLabel(rez.second), sizeof(label)); - topResults.push(std::pair(rez.first, label)); - topCandidates.pop(); - } - - return topResults; -}; - -dist_t fstdistfunc_scalar(const coord_t *x, const coord_t *y, size_t n) -{ - dist_t distance = 0.0; - - for (size_t i = 0; i < n; i++) - { - dist_t diff = x[i] - y[i]; - distance += diff * diff; - } - return distance; - -} - -#ifdef __x86_64__ -#include - -__attribute__((target("avx2"))) -dist_t fstdistfunc_avx2(const coord_t *x, const coord_t *y, size_t n) -{ - const size_t TmpResSz = sizeof(__m256) / sizeof(float); - float PORTABLE_ALIGN32 TmpRes[TmpResSz]; - size_t qty16 = n / 16; - const float *pEnd1 = x + (qty16 * 16); - __m256 diff, v1, v2; - __m256 sum = _mm256_set1_ps(0); - - while (x < pEnd1) { - v1 = _mm256_loadu_ps(x); - x += 8; - v2 = _mm256_loadu_ps(y); - y += 8; - diff = _mm256_sub_ps(v1, v2); - sum = _mm256_add_ps(sum, _mm256_mul_ps(diff, diff)); - - v1 = _mm256_loadu_ps(x); - x += 8; - v2 = _mm256_loadu_ps(y); - y += 8; - diff = _mm256_sub_ps(v1, v2); - sum = _mm256_add_ps(sum, _mm256_mul_ps(diff, diff)); - } - _mm256_store_ps(TmpRes, sum); - float res = TmpRes[0] + TmpRes[1] + TmpRes[2] + TmpRes[3] + TmpRes[4] + TmpRes[5] + TmpRes[6] + TmpRes[7]; - return (res); -} - -dist_t fstdistfunc_sse(const coord_t *x, const coord_t *y, size_t n) -{ - const size_t TmpResSz = sizeof(__m128) / sizeof(float); - float PORTABLE_ALIGN32 TmpRes[TmpResSz]; - size_t qty16 = n / 16; - const float *pEnd1 = x + (qty16 * 16); - - __m128 diff, v1, v2; - __m128 sum = _mm_set1_ps(0); - - while (x < pEnd1) { - v1 = _mm_loadu_ps(x); - x += 4; - v2 = _mm_loadu_ps(y); - y += 4; - diff = _mm_sub_ps(v1, v2); - sum = _mm_add_ps(sum, _mm_mul_ps(diff, diff)); - - v1 = _mm_loadu_ps(x); - x += 4; - v2 = _mm_loadu_ps(y); - y += 4; - diff = _mm_sub_ps(v1, v2); - sum = _mm_add_ps(sum, _mm_mul_ps(diff, diff)); - - v1 = _mm_loadu_ps(x); - x += 4; - v2 = _mm_loadu_ps(y); - y += 4; - diff = _mm_sub_ps(v1, v2); - sum = _mm_add_ps(sum, _mm_mul_ps(diff, diff)); - - v1 = _mm_loadu_ps(x); - x += 4; - v2 = _mm_loadu_ps(y); - y += 4; - diff = _mm_sub_ps(v1, v2); - sum = _mm_add_ps(sum, _mm_mul_ps(diff, diff)); - } - _mm_store_ps(TmpRes, sum); - float res = TmpRes[0] + TmpRes[1] + TmpRes[2] + TmpRes[3]; - return res; -} -#endif - -dist_t HierarchicalNSW::fstdistfunc(const coord_t *x, const coord_t *y) -{ -#ifndef __x86_64__ - return fstdistfunc_scalar(x, y, dim); -#else - if(use_avx2) - return fstdistfunc_avx2(x, y, dim); - - return fstdistfunc_sse(x, y, dim); -#endif -} - -bool hnsw_search(HierarchicalNSW* hnsw, const coord_t *point, size_t efSearch, size_t* n_results, label_t** results) -{ - try - { - auto result = hnsw->searchKnn(point, efSearch); - size_t nResults = result.size(); - *results = (label_t*)malloc(nResults*sizeof(label_t)); - for (size_t i = nResults; i-- != 0;) - { - (*results)[i] = result.top().second; - result.pop(); - } - *n_results = nResults; - return true; - } - catch (std::exception& x) - { - return false; - } -} - -bool hnsw_add_point(HierarchicalNSW* hnsw, const coord_t *point, label_t label) -{ - try - { - hnsw->addPoint(point, label); - return true; - } - catch (std::exception& x) - { - fprintf(stderr, "Catch %s\n", x.what()); - return false; - } -} - -void hnsw_init(HierarchicalNSW* hnsw, size_t dims, size_t maxelements, size_t M, size_t maxM, size_t efConstruction) -{ - new ((void*)hnsw) HierarchicalNSW(dims, maxelements, M, maxM, efConstruction); -} - - -int hnsw_dimensions(HierarchicalNSW* hnsw) -{ - return (int)hnsw->dim; -} - -size_t hnsw_count(HierarchicalNSW* hnsw) -{ - return hnsw->cur_element_count; -} - -size_t hnsw_sizeof(void) -{ - return sizeof(HierarchicalNSW); -} diff --git a/pgxn/hnsw/hnswalg.h b/pgxn/hnsw/hnswalg.h deleted file mode 100644 index f38aeac362..0000000000 --- a/pgxn/hnsw/hnswalg.h +++ /dev/null @@ -1,69 +0,0 @@ -#pragma once - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -extern "C" { -#include "hnsw.h" -} - -struct HierarchicalNSW -{ - size_t maxelements; - size_t cur_element_count; - - idx_t enterpoint_node; - - size_t dim; - size_t data_size; - size_t offset_data; - size_t offset_label; - size_t size_data_per_element; - size_t M; - size_t maxM; - size_t size_links_level0; - size_t efConstruction; - -#ifdef __x86_64__ - bool use_avx2; -#endif - - char data_level0_memory[0]; // varying size - - public: - HierarchicalNSW(size_t dim, size_t maxelements, size_t M, size_t maxM, size_t efConstruction); - ~HierarchicalNSW(); - - - inline coord_t *getDataByInternalId(idx_t internal_id) const { - return (coord_t *)&data_level0_memory[internal_id * size_data_per_element + offset_data]; - } - - inline idx_t *get_linklist0(idx_t internal_id) const { - return (idx_t*)&data_level0_memory[internal_id * size_data_per_element]; - } - - inline label_t *getExternalLabel(idx_t internal_id) const { - return (label_t *)&data_level0_memory[internal_id * size_data_per_element + offset_label]; - } - - std::priority_queue> searchBaseLayer(const coord_t *x, size_t ef); - - void getNeighborsByHeuristic(std::priority_queue> &topResults, size_t NN); - - void mutuallyConnectNewElement(const coord_t *x, idx_t id, std::priority_queue> topResults); - - void addPoint(const coord_t *point, label_t label); - - std::priority_queue> searchKnn(const coord_t *query_data, size_t k); - - dist_t fstdistfunc(const coord_t *x, const coord_t *y); -}; diff --git a/pgxn/hnsw/test/expected/knn.out b/pgxn/hnsw/test/expected/knn.out deleted file mode 100644 index a1cee4525e..0000000000 --- a/pgxn/hnsw/test/expected/knn.out +++ /dev/null @@ -1,28 +0,0 @@ -SET enable_seqscan = off; -CREATE TABLE t (val real[]); -INSERT INTO t (val) VALUES ('{0,0,0}'), ('{1,2,3}'), ('{1,1,1}'), (NULL); -CREATE INDEX ON t USING hnsw (val) WITH (maxelements = 10, dims=3, m=3); -INSERT INTO t (val) VALUES (array[1,2,4]); -explain SELECT * FROM t ORDER BY val <-> array[3,3,3]; - QUERY PLAN --------------------------------------------------------------------- - Index Scan using t_val_idx on t (cost=4.02..8.06 rows=3 width=36) - Order By: (val <-> '{3,3,3}'::real[]) -(2 rows) - -SELECT * FROM t ORDER BY val <-> array[3,3,3]; - val ---------- - {1,2,3} - {1,2,4} - {1,1,1} - {0,0,0} -(4 rows) - -SELECT COUNT(*) FROM t; - count -------- - 5 -(1 row) - -DROP TABLE t; diff --git a/pgxn/hnsw/test/sql/knn.sql b/pgxn/hnsw/test/sql/knn.sql deleted file mode 100644 index 0635bda4a2..0000000000 --- a/pgxn/hnsw/test/sql/knn.sql +++ /dev/null @@ -1,13 +0,0 @@ -SET enable_seqscan = off; - -CREATE TABLE t (val real[]); -INSERT INTO t (val) VALUES ('{0,0,0}'), ('{1,2,3}'), ('{1,1,1}'), (NULL); -CREATE INDEX ON t USING hnsw (val) WITH (maxelements = 10, dims=3, m=3); - -INSERT INTO t (val) VALUES (array[1,2,4]); - -explain SELECT * FROM t ORDER BY val <-> array[3,3,3]; -SELECT * FROM t ORDER BY val <-> array[3,3,3]; -SELECT COUNT(*) FROM t; - -DROP TABLE t; From 8327f68043e692c77f70d6a6dafa463636c01578 Mon Sep 17 00:00:00 2001 From: Heikki Linnakangas Date: Sat, 11 Jan 2025 19:39:27 +0200 Subject: [PATCH 046/111] Minor cleanup of extension build commands (#10356) There used to be some pg version dependencies in these extensions, but now that there isn't, follow the simpler pattern used in other extensions. No change in the produced images. --- compute/compute-node.Dockerfile | 34 ++++----------------------------- 1 file changed, 4 insertions(+), 30 deletions(-) diff --git a/compute/compute-node.Dockerfile b/compute/compute-node.Dockerfile index f4df507b74..449e12af5d 100644 --- a/compute/compute-node.Dockerfile +++ b/compute/compute-node.Dockerfile @@ -976,22 +976,9 @@ RUN apt update && apt install --no-install-recommends --no-install-suggests -y p FROM rust-extensions-build-pgrx12 AS pg-jsonschema-pg-build ARG PG_VERSION -# version 0.3.3 supports v17 # last release v0.3.3 - Oct 16, 2024 -# -# there were no breaking changes -# so we can use the same version for all postgres versions -RUN case "${PG_VERSION}" in \ - "v14" | "v15" | "v16" | "v17") \ - export PG_JSONSCHEMA_VERSION=0.3.3 \ - export PG_JSONSCHEMA_CHECKSUM=40c2cffab4187e0233cb8c3bde013be92218c282f95f4469c5282f6b30d64eac \ - ;; \ - *) \ - echo "unexpected PostgreSQL version" && exit 1 \ - ;; \ - esac && \ - wget https://github.com/supabase/pg_jsonschema/archive/refs/tags/v${PG_JSONSCHEMA_VERSION}.tar.gz -O pg_jsonschema.tar.gz && \ - echo "${PG_JSONSCHEMA_CHECKSUM} pg_jsonschema.tar.gz" | sha256sum --check && \ +RUN wget https://github.com/supabase/pg_jsonschema/archive/refs/tags/v0.3.3.tar.gz -O pg_jsonschema.tar.gz && \ + echo "40c2cffab4187e0233cb8c3bde013be92218c282f95f4469c5282f6b30d64eac pg_jsonschema.tar.gz" | sha256sum --check && \ mkdir pg_jsonschema-src && cd pg_jsonschema-src && tar xzf ../pg_jsonschema.tar.gz --strip-components=1 -C . && \ # see commit 252b3685a27a0f4c31a0f91e983c6314838e89e8 # `unsafe-postgres` feature allows to build pgx extensions @@ -1012,22 +999,9 @@ RUN case "${PG_VERSION}" in \ FROM rust-extensions-build-pgrx12 AS pg-graphql-pg-build ARG PG_VERSION -# version 1.5.9 supports v17 # last release v1.5.9 - Oct 16, 2024 -# -# there were no breaking changes -# so we can use the same version for all postgres versions -RUN case "${PG_VERSION}" in \ - "v14" | "v15" | "v16" | "v17") \ - export PG_GRAPHQL_VERSION=1.5.9 \ - export PG_GRAPHQL_CHECKSUM=cf768385a41278be1333472204fc0328118644ae443182cf52f7b9b23277e497 \ - ;; \ - *) \ - echo "unexpected PostgreSQL version" && exit 1 \ - ;; \ - esac && \ - wget https://github.com/supabase/pg_graphql/archive/refs/tags/v${PG_GRAPHQL_VERSION}.tar.gz -O pg_graphql.tar.gz && \ - echo "${PG_GRAPHQL_CHECKSUM} pg_graphql.tar.gz" | sha256sum --check && \ +RUN wget https://github.com/supabase/pg_graphql/archive/refs/tags/v1.5.9.tar.gz -O pg_graphql.tar.gz && \ + echo "cf768385a41278be1333472204fc0328118644ae443182cf52f7b9b23277e497 pg_graphql.tar.gz" | sha256sum --check && \ mkdir pg_graphql-src && cd pg_graphql-src && tar xzf ../pg_graphql.tar.gz --strip-components=1 -C . && \ sed -i 's/pgrx = "=0.12.6"/pgrx = { version = "0.12.6", features = [ "unsafe-postgres" ] }/g' Cargo.toml && \ cargo pgrx install --release && \ From cd982a82ecfd2d96c7251ff7203df38d0ed44539 Mon Sep 17 00:00:00 2001 From: Erik Grinaker Date: Mon, 13 Jan 2025 10:44:59 +0100 Subject: [PATCH 047/111] pageserver,safekeeper: increase heap profiling frequency to 2 MB (#10362) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Problem Currently, the heap profiling frequency is every 1 MB allocated. Taking a profile stack trace takes about 1 µs, and allocating 1 MB takes about 15 µs, so the overhead is about 6.7% which is a bit high. This is a fixed cost regardless of whether heap profiles are actually accessed. ## Summary of changes Increase the heap profiling sample frequency from 1 MB to 2 MB, which reduces the overhead to about 3.3%. This seems acceptable, considering performance-sensitive code will avoid allocations as far as possible anyway. --- pageserver/src/bin/pageserver.rs | 6 ++++-- safekeeper/benches/receive_wal.rs | 5 ++--- safekeeper/src/bin/safekeeper.rs | 6 ++++-- 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/pageserver/src/bin/pageserver.rs b/pageserver/src/bin/pageserver.rs index 567a69da3b..921c6a5092 100644 --- a/pageserver/src/bin/pageserver.rs +++ b/pageserver/src/bin/pageserver.rs @@ -53,10 +53,12 @@ project_build_tag!(BUILD_TAG); #[global_allocator] static GLOBAL: tikv_jemallocator::Jemalloc = tikv_jemallocator::Jemalloc; -/// Configure jemalloc to sample allocations for profiles every 1 MB (1 << 20). +/// Configure jemalloc to profile heap allocations by sampling stack traces every 2 MB (1 << 21). +/// This adds roughly 3% overhead for allocations on average, which is acceptable considering +/// performance-sensitive code will avoid allocations as far as possible anyway. #[allow(non_upper_case_globals)] #[export_name = "malloc_conf"] -pub static malloc_conf: &[u8] = b"prof:true,prof_active:true,lg_prof_sample:20\0"; +pub static malloc_conf: &[u8] = b"prof:true,prof_active:true,lg_prof_sample:21\0"; const PID_FILE_NAME: &str = "pageserver.pid"; diff --git a/safekeeper/benches/receive_wal.rs b/safekeeper/benches/receive_wal.rs index 996c4d9b8c..19c6662e74 100644 --- a/safekeeper/benches/receive_wal.rs +++ b/safekeeper/benches/receive_wal.rs @@ -21,14 +21,13 @@ const KB: usize = 1024; const MB: usize = 1024 * KB; const GB: usize = 1024 * MB; -/// Use jemalloc, and configure it to sample allocations for profiles every 1 MB. -/// This mirrors the configuration in bin/safekeeper.rs. +/// Use jemalloc and enable profiling, to mirror bin/safekeeper.rs. #[global_allocator] static GLOBAL: tikv_jemallocator::Jemalloc = tikv_jemallocator::Jemalloc; #[allow(non_upper_case_globals)] #[export_name = "malloc_conf"] -pub static malloc_conf: &[u8] = b"prof:true,prof_active:true,lg_prof_sample:20\0"; +pub static malloc_conf: &[u8] = b"prof:true,prof_active:true,lg_prof_sample:21\0"; // Register benchmarks with Criterion. criterion_group!( diff --git a/safekeeper/src/bin/safekeeper.rs b/safekeeper/src/bin/safekeeper.rs index 13f6e34575..bc7af02185 100644 --- a/safekeeper/src/bin/safekeeper.rs +++ b/safekeeper/src/bin/safekeeper.rs @@ -51,10 +51,12 @@ use utils::{ #[global_allocator] static GLOBAL: tikv_jemallocator::Jemalloc = tikv_jemallocator::Jemalloc; -/// Configure jemalloc to sample allocations for profiles every 1 MB (1 << 20). +/// Configure jemalloc to profile heap allocations by sampling stack traces every 2 MB (1 << 21). +/// This adds roughly 3% overhead for allocations on average, which is acceptable considering +/// performance-sensitive code will avoid allocations as far as possible anyway. #[allow(non_upper_case_globals)] #[export_name = "malloc_conf"] -pub static malloc_conf: &[u8] = b"prof:true,prof_active:true,lg_prof_sample:20\0"; +pub static malloc_conf: &[u8] = b"prof:true,prof_active:true,lg_prof_sample:21\0"; const PID_FILE_NAME: &str = "safekeeper.pid"; const ID_FILE_NAME: &str = "safekeeper.id"; From 22a6460010490857c927d1218b21f81d2fb0c06d Mon Sep 17 00:00:00 2001 From: Erik Grinaker Date: Mon, 13 Jan 2025 11:01:18 +0100 Subject: [PATCH 048/111] libs/utils: add `force` parameter for `/profile/cpu` (#10361) ## Problem It's only possible to take one CPU profile at a time. With Grafana continuous profiling, a (low-frequency) CPU profile will always be running, making it hard to take an ad hoc CPU profile at the same time. Resolves #10072. ## Summary of changes Add a `force` parameter for `/profile/cpu` which will end and return an already running CPU profile, starting a new one for the current caller. --- libs/utils/src/http/endpoint.rs | 46 ++++++++++++++++++++++----------- 1 file changed, 31 insertions(+), 15 deletions(-) diff --git a/libs/utils/src/http/endpoint.rs b/libs/utils/src/http/endpoint.rs index 4b4aa88d6b..a6ba685447 100644 --- a/libs/utils/src/http/endpoint.rs +++ b/libs/utils/src/http/endpoint.rs @@ -15,7 +15,7 @@ use once_cell::sync::Lazy; use regex::Regex; use routerify::ext::RequestExt; use routerify::{Middleware, RequestInfo, Router, RouterBuilder}; -use tokio::sync::{mpsc, Mutex}; +use tokio::sync::{mpsc, Mutex, Notify}; use tokio_stream::wrappers::ReceiverStream; use tokio_util::io::ReaderStream; use tracing::{debug, info, info_span, warn, Instrument}; @@ -358,25 +358,41 @@ pub async fn profile_cpu_handler(req: Request) -> Result, A Some(1001..) => return Err(ApiError::BadRequest(anyhow!("frequency must be <=1000 Hz"))), Some(frequency) => frequency, }; - - // Only allow one profiler at a time. - static PROFILE_LOCK: Lazy> = Lazy::new(|| Mutex::new(())); - let _lock = PROFILE_LOCK - .try_lock() - .map_err(|_| ApiError::Conflict("profiler already running".into()))?; + let force: bool = parse_query_param(&req, "force")?.unwrap_or_default(); // Take the profile. - let report = tokio::task::spawn_blocking(move || { + static PROFILE_LOCK: Lazy> = Lazy::new(|| Mutex::new(())); + static PROFILE_CANCEL: Lazy = Lazy::new(Notify::new); + + let report = { + // Only allow one profiler at a time. If force is true, cancel a running profile (e.g. a + // Grafana continuous profile). We use a try_lock() loop when cancelling instead of waiting + // for a lock(), to avoid races where the notify isn't currently awaited. + let _lock = loop { + match PROFILE_LOCK.try_lock() { + Ok(lock) => break lock, + Err(_) if force => PROFILE_CANCEL.notify_waiters(), + Err(_) => return Err(ApiError::Conflict("profiler already running".into())), + } + tokio::time::sleep(Duration::from_millis(1)).await; // don't busy-wait + }; + let guard = ProfilerGuardBuilder::default() .frequency(frequency_hz) .blocklist(&["libc", "libgcc", "pthread", "vdso"]) - .build()?; - std::thread::sleep(Duration::from_secs(seconds)); - guard.report().build() - }) - .await - .map_err(|join_err| ApiError::InternalServerError(join_err.into()))? - .map_err(|pprof_err| ApiError::InternalServerError(pprof_err.into()))?; + .build() + .map_err(|err| ApiError::InternalServerError(err.into()))?; + + tokio::select! { + _ = tokio::time::sleep(Duration::from_secs(seconds)) => {}, + _ = PROFILE_CANCEL.notified() => {}, + }; + + guard + .report() + .build() + .map_err(|err| ApiError::InternalServerError(err.into()))? + }; // Return the report in the requested format. match format { From de199d71e18888b059cc8846387d933b103f323e Mon Sep 17 00:00:00 2001 From: Conrad Ludgate Date: Mon, 13 Jan 2025 10:34:36 +0000 Subject: [PATCH 049/111] chore: Address lints introduced in rust 1.85.0 beta (#10340) With a new beta build of the rust compiler, it's good to check out the new lints. Either to find false positives, or find flaws in our code. Additionally, it helps reduce the effort required to update to 1.85 in 6 weeks. --- control_plane/src/local_env.rs | 1 - libs/pq_proto/src/lib.rs | 2 +- libs/utils/src/generation.rs | 6 +++--- libs/utils/src/lsn.rs | 2 +- pageserver/src/tenant/config.rs | 4 ++-- pageserver/src/tenant/disk_btree.rs | 10 +++++----- pageserver/src/tenant/storage_layer/inmemory_layer.rs | 4 ++-- .../timeline/walreceiver/walreceiver_connection.rs | 2 +- pageserver/src/walingest.rs | 2 +- 9 files changed, 16 insertions(+), 17 deletions(-) diff --git a/control_plane/src/local_env.rs b/control_plane/src/local_env.rs index 5b82acb3a5..2fe4cd5202 100644 --- a/control_plane/src/local_env.rs +++ b/control_plane/src/local_env.rs @@ -483,7 +483,6 @@ impl LocalEnv { .iter() .find(|(mapped_tenant_id, _)| mapped_tenant_id == &tenant_id) .map(|&(_, timeline_id)| timeline_id) - .map(TimelineId::from) } pub fn timeline_name_mappings(&self) -> HashMap { diff --git a/libs/pq_proto/src/lib.rs b/libs/pq_proto/src/lib.rs index 94714359a3..50b2c69d24 100644 --- a/libs/pq_proto/src/lib.rs +++ b/libs/pq_proto/src/lib.rs @@ -44,7 +44,7 @@ pub struct ProtocolVersion(u32); impl ProtocolVersion { pub const fn new(major: u16, minor: u16) -> Self { - Self((major as u32) << 16 | minor as u32) + Self(((major as u32) << 16) | minor as u32) } pub const fn minor(self) -> u16 { self.0 as u16 diff --git a/libs/utils/src/generation.rs b/libs/utils/src/generation.rs index 5970836033..44565ee6a2 100644 --- a/libs/utils/src/generation.rs +++ b/libs/utils/src/generation.rs @@ -112,9 +112,9 @@ impl Serialize for Generation { // We should never be asked to serialize a None. Structures // that include an optional generation should convert None to an // Option::None - Err(serde::ser::Error::custom( - "Tried to serialize invalid generation ({self})", - )) + Err(serde::ser::Error::custom(format!( + "Tried to serialize invalid generation ({self:?})" + ))) } } } diff --git a/libs/utils/src/lsn.rs b/libs/utils/src/lsn.rs index f188165600..c874fa30ff 100644 --- a/libs/utils/src/lsn.rs +++ b/libs/utils/src/lsn.rs @@ -260,7 +260,7 @@ impl FromStr for Lsn { { let left_num = u32::from_str_radix(left, 16).map_err(|_| LsnParseError)?; let right_num = u32::from_str_radix(right, 16).map_err(|_| LsnParseError)?; - Ok(Lsn((left_num as u64) << 32 | right_num as u64)) + Ok(Lsn(((left_num as u64) << 32) | right_num as u64)) } else { Err(LsnParseError) } diff --git a/pageserver/src/tenant/config.rs b/pageserver/src/tenant/config.rs index d54dded778..edf2e6a3aa 100644 --- a/pageserver/src/tenant/config.rs +++ b/pageserver/src/tenant/config.rs @@ -11,7 +11,7 @@ pub(crate) use pageserver_api::config::TenantConfigToml as TenantConf; use pageserver_api::models::CompactionAlgorithmSettings; use pageserver_api::models::EvictionPolicy; -use pageserver_api::models::{self, TenantConfigPatch, ThrottleConfig}; +use pageserver_api::models::{self, TenantConfigPatch}; use pageserver_api::shard::{ShardCount, ShardIdentity, ShardNumber, ShardStripeSize}; use serde::de::IntoDeserializer; use serde::{Deserialize, Serialize}; @@ -597,7 +597,7 @@ impl From for models::TenantConfig { .map(humantime), heatmap_period: value.heatmap_period.map(humantime), lazy_slru_download: value.lazy_slru_download, - timeline_get_throttle: value.timeline_get_throttle.map(ThrottleConfig::from), + timeline_get_throttle: value.timeline_get_throttle, image_layer_creation_check_threshold: value.image_layer_creation_check_threshold, lsn_lease_length: value.lsn_lease_length.map(humantime), lsn_lease_length_for_ts: value.lsn_lease_length_for_ts.map(humantime), diff --git a/pageserver/src/tenant/disk_btree.rs b/pageserver/src/tenant/disk_btree.rs index c77342b144..bb9df020b5 100644 --- a/pageserver/src/tenant/disk_btree.rs +++ b/pageserver/src/tenant/disk_btree.rs @@ -84,17 +84,17 @@ impl Value { fn to_u64(self) -> u64 { let b = &self.0; - (b[0] as u64) << 32 - | (b[1] as u64) << 24 - | (b[2] as u64) << 16 - | (b[3] as u64) << 8 + ((b[0] as u64) << 32) + | ((b[1] as u64) << 24) + | ((b[2] as u64) << 16) + | ((b[3] as u64) << 8) | b[4] as u64 } fn to_blknum(self) -> u32 { let b = &self.0; assert!(b[0] == 0x80); - (b[1] as u32) << 24 | (b[2] as u32) << 16 | (b[3] as u32) << 8 | b[4] as u32 + ((b[1] as u32) << 24) | ((b[2] as u32) << 16) | ((b[3] as u32) << 8) | b[4] as u32 } } diff --git a/pageserver/src/tenant/storage_layer/inmemory_layer.rs b/pageserver/src/tenant/storage_layer/inmemory_layer.rs index 71e53da20f..2b67f55a17 100644 --- a/pageserver/src/tenant/storage_layer/inmemory_layer.rs +++ b/pageserver/src/tenant/storage_layer/inmemory_layer.rs @@ -112,8 +112,8 @@ const MAX_SUPPORTED_BLOB_LEN_BITS: usize = { /// /// Layout: /// - 1 bit: `will_init` -/// - [`MAX_SUPPORTED_BLOB_LEN_BITS`]: `len` -/// - [`MAX_SUPPORTED_POS_BITS`]: `pos` +/// - [`MAX_SUPPORTED_BLOB_LEN_BITS`][]: `len` +/// - [`MAX_SUPPORTED_POS_BITS`](IndexEntry::MAX_SUPPORTED_POS_BITS): `pos` #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct IndexEntry(u64); diff --git a/pageserver/src/tenant/timeline/walreceiver/walreceiver_connection.rs b/pageserver/src/tenant/timeline/walreceiver/walreceiver_connection.rs index d74faa1af5..3a8796add8 100644 --- a/pageserver/src/tenant/timeline/walreceiver/walreceiver_connection.rs +++ b/pageserver/src/tenant/timeline/walreceiver/walreceiver_connection.rs @@ -403,7 +403,7 @@ pub(super) async fn handle_walreceiver_connection( // need to advance last record LSN on all shards. If we've not ingested the latest // record, then set the LSN of the modification past it. This way all shards // advance their last record LSN at the same time. - let needs_last_record_lsn_advance = match next_record_lsn.map(Lsn::from) { + let needs_last_record_lsn_advance = match next_record_lsn { Some(lsn) if lsn > modification.get_lsn() => { modification.set_lsn(lsn).unwrap(); true diff --git a/pageserver/src/walingest.rs b/pageserver/src/walingest.rs index e5b23fed51..7253af8507 100644 --- a/pageserver/src/walingest.rs +++ b/pageserver/src/walingest.rs @@ -308,7 +308,7 @@ impl WalIngest { epoch -= 1; } - Ok((epoch as u64) << 32 | xid as u64) + Ok(((epoch as u64) << 32) | xid as u64) } async fn ingest_clear_vm_bits( From 12053cf83260da4da46f4039d5bbfb4cbb8e57da Mon Sep 17 00:00:00 2001 From: John Spray Date: Mon, 13 Jan 2025 11:18:14 +0000 Subject: [PATCH 050/111] storage controller: improve consistency_check_api (#10363) ## Problem Limitations found while using this to investigate https://github.com/neondatabase/neon/issues/10234: - If we hit a node consistency issue, we drop out and don't check shards for consistency - The messages printed after a shard consistency issue are huge, and grafana appears to drop them. ## Summary of changes - Defer node consistency errors until the end of the function, so that we always proceed to check shards for consistency - Print out smaller log lines that just point out the diffs between expected and persistent state --- storage_controller/src/service.rs | 59 ++++++++++++++++++++++++++++--- 1 file changed, 54 insertions(+), 5 deletions(-) diff --git a/storage_controller/src/service.rs b/storage_controller/src/service.rs index 265b2798d2..430d884548 100644 --- a/storage_controller/src/service.rs +++ b/storage_controller/src/service.rs @@ -5256,7 +5256,8 @@ impl Service { expect_nodes.sort_by_key(|n| n.node_id); nodes.sort_by_key(|n| n.node_id); - if nodes != expect_nodes { + // Errors relating to nodes are deferred so that we don't skip the shard checks below if we have a node error + let node_result = if nodes != expect_nodes { tracing::error!("Consistency check failed on nodes."); tracing::error!( "Nodes in memory: {}", @@ -5268,10 +5269,12 @@ impl Service { serde_json::to_string(&nodes) .map_err(|e| ApiError::InternalServerError(e.into()))? ); - return Err(ApiError::InternalServerError(anyhow::anyhow!( + Err(ApiError::InternalServerError(anyhow::anyhow!( "Node consistency failure" - ))); - } + ))) + } else { + Ok(()) + }; let mut persistent_shards = self.persistence.load_active_tenant_shards().await?; persistent_shards @@ -5281,6 +5284,7 @@ impl Service { if persistent_shards != expect_shards { tracing::error!("Consistency check failed on shards."); + tracing::error!( "Shards in memory: {}", serde_json::to_string(&expect_shards) @@ -5291,12 +5295,57 @@ impl Service { serde_json::to_string(&persistent_shards) .map_err(|e| ApiError::InternalServerError(e.into()))? ); + + // The total dump log lines above are useful in testing but in the field grafana will + // usually just drop them because they're so large. So we also do some explicit logging + // of just the diffs. + let persistent_shards = persistent_shards + .into_iter() + .map(|tsp| (tsp.get_tenant_shard_id().unwrap(), tsp)) + .collect::>(); + let expect_shards = expect_shards + .into_iter() + .map(|tsp| (tsp.get_tenant_shard_id().unwrap(), tsp)) + .collect::>(); + for (tenant_shard_id, persistent_tsp) in &persistent_shards { + match expect_shards.get(tenant_shard_id) { + None => { + tracing::error!( + "Shard {} found in database but not in memory", + tenant_shard_id + ); + } + Some(expect_tsp) => { + if expect_tsp != persistent_tsp { + tracing::error!( + "Shard {} is inconsistent. In memory: {}, database has: {}", + tenant_shard_id, + serde_json::to_string(expect_tsp).unwrap(), + serde_json::to_string(&persistent_tsp).unwrap() + ); + } + } + } + } + + // Having already logged any differences, log any shards that simply aren't present in the database + for (tenant_shard_id, memory_tsp) in &expect_shards { + if !persistent_shards.contains_key(tenant_shard_id) { + tracing::error!( + "Shard {} found in memory but not in database: {}", + tenant_shard_id, + serde_json::to_string(memory_tsp) + .map_err(|e| ApiError::InternalServerError(e.into()))? + ); + } + } + return Err(ApiError::InternalServerError(anyhow::anyhow!( "Shard consistency failure" ))); } - Ok(()) + node_result } /// For debug/support: a JSON dump of the [`Scheduler`]. Returns a response so that From 09fe3b025c42e56d78812e1ff329551e2da17720 Mon Sep 17 00:00:00 2001 From: Heikki Linnakangas Date: Mon, 13 Jan 2025 13:35:39 +0200 Subject: [PATCH 051/111] Add a websockets tunnel and a test for the proxy's websockets support. (#3823) For testing the proxy's websockets support. I wrote this to test https://github.com/neondatabase/neon/issues/3822. Unfortunately, that bug can *not* be reproduced with this tunnel. The bug only appears when the client pipelines the first query with the authentication messages. The tunnel doesn't do that. --- Update (@conradludgate 2025-01-10): We have since added some websocket tests, but they manually implemented a very simplistic setup of the postgres protocol. Introducing the tunnel would make more complex testing simpler in the future. --------- Co-authored-by: Conrad Ludgate --- test_runner/regress/test_proxy_websockets.py | 55 +++++++ test_runner/websocket_tunnel.py | 154 +++++++++++++++++++ 2 files changed, 209 insertions(+) create mode 100755 test_runner/websocket_tunnel.py diff --git a/test_runner/regress/test_proxy_websockets.py b/test_runner/regress/test_proxy_websockets.py index ea01252ce4..f14317a39f 100644 --- a/test_runner/regress/test_proxy_websockets.py +++ b/test_runner/regress/test_proxy_websockets.py @@ -1,10 +1,15 @@ from __future__ import annotations +import asyncio import ssl +import asyncpg import pytest +import websocket_tunnel import websockets +from fixtures.log_helper import log from fixtures.neon_fixtures import NeonProxy +from fixtures.port_distributor import PortDistributor @pytest.mark.asyncio @@ -196,3 +201,53 @@ async def test_websockets_pipelined(static_proxy: NeonProxy): # close await websocket.send(b"X\x00\x00\x00\x04") await websocket.wait_closed() + + +@pytest.mark.asyncio +async def test_websockets_tunneled(static_proxy: NeonProxy, port_distributor: PortDistributor): + static_proxy.safe_psql("create user ws_auth with password 'ws' superuser") + + user = "ws_auth" + password = "ws" + + ssl_context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + ssl_context.load_verify_locations(str(static_proxy.test_output_dir / "proxy.crt")) + + # Launch a tunnel service so that we can speak the websockets protocol to + # the proxy + tunnel_port = port_distributor.get_port() + tunnel_server = await websocket_tunnel.start_server( + "127.0.0.1", + tunnel_port, + f"wss://{static_proxy.domain}:{static_proxy.external_http_port}/sql", + ssl_context, + ) + log.info(f"websockets tunnel listening for connections on port {tunnel_port}") + + async with tunnel_server: + + async def run_tunnel(): + try: + async with tunnel_server: + await tunnel_server.serve_forever() + except Exception as e: + log.error(f"Error in tunnel task: {e}") + + tunnel_task = asyncio.create_task(run_tunnel()) + + # Ok, the tunnel is now running. Check that we can connect to the proxy's + # websocket interface, through the tunnel + tunnel_connstring = f"postgres://{user}:{password}@127.0.0.1:{tunnel_port}/postgres" + + log.info(f"connecting to {tunnel_connstring}") + conn = await asyncpg.connect(tunnel_connstring) + res = await conn.fetchval("SELECT 123") + assert res == 123 + await conn.close() + log.info("Ran a query successfully through the tunnel") + + tunnel_server.close() + try: + await tunnel_task + except asyncio.CancelledError: + pass diff --git a/test_runner/websocket_tunnel.py b/test_runner/websocket_tunnel.py new file mode 100755 index 0000000000..facdb19140 --- /dev/null +++ b/test_runner/websocket_tunnel.py @@ -0,0 +1,154 @@ +#!/usr/bin/env python3 +# +# This program helps to test the WebSocket tunneling in proxy. It listens for a TCP +# connection on a port, and when you connect to it, it opens a websocket connection, +# and forwards all the traffic to the websocket connection, wrapped in WebSocket binary +# frames. +# +# This is used in the test_proxy::test_websockets test, but it is handy for manual testing too. +# +# Usage for manual testing: +# +# ## Launch Posgres on port 3000: +# postgres -D data -p3000 +# +# ## Launch proxy with WSS enabled: +# openssl req -new -x509 -days 365 -nodes -text -out server.crt -keyout server.key -subj '/CN=*.neon.localtest.me' +# ./target/debug/proxy --wss 127.0.0.1:40433 --http 127.0.0.1:28080 --mgmt 127.0.0.1:9099 --proxy 127.0.0.1:4433 --tls-key server.key --tls-cert server.crt --auth-backend postgres +# +# ## Launch the tunnel: +# +# poetry run ./test_runner/websocket_tunnel.py --ws-port 40433 --ws-url "wss://ep-test.neon.localtest.me" +# +# ## Now you can connect with psql: +# psql "postgresql://heikki@localhost:40433/postgres" +# + +import argparse +import asyncio +import logging +import ssl +from ssl import Purpose + +import websockets +from fixtures.log_helper import log + + +# Enable verbose logging of all the traffic +def enable_verbose_logging(): + logger = logging.getLogger("websockets") + logger.setLevel(logging.DEBUG) + logger.addHandler(logging.StreamHandler()) + + +async def start_server(tcp_listen_host, tcp_listen_port, ws_url, ctx): + server = await asyncio.start_server( + lambda r, w: handle_client(r, w, ws_url, ctx), tcp_listen_host, tcp_listen_port + ) + return server + + +async def handle_tcp_to_websocket(tcp_reader, ws): + try: + while not tcp_reader.at_eof(): + data = await tcp_reader.read(1024) + + await ws.send(data) + except websockets.exceptions.ConnectionClosedError as e: + log.debug(f"connection closed: {e}") + except websockets.exceptions.ConnectionClosedOK: + log.debug("connection closed") + except Exception as e: + log.error(e) + + +async def handle_websocket_to_tcp(ws, tcp_writer): + try: + async for message in ws: + tcp_writer.write(message) + await tcp_writer.drain() + except websockets.exceptions.ConnectionClosedError as e: + log.debug(f"connection closed: {e}") + except websockets.exceptions.ConnectionClosedOK: + log.debug("connection closed") + except Exception as e: + log.error(e) + + +async def handle_client(tcp_reader, tcp_writer, ws_url: str, ctx: ssl.SSLContext): + try: + log.info("Received TCP connection. Connecting to websockets proxy.") + + async with websockets.connect(ws_url, ssl=ctx) as ws: + try: + log.info("Connected to websockets proxy") + + async with asyncio.TaskGroup() as tg: + task1 = tg.create_task(handle_tcp_to_websocket(tcp_reader, ws)) + task2 = tg.create_task(handle_websocket_to_tcp(ws, tcp_writer)) + + done, pending = await asyncio.wait( + [task1, task2], return_when=asyncio.FIRST_COMPLETED + ) + tcp_writer.close() + await ws.close() + + except* Exception as ex: + log.error(ex.exceptions) + except Exception as e: + log.error(e) + + +async def main(): + parser = argparse.ArgumentParser() + parser.add_argument( + "--tcp-listen-addr", + default="localhost", + help="TCP addr to listen on", + ) + parser.add_argument( + "--tcp-listen-port", + default="40444", + help="TCP port to listen on", + ) + + parser.add_argument( + "--ws-url", + default="wss://localhost/", + help="websocket URL to connect to. This determines the Host header sent to the server", + ) + parser.add_argument( + "--ws-host", + default="127.0.0.1", + help="websockets host to connect to", + ) + parser.add_argument( + "--ws-port", + type=int, + default=443, + help="websockets port to connect to", + ) + parser.add_argument( + "--verbose", + action="store_true", + help="enable verbose logging", + ) + args = parser.parse_args() + + if args.verbose: + enable_verbose_logging() + + ctx = ssl.create_default_context(Purpose.SERVER_AUTH) + ctx.check_hostname = False + ctx.verify_mode = ssl.CERT_NONE + + server = await start_server(args.tcp_listen_addr, args.tcp_listen_port, args.ws_url, ctx) + print( + f"Listening for connections at {args.tcp_listen_addr}:{args.tcp_listen_port}, forwarding them to {args.ws_host}:{args.ws_port}" + ) + async with server: + await server.serve_forever() + + +if __name__ == "__main__": + asyncio.run(main()) From 0b9032065ef1e140ad54e09e81aaf00a1b809e87 Mon Sep 17 00:00:00 2001 From: Erik Grinaker Date: Mon, 13 Jan 2025 14:14:23 +0100 Subject: [PATCH 052/111] utils: allow 60-second CPU profiles (#10367) Taking continuous profiles every 20 seconds is likely too expensive (in dollar terms). Let's try 60-second profiles. We can now interrupt running profiles via `?force=true`, so this should be fine. --- libs/utils/src/http/endpoint.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libs/utils/src/http/endpoint.rs b/libs/utils/src/http/endpoint.rs index a6ba685447..ca65c39ad6 100644 --- a/libs/utils/src/http/endpoint.rs +++ b/libs/utils/src/http/endpoint.rs @@ -350,8 +350,8 @@ pub async fn profile_cpu_handler(req: Request) -> Result, A }; let seconds = match parse_query_param(&req, "seconds")? { None => 5, - Some(seconds @ 1..=30) => seconds, - Some(_) => return Err(ApiError::BadRequest(anyhow!("duration must be 1-30 secs"))), + Some(seconds @ 1..=60) => seconds, + Some(_) => return Err(ApiError::BadRequest(anyhow!("duration must be 1-60 secs"))), }; let frequency_hz = match parse_query_param(&req, "frequency")? { None => 99, From 9591d8789c33a1d0d6a1a53b0a4a410a2af0db20 Mon Sep 17 00:00:00 2001 From: Christian Schwarz Date: Mon, 13 Jan 2025 14:27:32 +0100 Subject: [PATCH 053/111] WIP --- pageserver/src/tenant/timeline/handle.rs | 49 +++++++++++++++--------- 1 file changed, 31 insertions(+), 18 deletions(-) diff --git a/pageserver/src/tenant/timeline/handle.rs b/pageserver/src/tenant/timeline/handle.rs index e82559b8b3..dc45dca9cd 100644 --- a/pageserver/src/tenant/timeline/handle.rs +++ b/pageserver/src/tenant/timeline/handle.rs @@ -107,12 +107,13 @@ use std::collections::hash_map; use std::collections::HashMap; -use std::sync::atomic::AtomicBool; +use std::ops::Deref; use std::sync::atomic::Ordering; use std::sync::Arc; use std::sync::Mutex; use std::sync::Weak; +use futures::lock::OwnedMutexGuard; use pageserver_api::shard::ShardIdentity; use tracing::instrument; use tracing::trace; @@ -170,12 +171,17 @@ pub(crate) struct ShardTimelineId { } /// See module-level comment. -pub(crate) struct Handle(Arc>); -struct HandleInner { - shut_down: AtomicBool, - timeline: T::Timeline, - // The timeline's gate held open. - _gate_guard: utils::sync::gate::GateGuard, +pub(crate) struct Handle( + // We need sth with interior mutability + Send-ness. + // tokio sync Mutex gives that, and adds OwnedMutexGuard, which is handy. + Arc>>, +); +enum HandleInner { + KeepingTimelineGateOpen { + timeline: T::Timeline, + gate_guard: utils::sync::gate::GateGuard, + }, + ShutDown, } /// Embedded in each [`Types::Timeline`] as the anchor for the only long-lived strong ref to `HandleInner`. @@ -248,7 +254,7 @@ impl Cache { let handle = self .get_impl(timeline_id, shard_selector, tenant_manager) .await?; - if handle.0.shut_down.load(Ordering::Relaxed) { + if handle.0.check_shut_down().is_none() { let removed = self .map .remove(&handle.0.timeline.shard_timeline_id()) @@ -367,10 +373,9 @@ impl Cache { let handle = Arc::new( // TODO: global metric that keeps track of the number of live HandlerTimeline instances // so we can identify reference cycle bugs. - HandleInner { - shut_down: AtomicBool::new(false), - _gate_guard: gate_guard, + HandleInner::KeepingTimelineGateOpen { timeline: timeline.clone(), + gate_guard, }, ); let handle = { @@ -409,6 +414,21 @@ impl Cache { } } +impl Handle { + pub(crate) async fn upgrade(&mut self) -> Option> { + // This could probably simplified with a + let guard = self.0.lock_owned().await; + let map_res = OwnedMutexGuard::try_map(guard, |f: &mut HandleInner| match f { + HandleInner::KeepingTimelineGateOpen { timeline, .. } => Some(&mut timeline), + HandleInner::ShutDown => None, + }); + let Ok(timeline) = map_res else { + return None; + }; + Some(timeline) + } +} + impl PerTimelineState { /// After this method returns, [`Cache::get`] will never again return a [`Handle`] /// to the [`Types::Timeline`] that embeds this per-timeline state. @@ -438,13 +458,6 @@ impl PerTimelineState { } } -impl std::ops::Deref for Handle { - type Target = T::Timeline; - fn deref(&self) -> &Self::Target { - &self.0.timeline - } -} - #[cfg(test)] impl Drop for HandleInner { fn drop(&mut self) { From d1bc36f53688452dfacb7e38c546d49fce73c9f3 Mon Sep 17 00:00:00 2001 From: John Spray Date: Mon, 13 Jan 2025 13:31:57 +0000 Subject: [PATCH 054/111] storage controller: fix retries of compute hook notifications while a secondary node is offline (#10352) ## Problem We would sometimes fail to retry compute notifications: 1. Try and send, set compute_notify_failure if we can't 2. On next reconcile, reconcile() fails for some other reason (e.g. tried to talk to an offline node), and we fail the `result.is_ok() && must_notify` condition around the re-sending. Closes: https://github.com/neondatabase/cloud/issues/22612 ## Summary of changes - Clarify the meaning of the reconcile result: it should be Ok(()) if configuring attached location worked, even if secondary or detach locations cannot be reached. - Skip trying to talk to secondaries if they're offline - Even if reconcile fails and we can't send the compute notification (we can't send it because we're not sure if it's really attached), make sure we save the `compute_notify_failure` flag so that subsequent reconciler runs will try again - Add a regression test for the above --- storage_controller/src/reconciler.rs | 21 +++- storage_controller/src/tenant_shard.rs | 7 +- .../regress/test_storage_controller.py | 116 ++++++++++++++++++ 3 files changed, 139 insertions(+), 5 deletions(-) diff --git a/storage_controller/src/reconciler.rs b/storage_controller/src/reconciler.rs index e0a854fff7..6f584e7267 100644 --- a/storage_controller/src/reconciler.rs +++ b/storage_controller/src/reconciler.rs @@ -696,6 +696,11 @@ impl Reconciler { /// First we apply special case handling (e.g. for live migrations), and then a /// general case reconciliation where we walk through the intent by pageserver /// and call out to the pageserver to apply the desired state. + /// + /// An Ok(()) result indicates that we successfully attached the tenant, but _not_ that + /// all locations for the tenant are in the expected state. When nodes that are to be detached + /// or configured as secondary are unavailable, we may return Ok(()) but leave the shard in a + /// state where it still requires later reconciliation. pub(crate) async fn reconcile(&mut self) -> Result<(), ReconcileError> { // Prepare: if we have uncertain `observed` state for our would-be attachement location, then refresh it self.maybe_refresh_observed().await?; @@ -784,10 +789,18 @@ impl Reconciler { tracing::info!(node_id=%node.get_id(), "Observed configuration already correct.") } _ => { - // In all cases other than a matching observed configuration, we will - // reconcile this location. - tracing::info!(node_id=%node.get_id(), "Observed configuration requires update."); - changes.push((node.clone(), wanted_conf)) + // Only try and configure secondary locations on nodes that are available. This + // allows the reconciler to "succeed" while some secondaries are offline (e.g. after + // a node failure, where the failed node will have a secondary intent) + if node.is_available() { + tracing::info!(node_id=%node.get_id(), "Observed configuration requires update."); + changes.push((node.clone(), wanted_conf)) + } else { + tracing::info!(node_id=%node.get_id(), "Skipping configuration as secondary, node is unavailable"); + self.observed + .locations + .insert(node.get_id(), ObservedStateLocation { conf: None }); + } } } } diff --git a/storage_controller/src/tenant_shard.rs b/storage_controller/src/tenant_shard.rs index c17989a316..e0a71b5822 100644 --- a/storage_controller/src/tenant_shard.rs +++ b/storage_controller/src/tenant_shard.rs @@ -1122,10 +1122,15 @@ impl TenantShard { let result = reconciler.reconcile().await; // If we know we had a pending compute notification from some previous action, send a notification irrespective - // of whether the above reconcile() did any work + // of whether the above reconcile() did any work. It has to be Ok() though, because otherwise we might be + // sending a notification of a location that isn't really attached. if result.is_ok() && must_notify { // If this fails we will send the need to retry in [`ReconcileResult::pending_compute_notification`] reconciler.compute_notify().await.ok(); + } else if must_notify { + // Carry this flag so that the reconciler's result will indicate that it still needs to retry + // the compute hook notification eventually. + reconciler.compute_notify_failure = true; } // Update result counter diff --git a/test_runner/regress/test_storage_controller.py b/test_runner/regress/test_storage_controller.py index da6d5b8622..616aee758d 100644 --- a/test_runner/regress/test_storage_controller.py +++ b/test_runner/regress/test_storage_controller.py @@ -822,6 +822,122 @@ def test_storage_controller_stuck_compute_hook( env.storage_controller.consistency_check() +@run_only_on_default_postgres("postgres behavior is not relevant") +def test_storage_controller_compute_hook_retry( + httpserver: HTTPServer, + neon_env_builder: NeonEnvBuilder, + httpserver_listen_address: ListenAddress, +): + """ + Test that when a reconciler can't do its compute hook notification, it will keep + trying until it succeeds. + + Reproducer for https://github.com/neondatabase/cloud/issues/22612 + """ + + neon_env_builder.num_pageservers = 2 + (host, port) = httpserver_listen_address + neon_env_builder.control_plane_compute_hook_api = f"http://{host}:{port}/notify" + + handle_params = {"status": 200} + + notifications = [] + + def handler(request: Request): + status = handle_params["status"] + log.info(f"Notify request[{status}]: {request}") + notifications.append(request.json) + return Response(status=status) + + httpserver.expect_request("/notify", method="PUT").respond_with_handler(handler) + + # Start running + env = neon_env_builder.init_configs() + env.start() + + tenant_id = TenantId.generate() + env.create_tenant(tenant_id, placement_policy='{"Attached": 1}') + + # Initial notification from tenant creation + assert len(notifications) == 1 + expect: dict[str, list[dict[str, int]] | str | None | int] = { + "tenant_id": str(tenant_id), + "stripe_size": None, + "shards": [{"node_id": int(env.pageservers[0].id), "shard_number": 0}], + "preferred_az": DEFAULT_AZ_ID, + } + assert notifications[0] == expect + + # Block notifications, and fail a node + handle_params["status"] = 423 + env.pageservers[0].stop() + env.storage_controller.allowed_errors.append(NOTIFY_BLOCKED_LOG) + env.storage_controller.allowed_errors.extend(NOTIFY_FAILURE_LOGS) + + # Avoid waiting for heartbeats + env.storage_controller.node_configure(env.pageservers[0].id, {"availability": "Offline"}) + + # Make reconciler run and fail: it should leave itself in a state where the shard will retry notification later, + # and we will check that that happens + notifications = [] + try: + assert env.storage_controller.reconcile_all() == 1 + except StorageControllerApiException as e: + assert "Control plane tenant busy" in str(e) + assert len(notifications) == 1 + assert ( + env.storage_controller.tenant_describe(tenant_id)["shards"][0][ + "is_pending_compute_notification" + ] + is True + ) + + # Try reconciling again, it should try notifying again + notifications = [] + try: + assert env.storage_controller.reconcile_all() == 1 + except StorageControllerApiException as e: + assert "Control plane tenant busy" in str(e) + assert len(notifications) == 1 + assert ( + env.storage_controller.tenant_describe(tenant_id)["shards"][0][ + "is_pending_compute_notification" + ] + is True + ) + + # The describe API should indicate that a notification is pending + assert ( + env.storage_controller.tenant_describe(tenant_id)["shards"][0][ + "is_pending_compute_notification" + ] + is True + ) + + # Unblock notifications: reconcile should work now + handle_params["status"] = 200 + notifications = [] + assert env.storage_controller.reconcile_all() == 1 + assert len(notifications) == 1 + assert ( + env.storage_controller.tenant_describe(tenant_id)["shards"][0][ + "is_pending_compute_notification" + ] + is False + ) + + # Reconciler should be idle now that it succeeded in its compute notification + notifications = [] + assert env.storage_controller.reconcile_all() == 0 + assert len(notifications) == 0 + assert ( + env.storage_controller.tenant_describe(tenant_id)["shards"][0][ + "is_pending_compute_notification" + ] + is False + ) + + @run_only_on_default_postgres("this test doesn't start an endpoint") def test_storage_controller_compute_hook_revert( httpserver: HTTPServer, From dda31b9cb6c2adad794c6ba7c2ad445eeff9e2fd Mon Sep 17 00:00:00 2001 From: Christian Schwarz Date: Mon, 13 Jan 2025 14:34:15 +0100 Subject: [PATCH 055/111] adjust shutdown --- pageserver/src/tenant/timeline.rs | 2 +- pageserver/src/tenant/timeline/handle.rs | 35 +++++++++++------------- 2 files changed, 17 insertions(+), 20 deletions(-) diff --git a/pageserver/src/tenant/timeline.rs b/pageserver/src/tenant/timeline.rs index e222a624de..75dab4e7dd 100644 --- a/pageserver/src/tenant/timeline.rs +++ b/pageserver/src/tenant/timeline.rs @@ -1826,7 +1826,7 @@ impl Timeline { self.cancel.cancel(); // Ensure Prevent new page service requests from starting. - self.handles.shutdown(); + self.handles.shutdown().await; // Transition the remote_client into a state where it's only useful for timeline deletion. // (The deletion use case is why we can't just hook up remote_client to Self::cancel).) diff --git a/pageserver/src/tenant/timeline/handle.rs b/pageserver/src/tenant/timeline/handle.rs index dc45dca9cd..e3c40b5f35 100644 --- a/pageserver/src/tenant/timeline/handle.rs +++ b/pageserver/src/tenant/timeline/handle.rs @@ -189,7 +189,7 @@ enum HandleInner { /// See module-level comment for details. pub struct PerTimelineState { // None = shutting down - handles: Mutex>>>>, + handles: Mutex>>>>>, } impl Default for PerTimelineState { @@ -430,14 +430,12 @@ impl Handle { } impl PerTimelineState { - /// After this method returns, [`Cache::get`] will never again return a [`Handle`] - /// to the [`Types::Timeline`] that embeds this per-timeline state. - /// Even if [`TenantManager::resolve`] would still resolve to it. + /// Invalidate all handles to this timeline in all [`Cache`]s. /// - /// Already-alive [`Handle`]s for will remain open, usable, and keeping the [`ArcTimeline`] alive. - /// That's ok because they're short-lived. See module-level comment for details. + /// After this method returns, all subsequent [`Handle::upgrade`] will fail + /// and they will not be holding the [`ArcTimeline`]'s gate open. #[instrument(level = "trace", skip_all)] - pub(super) fn shutdown(&self) { + pub(super) async fn shutdown(&self) { let handles = self .handles .lock() @@ -451,20 +449,19 @@ impl PerTimelineState { return; }; for handle in handles.values() { - // Make hits fail. - handle.shut_down.store(true, Ordering::Relaxed); + // Make further cache hits + let mut lock_guard = handle.lock().await; + match std::mem::replace(&mut *lock_guard, HandleInner::ShutDown) { + HandleInner::KeepingTimelineGateOpen { gate_guard, .. } => { + drop(gate_guard); + } + HandleInner::ShutDown => unreachable!(), + } } drop(handles); } } -#[cfg(test)] -impl Drop for HandleInner { - fn drop(&mut self) { - trace!("HandleInner dropped"); - } -} - // When dropping a [`Cache`], prune its handles in the [`PerTimelineState`] to break the reference cycle. impl Drop for Cache { fn drop(&mut self) { @@ -651,7 +648,7 @@ mod tests { assert!(Weak::ptr_eq(&handle.myself, &shard0.myself)); // SHUTDOWN - shard0.per_timeline_state.shutdown(); // keeping handle alive across shutdown + shard0.per_timeline_state.shutdown().await; // keeping handle alive across shutdown assert_eq!( 1, @@ -764,7 +761,7 @@ mod tests { assert_eq!(cache.map.len(), 2); // delete timeline A - timeline_a.per_timeline_state.shutdown(); + timeline_a.per_timeline_state.shutdown().await; mgr.shards.retain(|t| t.id != timeline_a.id); assert!( mgr.resolve(timeline_a.id, ShardSelector::Page(key)) @@ -892,7 +889,7 @@ mod tests { assert!(Weak::ptr_eq(&parent_handle.myself, &parent.myself)); // invalidate the cache - parent.per_timeline_state.shutdown(); + parent.per_timeline_state.shutdown().await; // the cache will now return the child, even though the parent handle still exists for i in 0..2 { From 8a0a0d06a8f96409e31e3bb7c911511e4cb3226a Mon Sep 17 00:00:00 2001 From: Christian Schwarz Date: Mon, 13 Jan 2025 14:35:14 +0100 Subject: [PATCH 056/111] renames --- pageserver/src/tenant/timeline/handle.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pageserver/src/tenant/timeline/handle.rs b/pageserver/src/tenant/timeline/handle.rs index e3c40b5f35..68ba5929e5 100644 --- a/pageserver/src/tenant/timeline/handle.rs +++ b/pageserver/src/tenant/timeline/handle.rs @@ -417,15 +417,15 @@ impl Cache { impl Handle { pub(crate) async fn upgrade(&mut self) -> Option> { // This could probably simplified with a - let guard = self.0.lock_owned().await; - let map_res = OwnedMutexGuard::try_map(guard, |f: &mut HandleInner| match f { + let lock_guard = self.0.lock_owned().await; + let res = OwnedMutexGuard::try_map(lock_guard, |f: &mut HandleInner| match f { HandleInner::KeepingTimelineGateOpen { timeline, .. } => Some(&mut timeline), HandleInner::ShutDown => None, }); - let Ok(timeline) = map_res else { + let Ok(mapped_lock_guard) = res else { return None; }; - Some(timeline) + Some(mapped_lock_guard) } } From b2d0e1a519616aea5e0d477c1f12940e44d2fee5 Mon Sep 17 00:00:00 2001 From: Alexander Bayandin Date: Mon, 13 Jan 2025 14:13:02 +0000 Subject: [PATCH 057/111] Link OpenSSL dynamically (#10302) ## Problem Statically linked OpenSSL is buggy in multithreaded environment: - https://github.com/neondatabase/cloud/issues/16155 - https://github.com/neondatabase/neon/issues/8275 ## Summary of changes - Link OpenSSL dynamically (revert OpenSSL part from https://github.com/neondatabase/neon/pull/8074) Before: ``` ldd /usr/local/v17/lib/libpq.so linux-vdso.so.1 (0x0000ffffb5ce4000) libm.so.6 => /lib/aarch64-linux-gnu/libm.so.6 (0x0000ffffb5c10000) libc.so.6 => /lib/aarch64-linux-gnu/libc.so.6 (0x0000ffffb5650000) /lib/ld-linux-aarch64.so.1 (0x0000ffffb5ca7000) ``` After: ``` ldd /usr/local/v17/lib/libpq.so linux-vdso.so.1 (0x0000ffffbf3e8000) libssl.so.3 => /lib/aarch64-linux-gnu/libssl.so.3 (0x0000ffffbf260000) libcrypto.so.3 => /lib/aarch64-linux-gnu/libcrypto.so.3 (0x0000ffffbec00000) libm.so.6 => /lib/aarch64-linux-gnu/libm.so.6 (0x0000ffffbf1c0000) libc.so.6 => /lib/aarch64-linux-gnu/libc.so.6 (0x0000ffffbea50000) /lib/ld-linux-aarch64.so.1 (0x0000ffffbf3ab000) ``` --- Dockerfile | 1 + Makefile | 3 --- build-tools.Dockerfile | 15 --------------- 3 files changed, 1 insertion(+), 18 deletions(-) diff --git a/Dockerfile b/Dockerfile index d3659f917a..2e4f8e5546 100644 --- a/Dockerfile +++ b/Dockerfile @@ -71,6 +71,7 @@ RUN set -e \ ca-certificates \ # System postgres for use with client libraries (e.g. in storage controller) postgresql-15 \ + openssl \ && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* \ && useradd -d /data neon \ && chown -R neon:neon /data diff --git a/Makefile b/Makefile index 9cffc74508..22ebfea7d5 100644 --- a/Makefile +++ b/Makefile @@ -3,7 +3,6 @@ ROOT_PROJECT_DIR := $(dir $(abspath $(lastword $(MAKEFILE_LIST)))) # Where to install Postgres, default is ./pg_install, maybe useful for package managers POSTGRES_INSTALL_DIR ?= $(ROOT_PROJECT_DIR)/pg_install/ -OPENSSL_PREFIX_DIR := /usr/local/openssl ICU_PREFIX_DIR := /usr/local/icu # @@ -26,11 +25,9 @@ endif ifeq ($(shell test -e /home/nonroot/.docker_build && echo -n yes),yes) # Exclude static build openssl, icu for local build (MacOS, Linux) # Only keep for build type release and debug - PG_CFLAGS += -I$(OPENSSL_PREFIX_DIR)/include PG_CONFIGURE_OPTS += --with-icu PG_CONFIGURE_OPTS += ICU_CFLAGS='-I/$(ICU_PREFIX_DIR)/include -DU_STATIC_IMPLEMENTATION' PG_CONFIGURE_OPTS += ICU_LIBS='-L$(ICU_PREFIX_DIR)/lib -L$(ICU_PREFIX_DIR)/lib64 -licui18n -licuuc -licudata -lstdc++ -Wl,-Bdynamic -lm' - PG_CONFIGURE_OPTS += LDFLAGS='-L$(OPENSSL_PREFIX_DIR)/lib -L$(OPENSSL_PREFIX_DIR)/lib64 -L$(ICU_PREFIX_DIR)/lib -L$(ICU_PREFIX_DIR)/lib64 -Wl,-Bstatic -lssl -lcrypto -Wl,-Bdynamic -lrt -lm -ldl -lpthread' endif UNAME_S := $(shell uname -s) diff --git a/build-tools.Dockerfile b/build-tools.Dockerfile index 79210a2e1b..cf6439d004 100644 --- a/build-tools.Dockerfile +++ b/build-tools.Dockerfile @@ -190,21 +190,6 @@ RUN for package in Capture::Tiny DateTime Devel::Cover Digest::MD5 File::Spec JS && make install \ && rm -rf ../lcov.tar.gz -# Compile and install the static OpenSSL library -ENV OPENSSL_VERSION=1.1.1w -ENV OPENSSL_PREFIX=/usr/local/openssl -RUN wget -O /tmp/openssl-${OPENSSL_VERSION}.tar.gz https://www.openssl.org/source/openssl-${OPENSSL_VERSION}.tar.gz && \ - echo "cf3098950cb4d853ad95c0841f1f9c6d3dc102dccfcacd521d93925208b76ac8 /tmp/openssl-${OPENSSL_VERSION}.tar.gz" | sha256sum --check && \ - cd /tmp && \ - tar xzvf /tmp/openssl-${OPENSSL_VERSION}.tar.gz && \ - rm /tmp/openssl-${OPENSSL_VERSION}.tar.gz && \ - cd /tmp/openssl-${OPENSSL_VERSION} && \ - ./config --prefix=${OPENSSL_PREFIX} -static --static no-shared -fPIC && \ - make -j "$(nproc)" && \ - make install && \ - cd /tmp && \ - rm -rf /tmp/openssl-${OPENSSL_VERSION} - # Use the same version of libicu as the compute nodes so that # clusters created using inidb on pageserver can be used by computes. # From b31ed0acd1c5e20281556de66fa52d797c156826 Mon Sep 17 00:00:00 2001 From: Erik Grinaker Date: Mon, 13 Jan 2025 15:23:42 +0100 Subject: [PATCH 058/111] utils: add ?force=true hint for CPU profiler (#10368) This makes it less annoying to try to take a CPU profile when a continuous profile is already running. --- libs/utils/src/http/endpoint.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/libs/utils/src/http/endpoint.rs b/libs/utils/src/http/endpoint.rs index ca65c39ad6..9f38373ca0 100644 --- a/libs/utils/src/http/endpoint.rs +++ b/libs/utils/src/http/endpoint.rs @@ -372,7 +372,11 @@ pub async fn profile_cpu_handler(req: Request) -> Result, A match PROFILE_LOCK.try_lock() { Ok(lock) => break lock, Err(_) if force => PROFILE_CANCEL.notify_waiters(), - Err(_) => return Err(ApiError::Conflict("profiler already running".into())), + Err(_) => { + return Err(ApiError::Conflict( + "profiler already running (use ?force=true to cancel it)".into(), + )) + } } tokio::time::sleep(Duration::from_millis(1)).await; // don't busy-wait }; From ceacc29609058f7ba18288ac141e7c55a4f01382 Mon Sep 17 00:00:00 2001 From: Konstantin Knizhnik Date: Mon, 13 Jan 2025 16:26:11 +0200 Subject: [PATCH 059/111] Start with minimal prefetch distance to minimize prefetch overhead for exact or limited index scans (#10359) ## Problem See https://neondb.slack.com/archives/C04DGM6SMTM/p1736526089437179 In case of queries index scan with LIMIT clause, multiple backends can concurrently send larger number of duplicated prefetch requests which are not stored in LFC and so actually do useless job. Current implementation of index prefetch starts with maximal prefetch distance (10 by default now) when there are no key bounds, so in queries with LIMIT clause like `select * from T order by pk limit 1` compute can send a lot of useless prefetch requests to page server. ## Summary of changes Always start with minimal prefetch distance even if there are not key boundaries. Related Postgres PRs: https://github.com/neondatabase/postgres/pull/552 https://github.com/neondatabase/postgres/pull/551 https://github.com/neondatabase/postgres/pull/550 https://github.com/neondatabase/postgres/pull/549 Co-authored-by: Konstantin Knizhnik --- vendor/postgres-v14 | 2 +- vendor/postgres-v15 | 2 +- vendor/postgres-v16 | 2 +- vendor/postgres-v17 | 2 +- vendor/revisions.json | 8 ++++---- 5 files changed, 8 insertions(+), 8 deletions(-) diff --git a/vendor/postgres-v14 b/vendor/postgres-v14 index c2f65b3201..210a0ba3af 160000 --- a/vendor/postgres-v14 +++ b/vendor/postgres-v14 @@ -1 +1 @@ -Subproject commit c2f65b3201591e02ce45b66731392f98d3388e73 +Subproject commit 210a0ba3afd8134ea910b203f274b165bd4f05d7 diff --git a/vendor/postgres-v15 b/vendor/postgres-v15 index f262d631ad..d3141e17a7 160000 --- a/vendor/postgres-v15 +++ b/vendor/postgres-v15 @@ -1 +1 @@ -Subproject commit f262d631ad477a1819e84a183e5a7ef561830085 +Subproject commit d3141e17a7155e3d07c8deba4a10c748a29ba1e6 diff --git a/vendor/postgres-v16 b/vendor/postgres-v16 index 97f9fde349..f63b141cfb 160000 --- a/vendor/postgres-v16 +++ b/vendor/postgres-v16 @@ -1 +1 @@ -Subproject commit 97f9fde349c6de6d573f5ce96db07eca60ce6185 +Subproject commit f63b141cfb0c813725a6b2574049565bff643018 diff --git a/vendor/postgres-v17 b/vendor/postgres-v17 index 7e3f3974bc..9c9e9a78a9 160000 --- a/vendor/postgres-v17 +++ b/vendor/postgres-v17 @@ -1 +1 @@ -Subproject commit 7e3f3974bc8895938308f94d0e96879ffae638cd +Subproject commit 9c9e9a78a93aebec2f6a2f54644442d35ffa245c diff --git a/vendor/revisions.json b/vendor/revisions.json index bff2f70931..d182b88008 100644 --- a/vendor/revisions.json +++ b/vendor/revisions.json @@ -1,18 +1,18 @@ { "v17": [ "17.2", - "7e3f3974bc8895938308f94d0e96879ffae638cd" + "9c9e9a78a93aebec2f6a2f54644442d35ffa245c" ], "v16": [ "16.6", - "97f9fde349c6de6d573f5ce96db07eca60ce6185" + "f63b141cfb0c813725a6b2574049565bff643018" ], "v15": [ "15.10", - "f262d631ad477a1819e84a183e5a7ef561830085" + "d3141e17a7155e3d07c8deba4a10c748a29ba1e6" ], "v14": [ "14.15", - "c2f65b3201591e02ce45b66731392f98d3388e73" + "210a0ba3afd8134ea910b203f274b165bd4f05d7" ] } From 9e03dda0c387e6ec1a6866aed982e78f1f25aab4 Mon Sep 17 00:00:00 2001 From: Christian Schwarz Date: Mon, 13 Jan 2025 15:49:45 +0100 Subject: [PATCH 060/111] handle downgrade during batching --- pageserver/src/page_service.rs | 46 ++++++++++++++------- pageserver/src/tenant/timeline/handle.rs | 51 ++++++++++++++++-------- 2 files changed, 65 insertions(+), 32 deletions(-) diff --git a/pageserver/src/page_service.rs b/pageserver/src/page_service.rs index b3e18fed99..cf2276f65e 100644 --- a/pageserver/src/page_service.rs +++ b/pageserver/src/page_service.rs @@ -558,31 +558,31 @@ enum BatchedFeMessage { Exists { span: Span, timer: SmgrOpTimer, - shard: timeline::handle::Handle, + shard: timeline::handle::WeakHandle, req: models::PagestreamExistsRequest, }, Nblocks { span: Span, timer: SmgrOpTimer, - shard: timeline::handle::Handle, + shard: timeline::handle::WeakHandle, req: models::PagestreamNblocksRequest, }, GetPage { span: Span, - shard: timeline::handle::Handle, + shard: timeline::handle::WeakHandle, effective_request_lsn: Lsn, pages: smallvec::SmallVec<[BatchedGetPageRequest; 1]>, }, DbSize { span: Span, timer: SmgrOpTimer, - shard: timeline::handle::Handle, + shard: timeline::handle::WeakHandle, req: models::PagestreamDbSizeRequest, }, GetSlruSegment { span: Span, timer: SmgrOpTimer, - shard: timeline::handle::Handle, + shard: timeline::handle::WeakHandle, req: models::PagestreamGetSlruSegmentRequest, }, RespondError { @@ -734,7 +734,7 @@ impl PageServerHandler { BatchedFeMessage::Exists { span, timer, - shard, + shard: shard.downgrade(), req, } } @@ -753,7 +753,7 @@ impl PageServerHandler { BatchedFeMessage::Nblocks { span, timer, - shard, + shard: shard.downgrade(), req, } } @@ -772,7 +772,7 @@ impl PageServerHandler { BatchedFeMessage::DbSize { span, timer, - shard, + shard: shard.downgrade(), req, } } @@ -791,7 +791,7 @@ impl PageServerHandler { BatchedFeMessage::GetSlruSegment { span, timer, - shard, + shard: shard.downgrade(), req, } } @@ -843,6 +843,7 @@ impl PageServerHandler { ) .await?; + // We're holding the Handle let effective_request_lsn = match Self::wait_or_get_last_lsn( &shard, req.hdr.request_lsn, @@ -860,7 +861,7 @@ impl PageServerHandler { }; BatchedFeMessage::GetPage { span, - shard, + shard: shard.downgrade(), effective_request_lsn, pages: smallvec::smallvec![BatchedGetPageRequest { req, timer }], } @@ -968,7 +969,14 @@ impl PageServerHandler { fail::fail_point!("ps::handle-pagerequest-message::exists"); ( vec![self - .handle_get_rel_exists_request(&shard, &req, ctx) + .handle_get_rel_exists_request( + &*shard + .upgrade() + // TODO: need sensitive to cancel() + .await?, + &req, + ctx, + ) .instrument(span.clone()) .await .map(|msg| (msg, timer)) @@ -985,7 +993,7 @@ impl PageServerHandler { fail::fail_point!("ps::handle-pagerequest-message::nblocks"); ( vec![self - .handle_get_nblocks_request(&shard, &req, ctx) + .handle_get_nblocks_request(&*shard.upgrade().await?, &req, ctx) .instrument(span.clone()) .await .map(|msg| (msg, timer)) @@ -1006,7 +1014,7 @@ impl PageServerHandler { trace!(npages, "handling getpage request"); let res = self .handle_get_page_at_lsn_request_batched( - &shard, + &*shard.upgrade().await?, effective_request_lsn, pages, ctx, @@ -1028,7 +1036,7 @@ impl PageServerHandler { fail::fail_point!("ps::handle-pagerequest-message::dbsize"); ( vec![self - .handle_db_size_request(&shard, &req, ctx) + .handle_db_size_request(&*shard.upgrade().await?, &req, ctx) .instrument(span.clone()) .await .map(|msg| (msg, timer)) @@ -1045,7 +1053,7 @@ impl PageServerHandler { fail::fail_point!("ps::handle-pagerequest-message::slrusegment"); ( vec![self - .handle_get_slru_segment_request(&shard, &req, ctx) + .handle_get_slru_segment_request(&*shard.upgrade().await?, &req, ctx) .instrument(span.clone()) .await .map(|msg| (msg, timer)) @@ -2391,6 +2399,14 @@ impl From for QueryError { } } +impl From for QueryError { + fn from(e: crate::tenant::timeline::handle::HandleUpgradeError) -> Self { + match e { + crate::tenant::timeline::handle::HandleUpgradeError::ShutDown => QueryError::Shutdown, + } + } +} + fn set_tracing_field_shard_id(timeline: &Timeline) { debug_assert_current_span_has_tenant_and_timeline_id_no_shard_id(); tracing::Span::current().record( diff --git a/pageserver/src/tenant/timeline/handle.rs b/pageserver/src/tenant/timeline/handle.rs index 68ba5929e5..983eae89ca 100644 --- a/pageserver/src/tenant/timeline/handle.rs +++ b/pageserver/src/tenant/timeline/handle.rs @@ -121,6 +121,7 @@ use utils::id::TimelineId; use utils::shard::ShardIndex; use utils::shard::ShardNumber; +use crate::page_service::GetActiveTimelineError; use crate::tenant::mgr::ShardSelector; /// The requirement for Debug is so that #[derive(Debug)] works in some places. @@ -153,7 +154,7 @@ pub(crate) struct Cache { map: Map, } -type Map = HashMap>>; +type Map = HashMap>; impl Default for Cache { fn default() -> Self { @@ -171,11 +172,8 @@ pub(crate) struct ShardTimelineId { } /// See module-level comment. -pub(crate) struct Handle( - // We need sth with interior mutability + Send-ness. - // tokio sync Mutex gives that, and adds OwnedMutexGuard, which is handy. - Arc>>, -); +pub(crate) struct Handle(tokio::sync::OwnedMutexGuard>); +pub(crate) struct WeakHandle(Arc>>); enum HandleInner { KeepingTimelineGateOpen { timeline: T::Timeline, @@ -414,18 +412,37 @@ impl Cache { } } -impl Handle { - pub(crate) async fn upgrade(&mut self) -> Option> { - // This could probably simplified with a +pub(crate) enum HandleUpgradeError { + ShutDown, +} + +impl WeakHandle { + pub(crate) async fn upgrade(&self) -> Result, HandleUpgradeError> { let lock_guard = self.0.lock_owned().await; - let res = OwnedMutexGuard::try_map(lock_guard, |f: &mut HandleInner| match f { - HandleInner::KeepingTimelineGateOpen { timeline, .. } => Some(&mut timeline), - HandleInner::ShutDown => None, - }); - let Ok(mapped_lock_guard) = res else { - return None; - }; - Some(mapped_lock_guard) + match &*lock_guard { + HandleInner::KeepingTimelineGateOpen { + timeline, + gate_guard, + } => Ok(Handle(lock_guard)), + HandleInner::ShutDown => Err(HandleUpgradeError::ShutDown), + } + } +} + +impl Deref for Handle { + type Target = T::Timeline; + + fn deref(&self) -> &Self::Target { + match &*self.0 { + HandleInner::KeepingTimelineGateOpen { timeline, .. } => timeline, + HandleInner::ShutDown => unreachable!(), + } + } +} + +impl Handle { + pub(crate) fn downgrade(&self) -> WeakHandle { + WeakHandle(Arc::clone(&self.0.mutex())) } } From ef8bfacd6b1a9fa55923e0158ba096afc6090143 Mon Sep 17 00:00:00 2001 From: John Spray Date: Mon, 13 Jan 2025 14:52:43 +0000 Subject: [PATCH 061/111] storage controller: API + CLI for migrating secondary locations (#10284) ## Problem Currently, if we want to move a secondary there isn't a neat way to do that: we just have migration API for the attached location, and it is only clean to use that if you've manually created a secondary via pageserver API in the place you're going to move it to. Secondary migration API enables: - Moving the secondary somewhere because we would like to later move the attached location there. - Move the secondary location because we just want to reclaim some disk space from its current location. ## Summary of changes - Add `/migrate_secondary` API - Add `tenant-shard-migrate-secondary` CLI - Add tests for above --- control_plane/src/storage_controller.rs | 5 +- control_plane/storcon_cli/src/main.rs | 31 ++++-- libs/pageserver_api/src/controller_api.rs | 1 - storage_controller/src/compute_hook.rs | 10 +- storage_controller/src/http.rs | 36 ++++++- storage_controller/src/service.rs | 63 ++++++++++++ .../regress/test_storage_controller.py | 96 +++++++++++++++---- 7 files changed, 210 insertions(+), 32 deletions(-) diff --git a/control_plane/src/storage_controller.rs b/control_plane/src/storage_controller.rs index 22d2420ed4..c41ff22d15 100644 --- a/control_plane/src/storage_controller.rs +++ b/control_plane/src/storage_controller.rs @@ -822,10 +822,7 @@ impl StorageController { self.dispatch( Method::PUT, format!("control/v1/tenant/{tenant_shard_id}/migrate"), - Some(TenantShardMigrateRequest { - tenant_shard_id, - node_id, - }), + Some(TenantShardMigrateRequest { node_id }), ) .await } diff --git a/control_plane/storcon_cli/src/main.rs b/control_plane/storcon_cli/src/main.rs index 617b2cd1ba..1653b3c845 100644 --- a/control_plane/storcon_cli/src/main.rs +++ b/control_plane/storcon_cli/src/main.rs @@ -112,6 +112,13 @@ enum Command { #[arg(long)] node: NodeId, }, + /// Migrate the secondary location for a tenant shard to a specific pageserver. + TenantShardMigrateSecondary { + #[arg(long)] + tenant_shard_id: TenantShardId, + #[arg(long)] + node: NodeId, + }, /// Cancel any ongoing reconciliation for this shard TenantShardCancelReconcile { #[arg(long)] @@ -540,10 +547,7 @@ async fn main() -> anyhow::Result<()> { tenant_shard_id, node, } => { - let req = TenantShardMigrateRequest { - tenant_shard_id, - node_id: node, - }; + let req = TenantShardMigrateRequest { node_id: node }; storcon_client .dispatch::( @@ -553,6 +557,20 @@ async fn main() -> anyhow::Result<()> { ) .await?; } + Command::TenantShardMigrateSecondary { + tenant_shard_id, + node, + } => { + let req = TenantShardMigrateRequest { node_id: node }; + + storcon_client + .dispatch::( + Method::PUT, + format!("control/v1/tenant/{tenant_shard_id}/migrate_secondary"), + Some(req), + ) + .await?; + } Command::TenantShardCancelReconcile { tenant_shard_id } => { storcon_client .dispatch::<(), ()>( @@ -915,10 +933,7 @@ async fn main() -> anyhow::Result<()> { .dispatch::( Method::PUT, format!("control/v1/tenant/{}/migrate", mv.tenant_shard_id), - Some(TenantShardMigrateRequest { - tenant_shard_id: mv.tenant_shard_id, - node_id: mv.to, - }), + Some(TenantShardMigrateRequest { node_id: mv.to }), ) .await .map_err(|e| (mv.tenant_shard_id, mv.from, mv.to, e)) diff --git a/libs/pageserver_api/src/controller_api.rs b/libs/pageserver_api/src/controller_api.rs index 7eb3547183..ba0eb0e4ae 100644 --- a/libs/pageserver_api/src/controller_api.rs +++ b/libs/pageserver_api/src/controller_api.rs @@ -179,7 +179,6 @@ pub struct TenantDescribeResponseShard { /// specifies some constraints, e.g. asking it to get off particular node(s) #[derive(Serialize, Deserialize, Debug)] pub struct TenantShardMigrateRequest { - pub tenant_shard_id: TenantShardId, pub node_id: NodeId, } diff --git a/storage_controller/src/compute_hook.rs b/storage_controller/src/compute_hook.rs index 69db48f8d1..3884a6df46 100644 --- a/storage_controller/src/compute_hook.rs +++ b/storage_controller/src/compute_hook.rs @@ -124,7 +124,10 @@ impl ComputeHookTenant { if let Some(shard_idx) = shard_idx { sharded.shards.remove(shard_idx); } else { - tracing::warn!("Shard not found while handling detach") + // This is a valid but niche case, where the tenant was previously attached + // as a Secondary location and then detached, so has no previously notified + // state. + tracing::info!("Shard not found while handling detach") } } ComputeHookTenant::Unsharded(_) => { @@ -761,7 +764,10 @@ impl ComputeHook { let mut state_locked = self.state.lock().unwrap(); match state_locked.entry(tenant_shard_id.tenant_id) { Entry::Vacant(_) => { - tracing::warn!("Compute hook tenant not found for detach"); + // This is a valid but niche case, where the tenant was previously attached + // as a Secondary location and then detached, so has no previously notified + // state. + tracing::info!("Compute hook tenant not found for detach"); } Entry::Occupied(mut e) => { let sharded = e.get().is_sharded(); diff --git a/storage_controller/src/http.rs b/storage_controller/src/http.rs index 5385e4ee0b..c8df4ffe28 100644 --- a/storage_controller/src/http.rs +++ b/storage_controller/src/http.rs @@ -690,7 +690,8 @@ async fn handle_node_list(req: Request) -> Result, ApiError }; let state = get_state(&req); - let nodes = state.service.node_list().await?; + let mut nodes = state.service.node_list().await?; + nodes.sort_by_key(|n| n.get_id()); let api_nodes = nodes.into_iter().map(|n| n.describe()).collect::>(); json_response(StatusCode::OK, api_nodes) @@ -1005,6 +1006,29 @@ async fn handle_tenant_shard_migrate( ) } +async fn handle_tenant_shard_migrate_secondary( + service: Arc, + req: Request, +) -> Result, ApiError> { + check_permissions(&req, Scope::Admin)?; + + let mut req = match maybe_forward(req).await { + ForwardOutcome::Forwarded(res) => { + return res; + } + ForwardOutcome::NotForwarded(req) => req, + }; + + let tenant_shard_id: TenantShardId = parse_request_param(&req, "tenant_shard_id")?; + let migrate_req = json_request::(&mut req).await?; + json_response( + StatusCode::OK, + service + .tenant_shard_migrate_secondary(tenant_shard_id, migrate_req) + .await?, + ) +} + async fn handle_tenant_shard_cancel_reconcile( service: Arc, req: Request, @@ -1855,6 +1879,16 @@ pub fn make_router( RequestName("control_v1_tenant_migrate"), ) }) + .put( + "/control/v1/tenant/:tenant_shard_id/migrate_secondary", + |r| { + tenant_service_handler( + r, + handle_tenant_shard_migrate_secondary, + RequestName("control_v1_tenant_migrate_secondary"), + ) + }, + ) .put( "/control/v1/tenant/:tenant_shard_id/cancel_reconcile", |r| { diff --git a/storage_controller/src/service.rs b/storage_controller/src/service.rs index 430d884548..8aa263f0c3 100644 --- a/storage_controller/src/service.rs +++ b/storage_controller/src/service.rs @@ -5055,6 +5055,69 @@ impl Service { Ok(TenantShardMigrateResponse {}) } + pub(crate) async fn tenant_shard_migrate_secondary( + &self, + tenant_shard_id: TenantShardId, + migrate_req: TenantShardMigrateRequest, + ) -> Result { + let waiter = { + let mut locked = self.inner.write().unwrap(); + let (nodes, tenants, scheduler) = locked.parts_mut(); + + let Some(node) = nodes.get(&migrate_req.node_id) else { + return Err(ApiError::BadRequest(anyhow::anyhow!( + "Node {} not found", + migrate_req.node_id + ))); + }; + + if !node.is_available() { + // Warn but proceed: the caller may intend to manually adjust the placement of + // a shard even if the node is down, e.g. if intervening during an incident. + tracing::warn!("Migrating to unavailable node {node}"); + } + + let Some(shard) = tenants.get_mut(&tenant_shard_id) else { + return Err(ApiError::NotFound( + anyhow::anyhow!("Tenant shard not found").into(), + )); + }; + + if shard.intent.get_secondary().len() == 1 + && shard.intent.get_secondary()[0] == migrate_req.node_id + { + tracing::info!( + "Migrating secondary to {node}: intent is unchanged {:?}", + shard.intent + ); + } else if shard.intent.get_attached() == &Some(migrate_req.node_id) { + tracing::info!("Migrating secondary to {node}: already attached where we were asked to create a secondary"); + } else { + let old_secondaries = shard.intent.get_secondary().clone(); + for secondary in old_secondaries { + shard.intent.remove_secondary(scheduler, secondary); + } + + shard.intent.push_secondary(scheduler, migrate_req.node_id); + shard.sequence = shard.sequence.next(); + tracing::info!( + "Migrating secondary to {node}: new intent {:?}", + shard.intent + ); + } + + self.maybe_reconcile_shard(shard, nodes) + }; + + if let Some(waiter) = waiter { + waiter.wait_timeout(RECONCILE_TIMEOUT).await?; + } else { + tracing::info!("Migration is a no-op"); + } + + Ok(TenantShardMigrateResponse {}) + } + /// 'cancel' in this context means cancel any ongoing reconcile pub(crate) async fn tenant_shard_cancel_reconcile( &self, diff --git a/test_runner/regress/test_storage_controller.py b/test_runner/regress/test_storage_controller.py index 616aee758d..3a55e75589 100644 --- a/test_runner/regress/test_storage_controller.py +++ b/test_runner/regress/test_storage_controller.py @@ -1052,7 +1052,7 @@ def test_storage_controller_debug_apis(neon_env_builder: NeonEnvBuilder): that just hits the endpoints to check that they don't bitrot. """ - neon_env_builder.num_pageservers = 2 + neon_env_builder.num_pageservers = 3 env = neon_env_builder.init_start() tenant_id = TenantId.generate() @@ -1077,7 +1077,7 @@ def test_storage_controller_debug_apis(neon_env_builder: NeonEnvBuilder): "GET", f"{env.storage_controller_api}/debug/v1/scheduler" ) # Two nodes, in a dict of node_id->node - assert len(response.json()["nodes"]) == 2 + assert len(response.json()["nodes"]) == 3 assert sum(v["shard_count"] for v in response.json()["nodes"].values()) == 3 assert all(v["may_schedule"] for v in response.json()["nodes"].values()) @@ -1088,13 +1088,25 @@ def test_storage_controller_debug_apis(neon_env_builder: NeonEnvBuilder): headers=env.storage_controller.headers(TokenScope.ADMIN), ) + # Secondary migration API: superficial check that it migrates + secondary_dest = env.pageservers[2].id + env.storage_controller.request( + "PUT", + f"{env.storage_controller_api}/control/v1/tenant/{tenant_id}-0002/migrate_secondary", + headers=env.storage_controller.headers(TokenScope.ADMIN), + json={"tenant_shard_id": f"{tenant_id}-0002", "node_id": secondary_dest}, + ) + assert env.storage_controller.tenant_describe(tenant_id)["shards"][0]["node_secondary"] == [ + secondary_dest + ] + # Node unclean drop API response = env.storage_controller.request( "POST", f"{env.storage_controller_api}/debug/v1/node/{env.pageservers[1].id}/drop", headers=env.storage_controller.headers(TokenScope.ADMIN), ) - assert len(env.storage_controller.node_list()) == 1 + assert len(env.storage_controller.node_list()) == 2 # Tenant unclean drop API response = env.storage_controller.request( @@ -1812,7 +1824,13 @@ def test_storcon_cli(neon_env_builder: NeonEnvBuilder): """ output_dir = neon_env_builder.test_output_dir shard_count = 4 - env = neon_env_builder.init_start(initial_tenant_shard_count=shard_count) + neon_env_builder.num_pageservers = 2 + env = neon_env_builder.init_configs() + env.start() + + tenant_id = TenantId.generate() + env.create_tenant(tenant_id, placement_policy='{"Attached":1}', shard_count=shard_count) + base_args = [env.neon_binpath / "storcon_cli", "--api", env.storage_controller_api] def storcon_cli(args): @@ -1841,7 +1859,7 @@ def test_storcon_cli(neon_env_builder: NeonEnvBuilder): # List nodes node_lines = storcon_cli(["nodes"]) # Table header, footer, and one line of data - assert len(node_lines) == 5 + assert len(node_lines) == 7 assert "localhost" in node_lines[3] # Pause scheduling onto a node @@ -1859,10 +1877,21 @@ def test_storcon_cli(neon_env_builder: NeonEnvBuilder): storcon_cli(["node-configure", "--node-id", "1", "--availability", "offline"]) assert "Offline" in storcon_cli(["nodes"])[3] + # Restore node, verify status changes in CLI output + env.pageservers[0].start() + + def is_online(): + assert "Offline" not in storcon_cli(["nodes"]) + + wait_until(is_online) + + # Let everything stabilize after node failure to avoid interfering with subsequent steps + env.storage_controller.reconcile_until_idle(timeout_secs=10) + # List tenants tenant_lines = storcon_cli(["tenants"]) assert len(tenant_lines) == 5 - assert str(env.initial_tenant) in tenant_lines[3] + assert str(tenant_id) in tenant_lines[3] # Setting scheduling policies intentionally result in warnings, they're for rare use. env.storage_controller.allowed_errors.extend( @@ -1870,23 +1899,58 @@ def test_storcon_cli(neon_env_builder: NeonEnvBuilder): ) # Describe a tenant - tenant_lines = storcon_cli(["tenant-describe", "--tenant-id", str(env.initial_tenant)]) + tenant_lines = storcon_cli(["tenant-describe", "--tenant-id", str(tenant_id)]) assert len(tenant_lines) >= 3 + shard_count * 2 - assert str(env.initial_tenant) in tenant_lines[0] + assert str(tenant_id) in tenant_lines[0] + + # Migrate an attached location + def other_ps_id(current_ps_id): + return ( + env.pageservers[0].id + if current_ps_id == env.pageservers[1].id + else env.pageservers[1].id + ) + + storcon_cli( + [ + "tenant-shard-migrate", + "--tenant-shard-id", + f"{tenant_id}-0004", + "--node", + str( + other_ps_id( + env.storage_controller.tenant_describe(tenant_id)["shards"][0]["node_attached"] + ) + ), + ] + ) + + # Migrate a secondary location + storcon_cli( + [ + "tenant-shard-migrate-secondary", + "--tenant-shard-id", + f"{tenant_id}-0004", + "--node", + str( + other_ps_id( + env.storage_controller.tenant_describe(tenant_id)["shards"][0][ + "node_secondary" + ][0] + ) + ), + ] + ) # Pause changes on a tenant - storcon_cli(["tenant-policy", "--tenant-id", str(env.initial_tenant), "--scheduling", "stop"]) + storcon_cli(["tenant-policy", "--tenant-id", str(tenant_id), "--scheduling", "stop"]) assert "Stop" in storcon_cli(["tenants"])[3] # Cancel ongoing reconcile on a tenant - storcon_cli( - ["tenant-shard-cancel-reconcile", "--tenant-shard-id", f"{env.initial_tenant}-0104"] - ) + storcon_cli(["tenant-shard-cancel-reconcile", "--tenant-shard-id", f"{tenant_id}-0104"]) # Change a tenant's placement - storcon_cli( - ["tenant-policy", "--tenant-id", str(env.initial_tenant), "--placement", "secondary"] - ) + storcon_cli(["tenant-policy", "--tenant-id", str(tenant_id), "--placement", "secondary"]) assert "Secondary" in storcon_cli(["tenants"])[3] # Modify a tenant's config @@ -1894,7 +1958,7 @@ def test_storcon_cli(neon_env_builder: NeonEnvBuilder): [ "patch-tenant-config", "--tenant-id", - str(env.initial_tenant), + str(tenant_id), "--config", json.dumps({"pitr_interval": "1m"}), ] From 96243af651f67747c9eeaf800f50777c75454143 Mon Sep 17 00:00:00 2001 From: Heikki Linnakangas Date: Mon, 13 Jan 2025 17:01:13 +0200 Subject: [PATCH 062/111] Stop building unnecessary extension tarballs (#10355) We build "custom extensions" from a different repository nowadays. --- compute/compute-node.Dockerfile | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/compute/compute-node.Dockerfile b/compute/compute-node.Dockerfile index 449e12af5d..f346f402d4 100644 --- a/compute/compute-node.Dockerfile +++ b/compute/compute-node.Dockerfile @@ -170,7 +170,6 @@ RUN case "${PG_VERSION}" in \ wget https://download.osgeo.org/postgis/source/postgis-${POSTGIS_VERSION}.tar.gz -O postgis.tar.gz && \ echo "${POSTGIS_CHECKSUM} postgis.tar.gz" | sha256sum --check && \ mkdir postgis-src && cd postgis-src && tar xzf ../postgis.tar.gz --strip-components=1 -C . && \ - find /usr/local/pgsql -type f | sed 's|^/usr/local/pgsql/||' > /before.txt &&\ ./autogen.sh && \ ./configure --with-sfcgal=/usr/local/bin/sfcgal-config && \ make -j $(getconf _NPROCESSORS_ONLN) && \ @@ -220,11 +219,7 @@ RUN case "${PG_VERSION}" in \ cmake -GNinja -DCMAKE_BUILD_TYPE=Release .. && \ ninja -j $(getconf _NPROCESSORS_ONLN) && \ ninja -j $(getconf _NPROCESSORS_ONLN) install && \ - echo 'trusted = true' >> /usr/local/pgsql/share/extension/pgrouting.control && \ - find /usr/local/pgsql -type f | sed 's|^/usr/local/pgsql/||' > /after.txt &&\ - cp /usr/local/pgsql/share/extension/pgrouting.control /extensions/postgis && \ - sort -o /before.txt /before.txt && sort -o /after.txt /after.txt && \ - comm -13 /before.txt /after.txt | tar --directory=/usr/local/pgsql --zstd -cf /extensions/postgis.tar.zst -T - + echo 'trusted = true' >> /usr/local/pgsql/share/extension/pgrouting.control ######################################################################################### # @@ -842,13 +837,8 @@ RUN case "${PG_VERSION}" in "v17") \ wget https://github.com/neondatabase/postgresql_anonymizer/archive/refs/tags/neon_1.1.1.tar.gz -O pg_anon.tar.gz && \ echo "321ea8d5c1648880aafde850a2c576e4a9e7b9933a34ce272efc839328999fa9 pg_anon.tar.gz" | sha256sum --check && \ mkdir pg_anon-src && cd pg_anon-src && tar xzf ../pg_anon.tar.gz --strip-components=1 -C . && \ - find /usr/local/pgsql -type f | sed 's|^/usr/local/pgsql/||' > /before.txt &&\ make -j $(getconf _NPROCESSORS_ONLN) install PG_CONFIG=/usr/local/pgsql/bin/pg_config && \ - echo 'trusted = true' >> /usr/local/pgsql/share/extension/anon.control && \ - find /usr/local/pgsql -type f | sed 's|^/usr/local/pgsql/||' > /after.txt &&\ - mkdir -p /extensions/anon && cp /usr/local/pgsql/share/extension/anon.control /extensions/anon && \ - sort -o /before.txt /before.txt && sort -o /after.txt /after.txt && \ - comm -13 /before.txt /after.txt | tar --directory=/usr/local/pgsql --zstd -cf /extensions/anon.tar.zst -T - + echo 'trusted = true' >> /usr/local/pgsql/share/extension/anon.control ######################################################################################### # From a338aee132497193c3587acd267babac66758d01 Mon Sep 17 00:00:00 2001 From: Conrad Ludgate Date: Mon, 13 Jan 2025 15:20:46 +0000 Subject: [PATCH 063/111] feat(local_proxy): use ed25519 signatures with pg_session_jwt (#10290) Generally ed25519 seems to be much preferred for cryptographic strength to P256 nowadays, and it is NIST approved finally. We should use it where we can as it's also faster than p256. This PR makes the re-signed JWTs between local_proxy and pg_session_jwt use ed25519. This does introduce a new dependency on ed25519, but I do recall some Neon Authorise customers asking for support for ed25519, so I am justifying this dependency addition in the context that we can then introduce support for customer ed25519 keys sources: * https://csrc.nist.gov/pubs/fips/186-5/final subsection 7 (EdDSA) * https://datatracker.ietf.org/doc/html/rfc8037#section-3.1 --- Cargo.lock | 55 +++++++++++++++++++++++++ compute/compute-node.Dockerfile | 4 +- proxy/Cargo.toml | 1 + proxy/src/serverless/backend.rs | 16 ++++--- proxy/src/serverless/local_conn_pool.rs | 28 +++++++------ 5 files changed, 85 insertions(+), 19 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f727741883..08453120c7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1605,6 +1605,32 @@ dependencies = [ "typenum", ] +[[package]] +name = "curve25519-dalek" +version = "4.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97fb8b7c4503de7d6ae7b42ab72a5a59857b4c937ec27a3d4539dba95b5ab2be" +dependencies = [ + "cfg-if", + "cpufeatures", + "curve25519-dalek-derive", + "digest", + "fiat-crypto", + "rustc_version", + "subtle", +] + +[[package]] +name = "curve25519-dalek-derive" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.90", +] + [[package]] name = "darling" version = "0.20.1" @@ -1875,6 +1901,28 @@ dependencies = [ "spki 0.7.3", ] +[[package]] +name = "ed25519" +version = "2.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "115531babc129696a58c64a4fef0a8bf9e9698629fb97e9e40767d235cfbcd53" +dependencies = [ + "signature 2.2.0", +] + +[[package]] +name = "ed25519-dalek" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4a3daa8e81a3963a60642bcc1f90a670680bd4a77535faa384e9d1c79d620871" +dependencies = [ + "curve25519-dalek", + "ed25519", + "rand_core 0.6.4", + "sha2", + "subtle", +] + [[package]] name = "either" version = "1.8.1" @@ -2113,6 +2161,12 @@ dependencies = [ "subtle", ] +[[package]] +name = "fiat-crypto" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28dea519a9695b9977216879a3ebfddf92f1c08c05d984f8996aecd6ecdc811d" + [[package]] name = "filetime" version = "0.2.22" @@ -4745,6 +4799,7 @@ dependencies = [ "consumption_metrics", "dashmap 5.5.0", "ecdsa 0.16.9", + "ed25519-dalek", "env_logger 0.10.2", "fallible-iterator", "flate2", diff --git a/compute/compute-node.Dockerfile b/compute/compute-node.Dockerfile index f346f402d4..2d38796d77 100644 --- a/compute/compute-node.Dockerfile +++ b/compute/compute-node.Dockerfile @@ -1055,8 +1055,8 @@ ARG PG_VERSION # NOTE: local_proxy depends on the version of pg_session_jwt # Do not update without approve from proxy team # Make sure the version is reflected in proxy/src/serverless/local_conn_pool.rs -RUN wget https://github.com/neondatabase/pg_session_jwt/archive/refs/tags/v0.1.2-v17.tar.gz -O pg_session_jwt.tar.gz && \ - echo "c8ecbed9cb8c6441bce5134a176002b043018adf9d05a08e457dda233090a86e pg_session_jwt.tar.gz" | sha256sum --check && \ +RUN wget https://github.com/neondatabase/pg_session_jwt/archive/refs/tags/v0.2.0.tar.gz -O pg_session_jwt.tar.gz && \ + echo "5ace028e591f2e000ca10afa5b1ca62203ebff014c2907c0ec3b29c36f28a1bb pg_session_jwt.tar.gz" | sha256sum --check && \ mkdir pg_session_jwt-src && cd pg_session_jwt-src && tar xzf ../pg_session_jwt.tar.gz --strip-components=1 -C . && \ sed -i 's/pgrx = "0.12.6"/pgrx = { version = "=0.12.6", features = [ "unsafe-postgres" ] }/g' Cargo.toml && \ cargo pgrx install --release diff --git a/proxy/Cargo.toml b/proxy/Cargo.toml index 2f63ee3acc..f362a45035 100644 --- a/proxy/Cargo.toml +++ b/proxy/Cargo.toml @@ -106,6 +106,7 @@ jose-jwk = { version = "0.1.2", features = ["p256", "p384", "rsa"] } signature = "2" ecdsa = "0.16" p256 = { version = "0.13", features = ["jwk"] } +ed25519-dalek = { version = "2", default-features = false, features = ["rand_core"] } rsa = "0.9" workspace_hack.workspace = true diff --git a/proxy/src/serverless/backend.rs b/proxy/src/serverless/backend.rs index b398c3ddd0..6d5fb13681 100644 --- a/proxy/src/serverless/backend.rs +++ b/proxy/src/serverless/backend.rs @@ -3,9 +3,9 @@ use std::sync::Arc; use std::time::Duration; use async_trait::async_trait; +use ed25519_dalek::SigningKey; use hyper_util::rt::{TokioExecutor, TokioIo, TokioTimer}; -use p256::ecdsa::SigningKey; -use p256::elliptic_curve::JwkEcKey; +use jose_jwk::jose_b64; use rand::rngs::OsRng; use tokio::net::{lookup_host, TcpStream}; use tracing::field::display; @@ -354,9 +354,15 @@ impl PoolingBackend { } } -fn create_random_jwk() -> (SigningKey, JwkEcKey) { - let key = SigningKey::random(&mut OsRng); - let jwk = p256::PublicKey::from(key.verifying_key()).to_jwk(); +fn create_random_jwk() -> (SigningKey, jose_jwk::Key) { + let key = SigningKey::generate(&mut OsRng); + + let jwk = jose_jwk::Key::Okp(jose_jwk::Okp { + crv: jose_jwk::OkpCurves::Ed25519, + x: jose_b64::serde::Bytes::from(key.verifying_key().to_bytes().to_vec()), + d: None, + }); + (key, jwk) } diff --git a/proxy/src/serverless/local_conn_pool.rs b/proxy/src/serverless/local_conn_pool.rs index c51a2bc9ba..fe33f0ff65 100644 --- a/proxy/src/serverless/local_conn_pool.rs +++ b/proxy/src/serverless/local_conn_pool.rs @@ -16,17 +16,16 @@ use std::sync::Arc; use std::task::{ready, Poll}; use std::time::Duration; +use ed25519_dalek::{Signature, Signer, SigningKey}; use futures::future::poll_fn; use futures::Future; use indexmap::IndexMap; use jose_jwk::jose_b64::base64ct::{Base64UrlUnpadded, Encoding}; -use p256::ecdsa::{Signature, SigningKey}; use parking_lot::RwLock; use postgres_client::tls::NoTlsStream; use postgres_client::types::ToSql; use postgres_client::AsyncMessage; use serde_json::value::RawValue; -use signature::Signer; use tokio::net::TcpStream; use tokio::time::Instant; use tokio_util::sync::CancellationToken; @@ -42,7 +41,7 @@ use crate::control_plane::messages::{ColdStartInfo, MetricsAuxInfo}; use crate::metrics::Metrics; pub(crate) const EXT_NAME: &str = "pg_session_jwt"; -pub(crate) const EXT_VERSION: &str = "0.1.2"; +pub(crate) const EXT_VERSION: &str = "0.2.0"; pub(crate) const EXT_SCHEMA: &str = "auth"; #[derive(Clone)] @@ -339,8 +338,8 @@ fn sign_jwt(sk: &SigningKey, payload: &[u8]) -> String { let cap = jwt.capacity(); // we only need an empty header with the alg specified. - // base64url(r#"{"alg":"ES256"}"#) == "eyJhbGciOiJFUzI1NiJ9" - jwt.push_str("eyJhbGciOiJFUzI1NiJ9."); + // base64url(r#"{"alg":"EdDSA"}"#) == "eyJhbGciOiJFZERTQSJ9" + jwt.push_str("eyJhbGciOiJFZERTQSJ9."); // encode the jwt payload in-place base64::encode_config_buf(payload, base64::URL_SAFE_NO_PAD, &mut jwt); @@ -366,14 +365,14 @@ fn sign_jwt(sk: &SigningKey, payload: &[u8]) -> String { #[cfg(test)] #[expect(clippy::unwrap_used)] mod tests { - use p256::ecdsa::SigningKey; + use ed25519_dalek::SigningKey; use typed_json::json; use super::resign_jwt; #[test] fn jwt_token_snapshot() { - let key = SigningKey::from_bytes(&[1; 32].into()).unwrap(); + let key = SigningKey::from_bytes(&[1; 32]); let data = json!({"foo":"bar","jti":"foo\nbar","nested":{"jti":"tricky nesting"}}).to_string(); @@ -381,12 +380,17 @@ mod tests { // To validate the JWT, copy the JWT string and paste it into https://jwt.io/. // In the public-key box, paste the following jwk public key - // `{"kty":"EC","crv":"P-256","x":"b_A7lJJBzh2t1DUZ5pYOCoW0GmmgXDKBA6orzhWUyhY","y":"PE91OlW_AdxT9sCwx-7ni0DG_30lqW4igrmJzvccFEo"}` + // `{"kty":"OKP","crv":"Ed25519","x":"iojj3XQJ8ZX9UtstPLpdcspnCb8dlBIb83SIAbQPb1w"}` + // Note - jwt.io doesn't support EdDSA :( + // https://github.com/jsonwebtoken/jsonwebtoken.github.io/issues/509 - // let pub_key = p256::ecdsa::VerifyingKey::from(&key); - // let pub_key = p256::PublicKey::from(pub_key); - // println!("{}", pub_key.to_jwk_string()); + // let jwk = jose_jwk::Key::Okp(jose_jwk::Okp { + // crv: jose_jwk::OkpCurves::Ed25519, + // x: jose_jwk::jose_b64::serde::Bytes::from(key.verifying_key().to_bytes().to_vec()), + // d: None, + // }); + // println!("{}", serde_json::to_string(&jwk).unwrap()); - assert_eq!(jwt, "eyJhbGciOiJFUzI1NiJ9.eyJmb28iOiJiYXIiLCJqdGkiOjIsIm5lc3RlZCI6eyJqdGkiOiJ0cmlja3kgbmVzdGluZyJ9fQ.pYf0LxoJ8sDgpmsYOgrbNecOSipnPBEGwnZzB-JhW2cONrKlqRsgXwK8_cOsyolGy-hTTe8GXbWTl_UdpF5RyA"); + assert_eq!(jwt, "eyJhbGciOiJFZERTQSJ9.eyJmb28iOiJiYXIiLCJqdGkiOjIsIm5lc3RlZCI6eyJqdGkiOiJ0cmlja3kgbmVzdGluZyJ9fQ.Cvyc2By33KI0f0obystwdy8PN111L3Sc9_Mr2CU3XshtSqSdxuRxNEZGbb_RvyJf2IzheC_s7aBZ-jLeQ9N0Bg"); } } From e9ed53b14f2a7feef862606626ae7d790a1346ce Mon Sep 17 00:00:00 2001 From: "Alex Chi Z." <4198311+skyzh@users.noreply.github.com> Date: Mon, 13 Jan 2025 10:43:01 -0500 Subject: [PATCH 064/111] feat(pageserver): support inherited sparse keyspace (#10313) ## Problem In preparation to https://github.com/neondatabase/neon/issues/9516. We need to store rel size and directory data in the sparse keyspace, but it does not support inheritance yet. ## Summary of changes Add a new type of keyspace "sparse but inherited" into the system. On the read path: we don't remove the key range when we descend into the ancestor. The search will stop when (1) the full key range is covered by image layers (which has already been implemented before), or (2) we reach the end of the ancestor chain. --------- Signed-off-by: Alex Chi Z --- libs/pageserver_api/src/key.rs | 39 ++++++- pageserver/src/tenant.rs | 135 ++++++++++++++++++++++++- pageserver/src/tenant/storage_layer.rs | 4 +- pageserver/src/tenant/timeline.rs | 25 ++++- 4 files changed, 193 insertions(+), 10 deletions(-) diff --git a/libs/pageserver_api/src/key.rs b/libs/pageserver_api/src/key.rs index f0cd713c38..328dea5dec 100644 --- a/libs/pageserver_api/src/key.rs +++ b/libs/pageserver_api/src/key.rs @@ -706,7 +706,7 @@ pub fn repl_origin_key_range() -> Range { /// Non inherited range for vectored get. pub const NON_INHERITED_RANGE: Range = AUX_FILES_KEY..AUX_FILES_KEY.next(); /// Sparse keyspace range for vectored get. Missing key error will be ignored for this range. -pub const NON_INHERITED_SPARSE_RANGE: Range = Key::metadata_key_range(); +pub const SPARSE_RANGE: Range = Key::metadata_key_range(); impl Key { // AUX_FILES currently stores only data for logical replication (slots etc), and @@ -714,7 +714,42 @@ impl Key { // switch (and generally it likely should be optional), so ignore these. #[inline(always)] pub fn is_inherited_key(self) -> bool { - !NON_INHERITED_RANGE.contains(&self) && !NON_INHERITED_SPARSE_RANGE.contains(&self) + if self.is_sparse() { + self.is_inherited_sparse_key() + } else { + !NON_INHERITED_RANGE.contains(&self) + } + } + + #[inline(always)] + pub fn is_sparse(self) -> bool { + self.field1 >= METADATA_KEY_BEGIN_PREFIX && self.field1 < METADATA_KEY_END_PREFIX + } + + /// Check if the key belongs to the inherited keyspace. + fn is_inherited_sparse_key(self) -> bool { + debug_assert!(self.is_sparse()); + self.field1 == RELATION_SIZE_PREFIX + } + + pub fn sparse_non_inherited_keyspace() -> Range { + // The two keys are adjacent; if we will have non-adjancent keys in the future, we should return a keyspace + debug_assert_eq!(AUX_KEY_PREFIX + 1, REPL_ORIGIN_KEY_PREFIX); + Key { + field1: AUX_KEY_PREFIX, + field2: 0, + field3: 0, + field4: 0, + field5: 0, + field6: 0, + }..Key { + field1: REPL_ORIGIN_KEY_PREFIX + 1, + field2: 0, + field3: 0, + field4: 0, + field5: 0, + field6: 0, + } } #[inline(always)] diff --git a/pageserver/src/tenant.rs b/pageserver/src/tenant.rs index 2928c435cb..070593b104 100644 --- a/pageserver/src/tenant.rs +++ b/pageserver/src/tenant.rs @@ -5682,7 +5682,7 @@ mod tests { use bytes::{Bytes, BytesMut}; use hex_literal::hex; use itertools::Itertools; - use pageserver_api::key::{Key, AUX_KEY_PREFIX, NON_INHERITED_RANGE}; + use pageserver_api::key::{Key, AUX_KEY_PREFIX, NON_INHERITED_RANGE, RELATION_SIZE_PREFIX}; use pageserver_api::keyspace::KeySpace; use pageserver_api::models::{CompactionAlgorithm, CompactionAlgorithmSettings}; use pageserver_api::value::Value; @@ -7741,7 +7741,18 @@ mod tests { let base_key = Key::from_hex("620000000033333333444444445500000000").unwrap(); let base_key_child = Key::from_hex("620000000033333333444444445500000001").unwrap(); let base_key_nonexist = Key::from_hex("620000000033333333444444445500000002").unwrap(); + let base_key_overwrite = Key::from_hex("620000000033333333444444445500000003").unwrap(); + + let base_inherited_key = Key::from_hex("610000000033333333444444445500000000").unwrap(); + let base_inherited_key_child = + Key::from_hex("610000000033333333444444445500000001").unwrap(); + let base_inherited_key_nonexist = + Key::from_hex("610000000033333333444444445500000002").unwrap(); + let base_inherited_key_overwrite = + Key::from_hex("610000000033333333444444445500000003").unwrap(); + assert_eq!(base_key.field1, AUX_KEY_PREFIX); // in case someone accidentally changed the prefix... + assert_eq!(base_inherited_key.field1, RELATION_SIZE_PREFIX); let tline = tenant .create_test_timeline_with_layers( @@ -7750,7 +7761,18 @@ mod tests { DEFAULT_PG_VERSION, &ctx, Vec::new(), // delta layers - vec![(Lsn(0x20), vec![(base_key, test_img("metadata key 1"))])], // image layers + vec![( + Lsn(0x20), + vec![ + (base_inherited_key, test_img("metadata inherited key 1")), + ( + base_inherited_key_overwrite, + test_img("metadata key overwrite 1a"), + ), + (base_key, test_img("metadata key 1")), + (base_key_overwrite, test_img("metadata key overwrite 1b")), + ], + )], // image layers Lsn(0x20), // it's fine to not advance LSN to 0x30 while using 0x30 to get below because `get_vectored_impl` does not wait for LSN ) .await?; @@ -7764,7 +7786,18 @@ mod tests { Vec::new(), // delta layers vec![( Lsn(0x30), - vec![(base_key_child, test_img("metadata key 2"))], + vec![ + ( + base_inherited_key_child, + test_img("metadata inherited key 2"), + ), + ( + base_inherited_key_overwrite, + test_img("metadata key overwrite 2a"), + ), + (base_key_child, test_img("metadata key 2")), + (base_key_overwrite, test_img("metadata key overwrite 2b")), + ], )], // image layers Lsn(0x30), ) @@ -7786,6 +7819,26 @@ mod tests { get_vectored_impl_wrapper(&tline, base_key_nonexist, lsn, &ctx).await?, None ); + assert_eq!( + get_vectored_impl_wrapper(&tline, base_key_overwrite, lsn, &ctx).await?, + Some(test_img("metadata key overwrite 1b")) + ); + assert_eq!( + get_vectored_impl_wrapper(&tline, base_inherited_key, lsn, &ctx).await?, + Some(test_img("metadata inherited key 1")) + ); + assert_eq!( + get_vectored_impl_wrapper(&tline, base_inherited_key_child, lsn, &ctx).await?, + None + ); + assert_eq!( + get_vectored_impl_wrapper(&tline, base_inherited_key_nonexist, lsn, &ctx).await?, + None + ); + assert_eq!( + get_vectored_impl_wrapper(&tline, base_inherited_key_overwrite, lsn, &ctx).await?, + Some(test_img("metadata key overwrite 1a")) + ); // test vectored get on child timeline assert_eq!( @@ -7800,6 +7853,82 @@ mod tests { get_vectored_impl_wrapper(&child, base_key_nonexist, lsn, &ctx).await?, None ); + assert_eq!( + get_vectored_impl_wrapper(&child, base_inherited_key, lsn, &ctx).await?, + Some(test_img("metadata inherited key 1")) + ); + assert_eq!( + get_vectored_impl_wrapper(&child, base_inherited_key_child, lsn, &ctx).await?, + Some(test_img("metadata inherited key 2")) + ); + assert_eq!( + get_vectored_impl_wrapper(&child, base_inherited_key_nonexist, lsn, &ctx).await?, + None + ); + assert_eq!( + get_vectored_impl_wrapper(&child, base_key_overwrite, lsn, &ctx).await?, + Some(test_img("metadata key overwrite 2b")) + ); + assert_eq!( + get_vectored_impl_wrapper(&child, base_inherited_key_overwrite, lsn, &ctx).await?, + Some(test_img("metadata key overwrite 2a")) + ); + + // test vectored scan on parent timeline + let mut reconstruct_state = ValuesReconstructState::new(); + let res = tline + .get_vectored_impl( + KeySpace::single(Key::metadata_key_range()), + lsn, + &mut reconstruct_state, + &ctx, + ) + .await?; + + assert_eq!( + res.into_iter() + .map(|(k, v)| (k, v.unwrap())) + .collect::>(), + vec![ + (base_inherited_key, test_img("metadata inherited key 1")), + ( + base_inherited_key_overwrite, + test_img("metadata key overwrite 1a") + ), + (base_key, test_img("metadata key 1")), + (base_key_overwrite, test_img("metadata key overwrite 1b")), + ] + ); + + // test vectored scan on child timeline + let mut reconstruct_state = ValuesReconstructState::new(); + let res = child + .get_vectored_impl( + KeySpace::single(Key::metadata_key_range()), + lsn, + &mut reconstruct_state, + &ctx, + ) + .await?; + + assert_eq!( + res.into_iter() + .map(|(k, v)| (k, v.unwrap())) + .collect::>(), + vec![ + (base_inherited_key, test_img("metadata inherited key 1")), + ( + base_inherited_key_child, + test_img("metadata inherited key 2") + ), + ( + base_inherited_key_overwrite, + test_img("metadata key overwrite 2a") + ), + (base_key_child, test_img("metadata key 2")), + (base_key_overwrite, test_img("metadata key overwrite 2b")), + ] + ); Ok(()) } diff --git a/pageserver/src/tenant/storage_layer.rs b/pageserver/src/tenant/storage_layer.rs index b8206fca5a..3913637ca0 100644 --- a/pageserver/src/tenant/storage_layer.rs +++ b/pageserver/src/tenant/storage_layer.rs @@ -12,7 +12,7 @@ pub mod merge_iterator; use crate::context::{AccessStatsBehavior, RequestContext}; use bytes::Bytes; -use pageserver_api::key::{Key, NON_INHERITED_SPARSE_RANGE}; +use pageserver_api::key::Key; use pageserver_api::keyspace::{KeySpace, KeySpaceRandomAccum}; use pageserver_api::record::NeonWalRecord; use pageserver_api::value::Value; @@ -209,7 +209,7 @@ impl ValuesReconstructState { .keys .entry(*key) .or_insert(Ok(VectoredValueReconstructState::default())); - let is_sparse_key = NON_INHERITED_SPARSE_RANGE.contains(key); + let is_sparse_key = key.is_sparse(); if let Ok(state) = state { let key_done = match state.situation { ValueReconstructSituation::Complete => { diff --git a/pageserver/src/tenant/timeline.rs b/pageserver/src/tenant/timeline.rs index c1b71262e0..f7227efeba 100644 --- a/pageserver/src/tenant/timeline.rs +++ b/pageserver/src/tenant/timeline.rs @@ -27,7 +27,7 @@ use pageserver_api::{ config::tenant_conf_defaults::DEFAULT_COMPACTION_THRESHOLD, key::{ KEY_SIZE, METADATA_KEY_BEGIN_PREFIX, METADATA_KEY_END_PREFIX, NON_INHERITED_RANGE, - NON_INHERITED_SPARSE_RANGE, + SPARSE_RANGE, }, keyspace::{KeySpaceAccum, KeySpaceRandomAccum, SparseKeyPartitioning}, models::{ @@ -3221,7 +3221,7 @@ impl Timeline { // We don't return a blanket [`GetVectoredError::MissingKey`] to avoid // stalling compaction. keyspace.remove_overlapping_with(&KeySpace { - ranges: vec![NON_INHERITED_RANGE, NON_INHERITED_SPARSE_RANGE], + ranges: vec![NON_INHERITED_RANGE, Key::sparse_non_inherited_keyspace()], }); // Keyspace is fully retrieved @@ -3242,7 +3242,11 @@ impl Timeline { // keys from `keyspace`, we expect there to be no overlap between it and the image covered key // space. If that's not the case, we had at least one key encounter a gap in the image layer // and stop the search as a result of that. - let removed = keyspace.remove_overlapping_with(&image_covered_keyspace); + let mut removed = keyspace.remove_overlapping_with(&image_covered_keyspace); + // Do not fire missing key error for sparse keys. + removed.remove_overlapping_with(&KeySpace { + ranges: vec![SPARSE_RANGE], + }); if !removed.is_empty() { break Some(removed); } @@ -3257,6 +3261,21 @@ impl Timeline { timeline = &*timeline_owned; }; + // Remove sparse keys from the keyspace so that it doesn't fire errors. + let missing_keyspace = if let Some(missing_keyspace) = missing_keyspace { + let mut missing_keyspace = missing_keyspace; + missing_keyspace.remove_overlapping_with(&KeySpace { + ranges: vec![SPARSE_RANGE], + }); + if missing_keyspace.is_empty() { + None + } else { + Some(missing_keyspace) + } + } else { + None + }; + if let Some(missing_keyspace) = missing_keyspace { return Err(GetVectoredError::MissingKey(MissingKeyError { key: missing_keyspace.start().unwrap(), /* better if we can store the full keyspace */ From fd1368d31e2d159a518ac78dd6d5146a45a6de72 Mon Sep 17 00:00:00 2001 From: John Spray Date: Mon, 13 Jan 2025 19:33:00 +0000 Subject: [PATCH 065/111] storcon: rework scheduler optimisation, prioritize AZ (#9916) ## Problem We want to do a more robust job of scheduling tenants into their home AZ: https://github.com/neondatabase/neon/issues/8264. Closes: https://github.com/neondatabase/neon/issues/8969 ## Summary of changes ### Scope This PR combines prioritizing AZ with a larger rework of how we do optimisation. The rationale is that just bumping AZ in the order of Score attributes is a very tiny change: the interesting part is lining up all the optimisation logic to respect this properly, which means rewriting it to use the same scores as the scheduler, rather than the fragile hand-crafted logic that we had before. Separating these changes out is possible, but would involve doing two rounds of test updates instead of one. ### Scheduling optimisation `TenantShard`'s `optimize_attachment` and `optimize_secondary` methods now both use the scheduler to pick a new "favourite" location. Then there is some refined logic for whether + how to migrate to it: - To decide if a new location is sufficiently "better", we generate scores using some projected ScheduleContexts that exclude the shard under consideration, so that we avoid migrating from a node with AffinityScore(2) to a node with AffinityScore(1), only to migrate back later. - Score types get a `for_optimization` method so that when we compare scores, we will only do an optimisation if the scores differ by their highest-ranking attributes, not just because one pageserver is lower in utilization. Eventually we _will_ want a mode that does this, but doing it here would make scheduling logic unstable and harder to test, and to do this correctly one needs to know the size of the tenant that one is migrating. - When we find a new attached location that we would like to move to, we will create a new secondary location there, even if we already had one on some other node. This handles the case where we have a home AZ A, and want to migrate the attachment between pageservers in that AZ while retaining a secondary location in some other AZ as well. - A unit test is added for https://github.com/neondatabase/neon/issues/8969, which is implicitly fixed by reworking optimisation to use the same scheduling scores as scheduling. --- libs/pageserver_api/src/controller_api.rs | 10 + storage_controller/Cargo.toml | 2 +- storage_controller/src/drain_utils.rs | 2 +- storage_controller/src/reconciler.rs | 16 +- storage_controller/src/scheduler.rs | 595 ++++++++--- storage_controller/src/service.rs | 244 ++++- .../src/service/context_iterator.rs | 9 +- storage_controller/src/tenant_shard.rs | 978 +++++++++++++----- .../performance/test_sharding_autosplit.py | 21 +- .../test_storage_controller_scale.py | 115 +- test_runner/regress/test_sharding.py | 32 +- .../regress/test_storage_controller.py | 40 +- 12 files changed, 1596 insertions(+), 468 deletions(-) diff --git a/libs/pageserver_api/src/controller_api.rs b/libs/pageserver_api/src/controller_api.rs index ba0eb0e4ae..f3aefc6df9 100644 --- a/libs/pageserver_api/src/controller_api.rs +++ b/libs/pageserver_api/src/controller_api.rs @@ -367,6 +367,16 @@ pub enum PlacementPolicy { Detached, } +impl PlacementPolicy { + pub fn want_secondaries(&self) -> usize { + match self { + PlacementPolicy::Attached(secondary_count) => *secondary_count, + PlacementPolicy::Secondary => 1, + PlacementPolicy::Detached => 0, + } + } +} + #[derive(Serialize, Deserialize, Debug)] pub struct TenantShardMigrateResponse {} diff --git a/storage_controller/Cargo.toml b/storage_controller/Cargo.toml index 5f3319512d..caaa22d0a5 100644 --- a/storage_controller/Cargo.toml +++ b/storage_controller/Cargo.toml @@ -55,4 +55,4 @@ r2d2 = { version = "0.8.10" } utils = { path = "../libs/utils/" } metrics = { path = "../libs/metrics/" } control_plane = { path = "../control_plane" } -workspace_hack = { version = "0.1", path = "../workspace_hack" } +workspace_hack = { version = "0.1", path = "../workspace_hack" } \ No newline at end of file diff --git a/storage_controller/src/drain_utils.rs b/storage_controller/src/drain_utils.rs index 47f4276ff2..8b7be88078 100644 --- a/storage_controller/src/drain_utils.rs +++ b/storage_controller/src/drain_utils.rs @@ -112,7 +112,7 @@ impl TenantShardDrain { } } - match scheduler.node_preferred(tenant_shard.intent.get_secondary()) { + match tenant_shard.preferred_secondary(scheduler) { Some(node) => Some(node), None => { tracing::warn!( diff --git a/storage_controller/src/reconciler.rs b/storage_controller/src/reconciler.rs index 6f584e7267..adced3b77d 100644 --- a/storage_controller/src/reconciler.rs +++ b/storage_controller/src/reconciler.rs @@ -826,7 +826,21 @@ impl Reconciler { if self.cancel.is_cancelled() { return Err(ReconcileError::Cancel); } - self.location_config(&node, conf, None, false).await?; + // We only try to configure secondary locations if the node is available. This does + // not stop us succeeding with the reconcile, because our core goal is to make the + // shard _available_ (the attached location), and configuring secondary locations + // can be done lazily when the node becomes available (via background reconciliation). + if node.is_available() { + self.location_config(&node, conf, None, false).await?; + } else { + // If the node is unavailable, we skip and consider the reconciliation successful: this + // is a common case where a pageserver is marked unavailable: we demote a location on + // that unavailable pageserver to secondary. + tracing::info!("Skipping configuring secondary location {node}, it is unavailable"); + self.observed + .locations + .insert(node.get_id(), ObservedStateLocation { conf: None }); + } } // The condition below identifies a detach. We must have no attached intent and diff --git a/storage_controller/src/scheduler.rs b/storage_controller/src/scheduler.rs index 51a4cf35be..04a594dcac 100644 --- a/storage_controller/src/scheduler.rs +++ b/storage_controller/src/scheduler.rs @@ -32,6 +32,9 @@ pub(crate) struct SchedulerNode { shard_count: usize, /// How many shards are currently attached on this node, via their [`crate::tenant_shard::IntentState`]. attached_shard_count: usize, + /// How many shards have a location on this node (via [`crate::tenant_shard::IntentState`]) _and_ this node + /// is in their preferred AZ (i.e. this is their 'home' location) + home_shard_count: usize, /// Availability zone id in which the node resides az: AvailabilityZone, @@ -47,6 +50,12 @@ pub(crate) trait NodeSchedulingScore: Debug + Ord + Copy + Sized { preferred_az: &Option, context: &ScheduleContext, ) -> Option; + + /// Return a score that drops any components based on node utilization: this is useful + /// for finding scores for scheduling optimisation, when we want to avoid rescheduling + /// shards due to e.g. disk usage, to avoid flapping. + fn for_optimization(&self) -> Self; + fn is_overloaded(&self) -> bool; fn node_id(&self) -> NodeId; } @@ -136,17 +145,13 @@ impl PartialOrd for SecondaryAzMatch { /// Ordering is given by member declaration order (top to bottom). #[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone, Copy)] pub(crate) struct NodeAttachmentSchedulingScore { - /// The number of shards belonging to the tenant currently being - /// scheduled that are attached to this node. - affinity_score: AffinityScore, /// Flag indicating whether this node matches the preferred AZ /// of the shard. For equal affinity scores, nodes in the matching AZ /// are considered first. az_match: AttachmentAzMatch, - /// Size of [`ScheduleContext::attached_nodes`] for the current node. - /// This normally tracks the number of attached shards belonging to the - /// tenant being scheduled that are already on this node. - attached_shards_in_context: usize, + /// The number of shards belonging to the tenant currently being + /// scheduled that are attached to this node. + affinity_score: AffinityScore, /// Utilisation score that combines shard count and disk utilisation utilization_score: u64, /// Total number of shards attached to this node. When nodes have identical utilisation, this @@ -177,13 +182,25 @@ impl NodeSchedulingScore for NodeAttachmentSchedulingScore { .copied() .unwrap_or(AffinityScore::FREE), az_match: AttachmentAzMatch(AzMatch::new(&node.az, preferred_az.as_ref())), - attached_shards_in_context: context.attached_nodes.get(node_id).copied().unwrap_or(0), utilization_score: utilization.cached_score(), total_attached_shard_count: node.attached_shard_count, node_id: *node_id, }) } + /// For use in scheduling optimisation, where we only want to consider the aspects + /// of the score that can only be resolved by moving things (such as inter-shard affinity + /// and AZ affinity), and ignore aspects that reflect the total utilization of a node (which + /// can fluctuate for other reasons) + fn for_optimization(&self) -> Self { + Self { + utilization_score: 0, + total_attached_shard_count: 0, + node_id: NodeId(0), + ..*self + } + } + fn is_overloaded(&self) -> bool { PageserverUtilization::is_overloaded(self.utilization_score) } @@ -208,9 +225,9 @@ pub(crate) struct NodeSecondarySchedulingScore { affinity_score: AffinityScore, /// Utilisation score that combines shard count and disk utilisation utilization_score: u64, - /// Total number of shards attached to this node. When nodes have identical utilisation, this - /// acts as an anti-affinity between attached shards. - total_attached_shard_count: usize, + /// Anti-affinity with other non-home locations: this gives the behavior that secondaries + /// will spread out across the nodes in an AZ. + total_non_home_shard_count: usize, /// Convenience to make selection deterministic in tests and empty systems node_id: NodeId, } @@ -237,11 +254,20 @@ impl NodeSchedulingScore for NodeSecondarySchedulingScore { .copied() .unwrap_or(AffinityScore::FREE), utilization_score: utilization.cached_score(), - total_attached_shard_count: node.attached_shard_count, + total_non_home_shard_count: (node.shard_count - node.home_shard_count), node_id: *node_id, }) } + fn for_optimization(&self) -> Self { + Self { + utilization_score: 0, + total_non_home_shard_count: 0, + node_id: NodeId(0), + ..*self + } + } + fn is_overloaded(&self) -> bool { PageserverUtilization::is_overloaded(self.utilization_score) } @@ -293,6 +319,10 @@ impl AffinityScore { pub(crate) fn inc(&mut self) { self.0 += 1; } + + pub(crate) fn dec(&mut self) { + self.0 -= 1; + } } impl std::ops::Add for AffinityScore { @@ -324,9 +354,6 @@ pub(crate) struct ScheduleContext { /// Sparse map of nodes: omitting a node implicitly makes its affinity [`AffinityScore::FREE`] pub(crate) nodes: HashMap, - /// Specifically how many _attached_ locations are on each node - pub(crate) attached_nodes: HashMap, - pub(crate) mode: ScheduleMode, } @@ -334,7 +361,6 @@ impl ScheduleContext { pub(crate) fn new(mode: ScheduleMode) -> Self { Self { nodes: HashMap::new(), - attached_nodes: HashMap::new(), mode, } } @@ -348,25 +374,31 @@ impl ScheduleContext { } } - pub(crate) fn push_attached(&mut self, node_id: NodeId) { - let entry = self.attached_nodes.entry(node_id).or_default(); - *entry += 1; - } + /// Remove `shard`'s contributions to this context. This is useful when considering scheduling + /// this shard afresh, where we don't want it to e.g. experience anti-affinity to its current location. + pub(crate) fn project_detach(&self, shard: &TenantShard) -> Self { + let mut new_context = self.clone(); - pub(crate) fn get_node_affinity(&self, node_id: NodeId) -> AffinityScore { - self.nodes - .get(&node_id) - .copied() - .unwrap_or(AffinityScore::FREE) - } + if let Some(attached) = shard.intent.get_attached() { + if let Some(score) = new_context.nodes.get_mut(attached) { + score.dec(); + } + } - pub(crate) fn get_node_attachments(&self, node_id: NodeId) -> usize { - self.attached_nodes.get(&node_id).copied().unwrap_or(0) + for secondary in shard.intent.get_secondary() { + if let Some(score) = new_context.nodes.get_mut(secondary) { + score.dec(); + } + } + + new_context } + /// For test, track the sum of AffinityScore values, which is effectively how many + /// attached or secondary locations have been registered with this context. #[cfg(test)] - pub(crate) fn attach_count(&self) -> usize { - self.attached_nodes.values().sum() + pub(crate) fn location_count(&self) -> usize { + self.nodes.values().map(|i| i.0).sum() } } @@ -388,6 +420,7 @@ impl Scheduler { SchedulerNode { shard_count: 0, attached_shard_count: 0, + home_shard_count: 0, may_schedule: node.may_schedule(), az: node.get_availability_zone_id().clone(), }, @@ -415,6 +448,7 @@ impl Scheduler { SchedulerNode { shard_count: 0, attached_shard_count: 0, + home_shard_count: 0, may_schedule: node.may_schedule(), az: node.get_availability_zone_id().clone(), }, @@ -427,6 +461,9 @@ impl Scheduler { Some(node) => { node.shard_count += 1; node.attached_shard_count += 1; + if Some(&node.az) == shard.preferred_az() { + node.home_shard_count += 1; + } } None => anyhow::bail!( "Tenant {} references nonexistent node {}", @@ -438,7 +475,12 @@ impl Scheduler { for node_id in shard.intent.get_secondary() { match expect_nodes.get_mut(node_id) { - Some(node) => node.shard_count += 1, + Some(node) => { + node.shard_count += 1; + if Some(&node.az) == shard.preferred_az() { + node.home_shard_count += 1; + } + } None => anyhow::bail!( "Tenant {} references nonexistent node {}", shard.tenant_shard_id, @@ -482,13 +524,20 @@ impl Scheduler { /// /// It is an error to call this for a node that is not known to the scheduler (i.e. passed into /// [`Self::new`] or [`Self::node_upsert`]) - pub(crate) fn update_node_ref_counts(&mut self, node_id: NodeId, update: RefCountUpdate) { + pub(crate) fn update_node_ref_counts( + &mut self, + node_id: NodeId, + preferred_az: Option<&AvailabilityZone>, + update: RefCountUpdate, + ) { let Some(node) = self.nodes.get_mut(&node_id) else { debug_assert!(false); tracing::error!("Scheduler missing node {node_id}"); return; }; + let is_home_az = Some(&node.az) == preferred_az; + match update { RefCountUpdate::PromoteSecondary => { node.attached_shard_count += 1; @@ -496,19 +545,31 @@ impl Scheduler { RefCountUpdate::Attach => { node.shard_count += 1; node.attached_shard_count += 1; + if is_home_az { + node.home_shard_count += 1; + } } RefCountUpdate::Detach => { node.shard_count -= 1; node.attached_shard_count -= 1; + if is_home_az { + node.home_shard_count -= 1; + } } RefCountUpdate::DemoteAttached => { node.attached_shard_count -= 1; } RefCountUpdate::AddSecondary => { node.shard_count += 1; + if is_home_az { + node.home_shard_count += 1; + } } RefCountUpdate::RemoveSecondary => { node.shard_count -= 1; + if is_home_az { + node.home_shard_count -= 1; + } } } @@ -594,6 +655,7 @@ impl Scheduler { entry.insert(SchedulerNode { shard_count: 0, attached_shard_count: 0, + home_shard_count: 0, may_schedule: node.may_schedule(), az: node.get_availability_zone_id().clone(), }); @@ -607,33 +669,20 @@ impl Scheduler { } } - /// Where we have several nodes to choose from, for example when picking a secondary location - /// to promote to an attached location, this method may be used to pick the best choice based - /// on the scheduler's knowledge of utilization and availability. - /// - /// If the input is empty, or all the nodes are not elegible for scheduling, return None: the - /// caller can pick a node some other way. - pub(crate) fn node_preferred(&self, nodes: &[NodeId]) -> Option { - if nodes.is_empty() { - return None; - } - - // TODO: When the utilization score returned by the pageserver becomes meaningful, - // schedule based on that instead of the shard count. - let node = nodes - .iter() - .map(|node_id| { - let may_schedule = self - .nodes - .get(node_id) - .map(|n| !matches!(n.may_schedule, MaySchedule::No)) - .unwrap_or(false); - (*node_id, may_schedule) - }) - .max_by_key(|(_n, may_schedule)| *may_schedule); - - // If even the preferred node has may_schedule==false, return None - node.and_then(|(node_id, may_schedule)| if may_schedule { Some(node_id) } else { None }) + /// Calculate a single node's score, used in optimizer logic to compare specific + /// nodes' scores. + pub(crate) fn compute_node_score( + &mut self, + node_id: NodeId, + preferred_az: &Option, + context: &ScheduleContext, + ) -> Option + where + Score: NodeSchedulingScore, + { + self.nodes + .get_mut(&node_id) + .and_then(|node| Score::generate(&node_id, node, preferred_az, context)) } /// Compute a schedulling score for each node that the scheduler knows of @@ -727,7 +776,7 @@ impl Scheduler { tracing::info!( "scheduler selected node {node_id} (elegible nodes {:?}, hard exclude: {hard_exclude:?}, soft exclude: {context:?})", scores.iter().map(|i| i.node_id().0).collect::>() - ); + ); } // Note that we do not update shard count here to reflect the scheduling: that @@ -743,47 +792,74 @@ impl Scheduler { } /// For choosing which AZ to schedule a new shard into, use this. It will return the - /// AZ with the lowest median utilization. + /// AZ with the the lowest number of shards currently scheduled in this AZ as their home + /// location. /// /// We use an AZ-wide measure rather than simply selecting the AZ of the least-loaded /// node, because while tenants start out single sharded, when they grow and undergo - /// shard-split, they will occupy space on many nodes within an AZ. + /// shard-split, they will occupy space on many nodes within an AZ. It is important + /// that we pick the AZ in a way that balances this _future_ load. /// - /// We use median rather than total free space or mean utilization, because - /// we wish to avoid preferring AZs that have low-load nodes resulting from - /// recent replacements. - /// - /// The practical result is that we will pick an AZ based on its median node, and - /// then actually _schedule_ the new shard onto the lowest-loaded node in that AZ. + /// Once we've picked an AZ, subsequent scheduling within that AZ will be driven by + /// nodes' utilization scores. pub(crate) fn get_az_for_new_tenant(&self) -> Option { if self.nodes.is_empty() { return None; } - let mut scores_by_az = HashMap::new(); - for (node_id, node) in &self.nodes { - let az_scores = scores_by_az.entry(&node.az).or_insert_with(Vec::new); - let score = match &node.may_schedule { - MaySchedule::Yes(utilization) => utilization.score(), - MaySchedule::No => PageserverUtilization::full().score(), - }; - az_scores.push((node_id, node, score)); + #[derive(Default)] + struct AzScore { + home_shard_count: usize, + scheduleable: bool, } - // Sort by utilization. Also include the node ID to break ties. - for scores in scores_by_az.values_mut() { - scores.sort_by_key(|i| (i.2, i.0)); + let mut azs: HashMap<&AvailabilityZone, AzScore> = HashMap::new(); + for node in self.nodes.values() { + let az = azs.entry(&node.az).or_default(); + az.home_shard_count += node.home_shard_count; + az.scheduleable |= matches!(node.may_schedule, MaySchedule::Yes(_)); } - let mut median_by_az = scores_by_az + // If any AZs are schedulable, then filter out the non-schedulable ones (i.e. AZs where + // all nodes are overloaded or otherwise unschedulable). + if azs.values().any(|i| i.scheduleable) { + azs.retain(|_, i| i.scheduleable); + } + + // Find the AZ with the lowest number of shards currently allocated + Some( + azs.into_iter() + .min_by_key(|i| (i.1.home_shard_count, i.0)) + .unwrap() + .0 + .clone(), + ) + } + + pub(crate) fn get_node_az(&self, node_id: &NodeId) -> Option { + self.nodes.get(node_id).map(|n| n.az.clone()) + } + + /// For use when choosing a preferred secondary location: filter out nodes that are not + /// available, and gather their AZs. + pub(crate) fn filter_usable_nodes( + &self, + nodes: &[NodeId], + ) -> Vec<(NodeId, Option)> { + nodes .iter() - .map(|(az, nodes)| (*az, nodes.get(nodes.len() / 2).unwrap().2)) - .collect::>(); - // Sort by utilization. Also include the AZ to break ties. - median_by_az.sort_by_key(|i| (i.1, i.0)); - - // Return the AZ with the lowest median utilization - Some(median_by_az.first().unwrap().0.clone()) + .filter_map(|node_id| { + let node = self + .nodes + .get(node_id) + .expect("Referenced nodes always exist"); + if matches!(node.may_schedule, MaySchedule::Yes(_)) { + Some((*node_id, Some(node.az.clone()))) + } else { + None + } + }) + .collect() } /// Unit test access to internal state @@ -843,7 +919,14 @@ pub(crate) mod test_utils { #[cfg(test)] mod tests { - use pageserver_api::{controller_api::NodeAvailability, models::utilization::test_utilization}; + use pageserver_api::{ + controller_api::NodeAvailability, models::utilization::test_utilization, + shard::ShardIdentity, + }; + use utils::{ + id::TenantId, + shard::{ShardCount, ShardNumber, TenantShardId}, + }; use super::*; @@ -853,8 +936,8 @@ mod tests { let nodes = test_utils::make_test_nodes(2, &[]); let mut scheduler = Scheduler::new(nodes.values()); - let mut t1_intent = IntentState::new(); - let mut t2_intent = IntentState::new(); + let mut t1_intent = IntentState::new(None); + let mut t2_intent = IntentState::new(None); let context = ScheduleContext::default(); @@ -930,7 +1013,7 @@ mod tests { let scheduled = scheduler .schedule_shard::(&[], &None, context) .unwrap(); - let mut intent = IntentState::new(); + let mut intent = IntentState::new(None); intent.set_attached(scheduler, Some(scheduled)); scheduled_intents.push(intent); assert_eq!(scheduled, expect_node); @@ -1063,7 +1146,7 @@ mod tests { let scheduled = scheduler .schedule_shard::(&[], &preferred_az, context) .unwrap(); - let mut intent = IntentState::new(); + let mut intent = IntentState::new(preferred_az.clone()); intent.set_attached(scheduler, Some(scheduled)); scheduled_intents.push(intent); assert_eq!(scheduled, expect_node); @@ -1089,9 +1172,9 @@ mod tests { &mut context, ); - // Node 2 is not in "az-a", but it has the lowest affinity so we prefer that. + // Node 1 and 3 (az-a) have same affinity score, so prefer the lowest node id. assert_scheduler_chooses::( - NodeId(2), + NodeId(1), Some(az_a_tag.clone()), &mut scheduled_intents, &mut scheduler, @@ -1107,26 +1190,6 @@ mod tests { &mut context, ); - // Avoid nodes in "az-b" for the secondary location. - // Nodes 1 and 3 are identically loaded, so prefer the lowest node id. - assert_scheduler_chooses::( - NodeId(1), - Some(az_b_tag.clone()), - &mut scheduled_intents, - &mut scheduler, - &mut context, - ); - - // Avoid nodes in "az-b" for the secondary location. - // Node 3 has lower affinity score than 1, so prefer that. - assert_scheduler_chooses::( - NodeId(3), - Some(az_b_tag.clone()), - &mut scheduled_intents, - &mut scheduler, - &mut context, - ); - for mut intent in scheduled_intents { intent.clear(&mut scheduler); } @@ -1150,34 +1213,292 @@ mod tests { let mut scheduler = Scheduler::new(nodes.values()); - /// Force the utilization of a node in Scheduler's state to a particular - /// number of bytes used. - fn set_utilization(scheduler: &mut Scheduler, node_id: NodeId, shard_count: u32) { - let mut node = Node::new( - node_id, - "".to_string(), - 0, - "".to_string(), - 0, - scheduler.nodes.get(&node_id).unwrap().az.clone(), - ); - node.set_availability(NodeAvailability::Active(test_utilization::simple( - shard_count, - 0, - ))); - scheduler.node_upsert(&node); + /// Force the `home_shard_count` of a node directly: this is the metric used + /// by the scheduler when picking AZs. + fn set_shard_count(scheduler: &mut Scheduler, node_id: NodeId, shard_count: usize) { + let node = scheduler.nodes.get_mut(&node_id).unwrap(); + node.home_shard_count = shard_count; } // Initial empty state. Scores are tied, scheduler prefers lower AZ ID. assert_eq!(scheduler.get_az_for_new_tenant(), Some(az_a_tag.clone())); - // Put some utilization on one node in AZ A: this should change nothing, as the median hasn't changed - set_utilization(&mut scheduler, NodeId(1), 1000000); - assert_eq!(scheduler.get_az_for_new_tenant(), Some(az_a_tag.clone())); - - // Put some utilization on a second node in AZ A: now the median has changed, so the scheduler - // should prefer the other AZ. - set_utilization(&mut scheduler, NodeId(2), 1000000); + // Home shard count is higher in AZ A, so AZ B will be preferred + set_shard_count(&mut scheduler, NodeId(1), 10); assert_eq!(scheduler.get_az_for_new_tenant(), Some(az_b_tag.clone())); + + // Total home shard count is higher in AZ B, so we revert to preferring AZ A + set_shard_count(&mut scheduler, NodeId(4), 6); + set_shard_count(&mut scheduler, NodeId(5), 6); + assert_eq!(scheduler.get_az_for_new_tenant(), Some(az_a_tag.clone())); + } + + /// Test that when selecting AZs for many new tenants, we get the expected balance across nodes + #[test] + fn az_selection_many() { + let az_a_tag = AvailabilityZone("az-a".to_string()); + let az_b_tag = AvailabilityZone("az-b".to_string()); + let az_c_tag = AvailabilityZone("az-c".to_string()); + let nodes = test_utils::make_test_nodes( + 6, + &[ + az_a_tag.clone(), + az_b_tag.clone(), + az_c_tag.clone(), + az_a_tag.clone(), + az_b_tag.clone(), + az_c_tag.clone(), + ], + ); + + let mut scheduler = Scheduler::new(nodes.values()); + + // We should get 1/6th of these on each node, give or take a few... + let total_tenants = 300; + + // ...where the 'few' is the number of AZs, because the scheduling will sometimes overshoot + // on one AZ before correcting itself. This is because we select the 'home' AZ based on + // an AZ-wide metric, but we select the location for secondaries on a purely node-based + // metric (while excluding the home AZ). + let grace = 3; + + let mut scheduled_shards = Vec::new(); + for _i in 0..total_tenants { + let preferred_az = scheduler.get_az_for_new_tenant().unwrap(); + + let mut node_home_counts = scheduler + .nodes + .iter() + .map(|(node_id, node)| (node_id, node.home_shard_count)) + .collect::>(); + node_home_counts.sort_by_key(|i| i.0); + eprintln!("Selected {}, vs nodes {:?}", preferred_az, node_home_counts); + + let tenant_shard_id = TenantShardId { + tenant_id: TenantId::generate(), + shard_number: ShardNumber(0), + shard_count: ShardCount(1), + }; + + let shard_identity = ShardIdentity::new( + tenant_shard_id.shard_number, + tenant_shard_id.shard_count, + pageserver_api::shard::ShardStripeSize(1), + ) + .unwrap(); + let mut shard = TenantShard::new( + tenant_shard_id, + shard_identity, + pageserver_api::controller_api::PlacementPolicy::Attached(1), + Some(preferred_az), + ); + + let mut context = ScheduleContext::default(); + shard.schedule(&mut scheduler, &mut context).unwrap(); + eprintln!("Scheduled shard at {:?}", shard.intent); + + scheduled_shards.push(shard); + } + + for (node_id, node) in &scheduler.nodes { + eprintln!( + "Node {}: {} {} {}", + node_id, node.shard_count, node.attached_shard_count, node.home_shard_count + ); + } + + for node in scheduler.nodes.values() { + assert!((node.home_shard_count as i64 - total_tenants as i64 / 6).abs() < grace); + } + + for mut shard in scheduled_shards { + shard.intent.clear(&mut scheduler); + } + } + + #[test] + /// Make sure that when we have an odd number of nodes and an even number of shards, we still + /// get scheduling stability. + fn odd_nodes_stability() { + let az_a = AvailabilityZone("az-a".to_string()); + let az_b = AvailabilityZone("az-b".to_string()); + + let nodes = test_utils::make_test_nodes( + 10, + &[ + az_a.clone(), + az_a.clone(), + az_a.clone(), + az_a.clone(), + az_a.clone(), + az_b.clone(), + az_b.clone(), + az_b.clone(), + az_b.clone(), + az_b.clone(), + ], + ); + let mut scheduler = Scheduler::new(nodes.values()); + + // Need to keep these alive because they contribute to shard counts via RAII + let mut scheduled_shards = Vec::new(); + + let mut context = ScheduleContext::default(); + + fn schedule_shard( + tenant_shard_id: TenantShardId, + expect_attached: NodeId, + expect_secondary: NodeId, + scheduled_shards: &mut Vec, + scheduler: &mut Scheduler, + preferred_az: Option, + context: &mut ScheduleContext, + ) { + let shard_identity = ShardIdentity::new( + tenant_shard_id.shard_number, + tenant_shard_id.shard_count, + pageserver_api::shard::ShardStripeSize(1), + ) + .unwrap(); + let mut shard = TenantShard::new( + tenant_shard_id, + shard_identity, + pageserver_api::controller_api::PlacementPolicy::Attached(1), + preferred_az, + ); + + shard.schedule(scheduler, context).unwrap(); + + assert_eq!(shard.intent.get_attached().unwrap(), expect_attached); + assert_eq!( + shard.intent.get_secondary().first().unwrap(), + &expect_secondary + ); + + scheduled_shards.push(shard); + } + + let tenant_id = TenantId::generate(); + + schedule_shard( + TenantShardId { + tenant_id, + shard_number: ShardNumber(0), + shard_count: ShardCount(8), + }, + NodeId(1), + NodeId(6), + &mut scheduled_shards, + &mut scheduler, + Some(az_a.clone()), + &mut context, + ); + + schedule_shard( + TenantShardId { + tenant_id, + shard_number: ShardNumber(1), + shard_count: ShardCount(8), + }, + NodeId(2), + NodeId(7), + &mut scheduled_shards, + &mut scheduler, + Some(az_a.clone()), + &mut context, + ); + + schedule_shard( + TenantShardId { + tenant_id, + shard_number: ShardNumber(2), + shard_count: ShardCount(8), + }, + NodeId(3), + NodeId(8), + &mut scheduled_shards, + &mut scheduler, + Some(az_a.clone()), + &mut context, + ); + + schedule_shard( + TenantShardId { + tenant_id, + shard_number: ShardNumber(3), + shard_count: ShardCount(8), + }, + NodeId(4), + NodeId(9), + &mut scheduled_shards, + &mut scheduler, + Some(az_a.clone()), + &mut context, + ); + + schedule_shard( + TenantShardId { + tenant_id, + shard_number: ShardNumber(4), + shard_count: ShardCount(8), + }, + NodeId(5), + NodeId(10), + &mut scheduled_shards, + &mut scheduler, + Some(az_a.clone()), + &mut context, + ); + + schedule_shard( + TenantShardId { + tenant_id, + shard_number: ShardNumber(5), + shard_count: ShardCount(8), + }, + NodeId(1), + NodeId(6), + &mut scheduled_shards, + &mut scheduler, + Some(az_a.clone()), + &mut context, + ); + + schedule_shard( + TenantShardId { + tenant_id, + shard_number: ShardNumber(6), + shard_count: ShardCount(8), + }, + NodeId(2), + NodeId(7), + &mut scheduled_shards, + &mut scheduler, + Some(az_a.clone()), + &mut context, + ); + + schedule_shard( + TenantShardId { + tenant_id, + shard_number: ShardNumber(7), + shard_count: ShardCount(8), + }, + NodeId(3), + NodeId(8), + &mut scheduled_shards, + &mut scheduler, + Some(az_a.clone()), + &mut context, + ); + + // Assert that the optimizer suggests nochanges, i.e. our initial scheduling was stable. + for shard in &scheduled_shards { + assert_eq!(shard.optimize_attachment(&mut scheduler, &context), None); + } + + for mut shard in scheduled_shards { + shard.intent.clear(&mut scheduler); + } } } diff --git a/storage_controller/src/service.rs b/storage_controller/src/service.rs index 8aa263f0c3..dadcc44cfb 100644 --- a/storage_controller/src/service.rs +++ b/storage_controller/src/service.rs @@ -1404,7 +1404,11 @@ impl Service { // We will populate intent properly later in [`Self::startup_reconcile`], initially populate // it with what we can infer: the node for which a generation was most recently issued. - let mut intent = IntentState::new(); + let mut intent = IntentState::new( + tsp.preferred_az_id + .as_ref() + .map(|az| AvailabilityZone(az.clone())), + ); if let Some(generation_pageserver) = tsp.generation_pageserver.map(|n| NodeId(n as u64)) { if nodes.contains_key(&generation_pageserver) { @@ -2474,18 +2478,29 @@ impl Service { tenant_id: TenantId, _guard: &TracingExclusiveGuard, ) -> Result<(), ApiError> { - let present_in_memory = { + // Check if the tenant is present in memory, and select an AZ to use when loading + // if we will load it. + let load_in_az = { let locked = self.inner.read().unwrap(); - locked + let existing = locked .tenants .range(TenantShardId::tenant_range(tenant_id)) - .next() - .is_some() - }; + .next(); - if present_in_memory { - return Ok(()); - } + // If the tenant is not present in memory, we expect to load it from database, + // so let's figure out what AZ to load it into while we have self.inner locked. + if existing.is_none() { + locked + .scheduler + .get_az_for_new_tenant() + .ok_or(ApiError::BadRequest(anyhow::anyhow!( + "No AZ with nodes found to load tenant" + )))? + } else { + // We already have this tenant in memory + return Ok(()); + } + }; let tenant_shards = self.persistence.load_tenant(tenant_id).await?; if tenant_shards.is_empty() { @@ -2494,8 +2509,20 @@ impl Service { )); } - // TODO: choose a fresh AZ to use for this tenant when un-detaching: there definitely isn't a running - // compute, so no benefit to making AZ sticky across detaches. + // Update the persistent shards with the AZ that we are about to apply to in-memory state + self.persistence + .set_tenant_shard_preferred_azs( + tenant_shards + .iter() + .map(|t| { + ( + t.get_tenant_shard_id().expect("Corrupt shard in database"), + load_in_az.clone(), + ) + }) + .collect(), + ) + .await?; let mut locked = self.inner.write().unwrap(); tracing::info!( @@ -2505,7 +2532,7 @@ impl Service { ); locked.tenants.extend(tenant_shards.into_iter().map(|p| { - let intent = IntentState::new(); + let intent = IntentState::new(Some(load_in_az.clone())); let shard = TenantShard::from_persistent(p, intent).expect("Corrupt shard row in database"); @@ -4236,6 +4263,22 @@ impl Service { } tracing::info!("Restoring parent shard {tenant_shard_id}"); + + // Drop any intents that refer to unavailable nodes, to enable this abort to proceed even + // if the original attachment location is offline. + if let Some(node_id) = shard.intent.get_attached() { + if !nodes.get(node_id).unwrap().is_available() { + tracing::info!("Demoting attached intent for {tenant_shard_id} on unavailable node {node_id}"); + shard.intent.demote_attached(scheduler, *node_id); + } + } + for node_id in shard.intent.get_secondary().clone() { + if !nodes.get(&node_id).unwrap().is_available() { + tracing::info!("Dropping secondary intent for {tenant_shard_id} on unavailable node {node_id}"); + shard.intent.remove_secondary(scheduler, node_id); + } + } + shard.splitting = SplitState::Idle; if let Err(e) = shard.schedule(scheduler, &mut ScheduleContext::default()) { // If this shard can't be scheduled now (perhaps due to offline nodes or @@ -4389,15 +4432,13 @@ impl Service { let mut child_state = TenantShard::new(child, child_shard, policy.clone(), preferred_az.clone()); - child_state.intent = IntentState::single(scheduler, Some(pageserver)); + child_state.intent = + IntentState::single(scheduler, Some(pageserver), preferred_az.clone()); child_state.observed = ObservedState { locations: child_observed, }; child_state.generation = Some(generation); child_state.config = config.clone(); - if let Some(preferred_az) = &preferred_az { - child_state.set_preferred_az(preferred_az.clone()); - } // The child's TenantShard::splitting is intentionally left at the default value of Idle, // as at this point in the split process we have succeeded and this part is infallible: @@ -5014,6 +5055,8 @@ impl Service { // If our new attached node was a secondary, it no longer should be. shard.intent.remove_secondary(scheduler, migrate_req.node_id); + shard.intent.set_attached(scheduler, Some(migrate_req.node_id)); + // If we were already attached to something, demote that to a secondary if let Some(old_attached) = old_attached { if n > 0 { @@ -5025,8 +5068,6 @@ impl Service { shard.intent.push_secondary(scheduler, old_attached); } } - - shard.intent.set_attached(scheduler, Some(migrate_req.node_id)); } PlacementPolicy::Secondary => { shard.intent.clear(scheduler); @@ -5712,7 +5753,7 @@ impl Service { register_req.listen_http_port, register_req.listen_pg_addr, register_req.listen_pg_port, - register_req.availability_zone_id, + register_req.availability_zone_id.clone(), ); // TODO: idempotency if the node already exists in the database @@ -5732,8 +5773,9 @@ impl Service { .set(locked.nodes.len() as i64); tracing::info!( - "Registered pageserver {}, now have {} pageservers", + "Registered pageserver {} ({}), now have {} pageservers", register_req.node_id, + register_req.availability_zone_id, locked.nodes.len() ); Ok(()) @@ -6467,6 +6509,7 @@ impl Service { // Shard was dropped between planning and execution; continue; }; + tracing::info!("Applying optimization: {optimization:?}"); if shard.apply_optimization(scheduler, optimization) { optimizations_applied += 1; if self.maybe_reconcile_shard(shard, nodes).is_some() { @@ -6497,7 +6540,13 @@ impl Service { let mut work = Vec::new(); let mut locked = self.inner.write().unwrap(); - let (nodes, tenants, scheduler) = locked.parts_mut(); + let (_nodes, tenants, scheduler) = locked.parts_mut(); + + // We are going to plan a bunch of optimisations before applying any of them, so the + // utilisation stats on nodes will be effectively stale for the >1st optimisation we + // generate. To avoid this causing unstable migrations/flapping, it's important that the + // code in TenantShard for finding optimisations uses [`NodeAttachmentSchedulingScore::disregard_utilization`] + // to ignore the utilisation component of the score. for (_tenant_id, schedule_context, shards) in TenantShardContextIterator::new(tenants, ScheduleMode::Speculative) @@ -6528,13 +6577,28 @@ impl Service { continue; } - // TODO: optimization calculations are relatively expensive: create some fast-path for - // the common idle case (avoiding the search on tenants that we have recently checked) + // Fast path: we may quickly identify shards that don't have any possible optimisations + if !shard.maybe_optimizable(scheduler, &schedule_context) { + if cfg!(feature = "testing") { + // Check that maybe_optimizable doesn't disagree with the actual optimization functions. + // Only do this in testing builds because it is not a correctness-critical check, so we shouldn't + // panic in prod if we hit this, or spend cycles on it in prod. + assert!(shard + .optimize_attachment(scheduler, &schedule_context) + .is_none()); + assert!(shard + .optimize_secondary(scheduler, &schedule_context) + .is_none()); + } + continue; + } + if let Some(optimization) = - // If idle, maybe ptimize attachments: if a shard has a secondary location that is preferable to + // If idle, maybe optimize attachments: if a shard has a secondary location that is preferable to // its primary location based on soft constraints, cut it over. - shard.optimize_attachment(nodes, &schedule_context) + shard.optimize_attachment(scheduler, &schedule_context) { + tracing::info!(tenant_shard_id=%shard.tenant_shard_id, "Identified optimization for attachment: {optimization:?}"); work.push((shard.tenant_shard_id, optimization)); break; } else if let Some(optimization) = @@ -6544,6 +6608,7 @@ impl Service { // in the same tenant with secondary locations on the node where they originally split. shard.optimize_secondary(scheduler, &schedule_context) { + tracing::info!(tenant_shard_id=%shard.tenant_shard_id, "Identified optimization for secondary: {optimization:?}"); work.push((shard.tenant_shard_id, optimization)); break; } @@ -6592,8 +6657,10 @@ impl Service { } } } - ScheduleOptimizationAction::ReplaceSecondary(_) => { - // No extra checks needed to replace a secondary: this does not interrupt client access + ScheduleOptimizationAction::ReplaceSecondary(_) + | ScheduleOptimizationAction::CreateSecondary(_) + | ScheduleOptimizationAction::RemoveSecondary(_) => { + // No extra checks needed to manage secondaries: this does not interrupt client access validated_work.push((tenant_shard_id, optimization)) } }; @@ -6665,26 +6732,35 @@ impl Service { /// we have this helper to move things along faster. #[cfg(feature = "testing")] async fn kick_secondary_download(&self, tenant_shard_id: TenantShardId) { - let (attached_node, secondary_node) = { + let (attached_node, secondaries) = { let locked = self.inner.read().unwrap(); let Some(shard) = locked.tenants.get(&tenant_shard_id) else { + tracing::warn!( + "Skipping kick of secondary download for {tenant_shard_id}: not found" + ); return; }; - let (Some(attached), Some(secondary)) = ( - shard.intent.get_attached(), - shard.intent.get_secondary().first(), - ) else { + + let Some(attached) = shard.intent.get_attached() else { + tracing::warn!( + "Skipping kick of secondary download for {tenant_shard_id}: no attached" + ); return; }; - ( - locked.nodes.get(attached).unwrap().clone(), - locked.nodes.get(secondary).unwrap().clone(), - ) + + let secondaries = shard + .intent + .get_secondary() + .iter() + .map(|n| locked.nodes.get(n).unwrap().clone()) + .collect::>(); + + (locked.nodes.get(attached).unwrap().clone(), secondaries) }; // Make remote API calls to upload + download heatmaps: we ignore errors because this is just // a 'kick' to let scheduling optimisation run more promptly. - attached_node + match attached_node .with_client_retries( |client| async move { client.tenant_heatmap_upload(tenant_shard_id).await }, &self.config.jwt_token, @@ -6693,22 +6769,57 @@ impl Service { SHORT_RECONCILE_TIMEOUT, &self.cancel, ) - .await; + .await + { + Some(Err(e)) => { + tracing::info!( + "Failed to upload heatmap from {attached_node} for {tenant_shard_id}: {e}" + ); + } + None => { + tracing::info!( + "Cancelled while uploading heatmap from {attached_node} for {tenant_shard_id}" + ); + } + Some(Ok(_)) => { + tracing::info!( + "Successfully uploaded heatmap from {attached_node} for {tenant_shard_id}" + ); + } + } - secondary_node - .with_client_retries( - |client| async move { - client - .tenant_secondary_download(tenant_shard_id, Some(Duration::from_secs(1))) - .await - }, - &self.config.jwt_token, - 3, - 10, - SHORT_RECONCILE_TIMEOUT, - &self.cancel, - ) - .await; + for secondary_node in secondaries { + match secondary_node + .with_client_retries( + |client| async move { + client + .tenant_secondary_download( + tenant_shard_id, + Some(Duration::from_secs(1)), + ) + .await + }, + &self.config.jwt_token, + 3, + 10, + SHORT_RECONCILE_TIMEOUT, + &self.cancel, + ) + .await + { + Some(Err(e)) => { + tracing::info!( + "Failed to download heatmap from {secondary_node} for {tenant_shard_id}: {e}" + ); + } + None => { + tracing::info!("Cancelled while downloading heatmap from {secondary_node} for {tenant_shard_id}"); + } + Some(Ok(progress)) => { + tracing::info!("Successfully downloaded heatmap from {secondary_node} for {tenant_shard_id}: {progress:?}"); + } + } + } } /// Look for shards which are oversized and in need of splitting @@ -7144,9 +7255,15 @@ impl Service { fn fill_node_plan(&self, node_id: NodeId) -> Vec { let mut locked = self.inner.write().unwrap(); let fill_requirement = locked.scheduler.compute_fill_requirement(node_id); + let (nodes, tenants, _scheduler) = locked.parts_mut(); - let mut tids_by_node = locked - .tenants + let node_az = nodes + .get(&node_id) + .expect("Node must exist") + .get_availability_zone_id() + .clone(); + + let mut tids_by_node = tenants .iter_mut() .filter_map(|(tid, tenant_shard)| { if !matches!( @@ -7159,6 +7276,25 @@ impl Service { return None; } + // AZ check: when filling nodes after a restart, our intent is to move _back_ the + // shards which belong on this node, not to promote shards whose scheduling preference + // would be on their currently attached node. So will avoid promoting shards whose + // home AZ doesn't match the AZ of the node we're filling. + match tenant_shard.preferred_az() { + None => { + // Shard doesn't have an AZ preference: it is elegible to be moved. + } + Some(az) if az == &node_az => { + // This shard's home AZ is equal to the node we're filling: it is + // elegible to be moved: fall through; + } + Some(_) => { + // This shard's home AZ is somewhere other than the node we're filling: + // do not include it in the fill plan. + return None; + } + } + if tenant_shard.intent.get_secondary().contains(&node_id) { if let Some(primary) = tenant_shard.intent.get_attached() { return Some((*primary, *tid)); diff --git a/storage_controller/src/service/context_iterator.rs b/storage_controller/src/service/context_iterator.rs index d38010a27e..dd6913e988 100644 --- a/storage_controller/src/service/context_iterator.rs +++ b/storage_controller/src/service/context_iterator.rs @@ -43,9 +43,6 @@ impl<'a> Iterator for TenantShardContextIterator<'a> { // Accumulate the schedule context for all the shards in a tenant schedule_context.avoid(&shard.intent.all_pageservers()); - if let Some(attached) = shard.intent.get_attached() { - schedule_context.push_attached(*attached); - } tenant_shards.push(shard); if tenant_shard_id.shard_number.0 == tenant_shard_id.shard_count.count() - 1 { @@ -115,7 +112,7 @@ mod tests { assert_eq!(tenant_id, t1_id); assert_eq!(shards[0].tenant_shard_id.shard_number, ShardNumber(0)); assert_eq!(shards.len(), 1); - assert_eq!(context.attach_count(), 1); + assert_eq!(context.location_count(), 2); let (tenant_id, context, shards) = iter.next().unwrap(); assert_eq!(tenant_id, t2_id); @@ -124,13 +121,13 @@ mod tests { assert_eq!(shards[2].tenant_shard_id.shard_number, ShardNumber(2)); assert_eq!(shards[3].tenant_shard_id.shard_number, ShardNumber(3)); assert_eq!(shards.len(), 4); - assert_eq!(context.attach_count(), 4); + assert_eq!(context.location_count(), 8); let (tenant_id, context, shards) = iter.next().unwrap(); assert_eq!(tenant_id, t3_id); assert_eq!(shards[0].tenant_shard_id.shard_number, ShardNumber(0)); assert_eq!(shards.len(), 1); - assert_eq!(context.attach_count(), 1); + assert_eq!(context.location_count(), 2); for shard in tenants.values_mut() { shard.intent.clear(&mut scheduler); diff --git a/storage_controller/src/tenant_shard.rs b/storage_controller/src/tenant_shard.rs index e0a71b5822..2ba2a57eba 100644 --- a/storage_controller/src/tenant_shard.rs +++ b/storage_controller/src/tenant_shard.rs @@ -11,16 +11,14 @@ use crate::{ persistence::TenantShardPersistence, reconciler::{ReconcileUnits, ReconcilerConfig}, scheduler::{ - AffinityScore, AttachedShardTag, MaySchedule, RefCountUpdate, ScheduleContext, - SecondaryShardTag, + AffinityScore, AttachedShardTag, NodeSchedulingScore, NodeSecondarySchedulingScore, + RefCountUpdate, ScheduleContext, SecondaryShardTag, ShardTag, }, service::ReconcileResultRequest, }; use futures::future::{self, Either}; use itertools::Itertools; -use pageserver_api::controller_api::{ - AvailabilityZone, NodeSchedulingPolicy, PlacementPolicy, ShardSchedulingPolicy, -}; +use pageserver_api::controller_api::{AvailabilityZone, PlacementPolicy, ShardSchedulingPolicy}; use pageserver_api::{ models::{LocationConfig, LocationConfigMode, TenantConfig}, shard::{ShardIdentity, TenantShardId}, @@ -33,6 +31,7 @@ use utils::{ generation::Generation, id::NodeId, seqwait::{SeqWait, SeqWaitError}, + shard::ShardCount, sync::gate::GateGuard, }; @@ -147,45 +146,67 @@ pub(crate) struct TenantShard { // Support/debug tool: if something is going wrong or flapping with scheduling, this may // be set to a non-active state to avoid making changes while the issue is fixed. scheduling_policy: ShardSchedulingPolicy, +} + +#[derive(Clone, Debug, Serialize)] +pub(crate) struct IntentState { + attached: Option, + secondary: Vec, // We should attempt to schedule this shard in the provided AZ to // decrease chances of cross-AZ compute. preferred_az_id: Option, } -#[derive(Default, Clone, Debug, Serialize)] -pub(crate) struct IntentState { - attached: Option, - secondary: Vec, -} - impl IntentState { - pub(crate) fn new() -> Self { + pub(crate) fn new(preferred_az_id: Option) -> Self { Self { attached: None, secondary: vec![], + preferred_az_id, } } - pub(crate) fn single(scheduler: &mut Scheduler, node_id: Option) -> Self { + pub(crate) fn single( + scheduler: &mut Scheduler, + node_id: Option, + preferred_az_id: Option, + ) -> Self { if let Some(node_id) = node_id { - scheduler.update_node_ref_counts(node_id, RefCountUpdate::Attach); + scheduler.update_node_ref_counts( + node_id, + preferred_az_id.as_ref(), + RefCountUpdate::Attach, + ); } Self { attached: node_id, secondary: vec![], + preferred_az_id, } } pub(crate) fn set_attached(&mut self, scheduler: &mut Scheduler, new_attached: Option) { if self.attached != new_attached { if let Some(old_attached) = self.attached.take() { - scheduler.update_node_ref_counts(old_attached, RefCountUpdate::Detach); + scheduler.update_node_ref_counts( + old_attached, + self.preferred_az_id.as_ref(), + RefCountUpdate::Detach, + ); } if let Some(new_attached) = &new_attached { - scheduler.update_node_ref_counts(*new_attached, RefCountUpdate::Attach); + scheduler.update_node_ref_counts( + *new_attached, + self.preferred_az_id.as_ref(), + RefCountUpdate::Attach, + ); } self.attached = new_attached; } + + if let Some(new_attached) = &new_attached { + assert!(!self.secondary.contains(new_attached)); + } } /// Like set_attached, but the node is from [`Self::secondary`]. This swaps the node from @@ -204,15 +225,28 @@ impl IntentState { let demoted = self.attached; self.attached = Some(promote_secondary); - scheduler.update_node_ref_counts(promote_secondary, RefCountUpdate::PromoteSecondary); + scheduler.update_node_ref_counts( + promote_secondary, + self.preferred_az_id.as_ref(), + RefCountUpdate::PromoteSecondary, + ); if let Some(demoted) = demoted { - scheduler.update_node_ref_counts(demoted, RefCountUpdate::DemoteAttached); + scheduler.update_node_ref_counts( + demoted, + self.preferred_az_id.as_ref(), + RefCountUpdate::DemoteAttached, + ); } } pub(crate) fn push_secondary(&mut self, scheduler: &mut Scheduler, new_secondary: NodeId) { - debug_assert!(!self.secondary.contains(&new_secondary)); - scheduler.update_node_ref_counts(new_secondary, RefCountUpdate::AddSecondary); + assert!(!self.secondary.contains(&new_secondary)); + assert!(self.attached != Some(new_secondary)); + scheduler.update_node_ref_counts( + new_secondary, + self.preferred_az_id.as_ref(), + RefCountUpdate::AddSecondary, + ); self.secondary.push(new_secondary); } @@ -220,27 +254,43 @@ impl IntentState { pub(crate) fn remove_secondary(&mut self, scheduler: &mut Scheduler, node_id: NodeId) { let index = self.secondary.iter().position(|n| *n == node_id); if let Some(index) = index { - scheduler.update_node_ref_counts(node_id, RefCountUpdate::RemoveSecondary); + scheduler.update_node_ref_counts( + node_id, + self.preferred_az_id.as_ref(), + RefCountUpdate::RemoveSecondary, + ); self.secondary.remove(index); } } pub(crate) fn clear_secondary(&mut self, scheduler: &mut Scheduler) { for secondary in self.secondary.drain(..) { - scheduler.update_node_ref_counts(secondary, RefCountUpdate::RemoveSecondary); + scheduler.update_node_ref_counts( + secondary, + self.preferred_az_id.as_ref(), + RefCountUpdate::RemoveSecondary, + ); } } /// Remove the last secondary node from the list of secondaries pub(crate) fn pop_secondary(&mut self, scheduler: &mut Scheduler) { if let Some(node_id) = self.secondary.pop() { - scheduler.update_node_ref_counts(node_id, RefCountUpdate::RemoveSecondary); + scheduler.update_node_ref_counts( + node_id, + self.preferred_az_id.as_ref(), + RefCountUpdate::RemoveSecondary, + ); } } pub(crate) fn clear(&mut self, scheduler: &mut Scheduler) { if let Some(old_attached) = self.attached.take() { - scheduler.update_node_ref_counts(old_attached, RefCountUpdate::Detach); + scheduler.update_node_ref_counts( + old_attached, + self.preferred_az_id.as_ref(), + RefCountUpdate::Detach, + ); } self.clear_secondary(scheduler); @@ -275,7 +325,11 @@ impl IntentState { if self.attached == Some(node_id) { self.attached = None; self.secondary.push(node_id); - scheduler.update_node_ref_counts(node_id, RefCountUpdate::DemoteAttached); + scheduler.update_node_ref_counts( + node_id, + self.preferred_az_id.as_ref(), + RefCountUpdate::DemoteAttached, + ); true } else { false @@ -315,6 +369,7 @@ pub(crate) struct ObservedStateLocation { /// we know that we might have some state on this node. pub(crate) conf: Option, } + pub(crate) struct ReconcilerWaiter { // For observability purposes, remember the ID of the shard we're // waiting for. @@ -360,6 +415,10 @@ pub(crate) enum ScheduleOptimizationAction { ReplaceSecondary(ReplaceSecondary), // Migrate attachment to an existing secondary location MigrateAttachment(MigrateAttachment), + // Create a secondary location, with the intent of later migrating to it + CreateSecondary(NodeId), + // Remove a secondary location that we previously created to facilitate a migration + RemoveSecondary(NodeId), } #[derive(Eq, PartialEq, Debug, Clone)] @@ -486,7 +545,7 @@ impl TenantShard { Self { tenant_shard_id, policy, - intent: IntentState::default(), + intent: IntentState::new(preferred_az_id), generation: Some(Generation::new(0)), shard, observed: ObservedState::default(), @@ -500,7 +559,6 @@ impl TenantShard { last_error: Arc::default(), pending_compute_notification: false, scheduling_policy: ShardSchedulingPolicy::default(), - preferred_az_id, } } @@ -563,7 +621,7 @@ impl TenantShard { return Ok((false, node_id)); } - if let Some(promote_secondary) = scheduler.node_preferred(&self.intent.secondary) { + if let Some(promote_secondary) = self.preferred_secondary(scheduler) { // Promote a secondary tracing::debug!("Promoted secondary {} to attached", promote_secondary); self.intent.promote_attached(scheduler, promote_secondary); @@ -572,7 +630,7 @@ impl TenantShard { // Pick a fresh node: either we had no secondaries or none were schedulable let node_id = scheduler.schedule_shard::( &self.intent.secondary, - &self.preferred_az_id, + &self.intent.preferred_az_id, context, )?; tracing::debug!("Selected {} as attached", node_id); @@ -594,9 +652,6 @@ impl TenantShard { let r = self.do_schedule(scheduler, context); context.avoid(&self.intent.all_pageservers()); - if let Some(attached) = self.intent.get_attached() { - context.push_attached(*attached); - } r } @@ -631,24 +686,7 @@ impl TenantShard { use PlacementPolicy::*; match self.policy { Attached(secondary_count) => { - let retain_secondaries = if self.intent.attached.is_none() - && scheduler.node_preferred(&self.intent.secondary).is_some() - { - // If we have no attached, and one of the secondaries is elegible to be promoted, retain - // one more secondary than we usually would, as one of them will become attached futher down this function. - secondary_count + 1 - } else { - secondary_count - }; - - while self.intent.secondary.len() > retain_secondaries { - // We have no particular preference for one secondary location over another: just - // arbitrarily drop from the end - self.intent.pop_secondary(scheduler); - modified = true; - } - - // Should have exactly one attached, and N secondaries + // Should have exactly one attached, and at least N secondaries let (modified_attached, attached_node_id) = self.schedule_attached(scheduler, context)?; modified |= modified_attached; @@ -657,7 +695,7 @@ impl TenantShard { while self.intent.secondary.len() < secondary_count { let node_id = scheduler.schedule_shard::( &used_pageservers, - &self.preferred_az_id, + &self.intent.preferred_az_id, context, )?; self.intent.push_secondary(scheduler, node_id); @@ -674,7 +712,7 @@ impl TenantShard { // Populate secondary by scheduling a fresh node let node_id = scheduler.schedule_shard::( &[], - &self.preferred_az_id, + &self.intent.preferred_az_id, context, )?; self.intent.push_secondary(scheduler, node_id); @@ -718,7 +756,7 @@ impl TenantShard { ) -> Result<(), ScheduleError> { let promote_to = match promote_to { Some(node) => node, - None => match scheduler.node_preferred(self.intent.get_secondary()) { + None => match self.preferred_secondary(scheduler) { Some(node) => node, None => { return Err(ScheduleError::ImpossibleConstraint); @@ -745,90 +783,276 @@ impl TenantShard { Ok(()) } + /// Returns None if the current location's score is unavailable, i.e. cannot draw a conclusion + fn is_better_location( + &self, + scheduler: &mut Scheduler, + schedule_context: &ScheduleContext, + current: NodeId, + candidate: NodeId, + ) -> Option { + let Some(candidate_score) = scheduler.compute_node_score::( + candidate, + &self.intent.preferred_az_id, + schedule_context, + ) else { + // The candidate node is unavailable for scheduling or otherwise couldn't get a score + return None; + }; + + match scheduler.compute_node_score::( + current, + &self.intent.preferred_az_id, + schedule_context, + ) { + Some(current_score) => { + // Ignore utilization components when comparing scores: we don't want to migrate + // because of transient load variations, it risks making the system thrash, and + // migrating for utilization requires a separate high level view of the system to + // e.g. prioritize moving larger or smaller tenants, rather than arbitrarily + // moving things around in the order that we hit this function. + let candidate_score = candidate_score.for_optimization(); + let current_score = current_score.for_optimization(); + + if candidate_score < current_score { + tracing::info!("Found a lower scoring location! {candidate} is better than {current} ({candidate_score:?} is better than {current_score:?})"); + Some(true) + } else { + // The candidate node is no better than our current location, so don't migrate + tracing::debug!( + "Candidate node {candidate} is no better than our current location {current} (candidate {candidate_score:?} vs current {current_score:?})", + ); + Some(false) + } + } + None => { + // The current node is unavailable for scheduling, so we can't make any sensible + // decisions about optimisation. This should be a transient state -- if the node + // is offline then it will get evacuated, if is blocked by a scheduling mode + // then we will respect that mode by doing nothing. + tracing::debug!("Current node {current} is unavailable for scheduling"); + None + } + } + } + + fn find_better_location( + &self, + scheduler: &mut Scheduler, + schedule_context: &ScheduleContext, + current: NodeId, + hard_exclude: &[NodeId], + ) -> Option { + // Look for a lower-scoring location to attach to + let Ok(candidate_node) = scheduler.schedule_shard::( + hard_exclude, + &self.intent.preferred_az_id, + schedule_context, + ) else { + // A scheduling error means we have no possible candidate replacements + tracing::debug!("No candidate node found"); + return None; + }; + + if candidate_node == current { + // We're already at the best possible location, so don't migrate + tracing::debug!("Candidate node {candidate_node} is already in use"); + return None; + } + + self.is_better_location::(scheduler, schedule_context, current, candidate_node) + .and_then(|better| if better { Some(candidate_node) } else { None }) + } + + /// This function is an optimization, used to avoid doing large numbers of scheduling operations + /// when looking for optimizations. This function uses knowledge of how scores work to do some + /// fast checks for whether it may to be possible to improve a score. + /// + /// If we return true, it only means that optimization _might_ be possible, not that it necessarily is. If we + /// return no, it definitely means that calling [`Self::optimize_attachment`] or [`Self::optimize_secondary`] would do no + /// work. + pub(crate) fn maybe_optimizable( + &self, + scheduler: &mut Scheduler, + schedule_context: &ScheduleContext, + ) -> bool { + // Sharded tenant: check if any locations have a nonzero affinity score + if self.shard.count >= ShardCount(1) { + let schedule_context = schedule_context.project_detach(self); + for node in self.intent.all_pageservers() { + if let Some(af) = schedule_context.nodes.get(&node) { + if *af > AffinityScore(0) { + return true; + } + } + } + } + + // Attached tenant: check if the attachment is outside the preferred AZ + if let PlacementPolicy::Attached(_) = self.policy { + if let Some(attached) = self.intent.get_attached() { + if scheduler.get_node_az(attached) != self.intent.preferred_az_id { + return true; + } + } + } + + // Tenant with secondary locations: check if any are within the preferred AZ + for secondary in self.intent.get_secondary() { + if scheduler.get_node_az(secondary) == self.intent.preferred_az_id { + return true; + } + } + + // Does the tenant have excess secondaries? + if self.intent.get_secondary().len() > self.policy.want_secondaries() { + return true; + } + + // Fall through: no optimizations possible + false + } + /// Optimize attachments: if a shard has a secondary location that is preferable to /// its primary location based on soft constraints, switch that secondary location /// to be attached. #[instrument(skip_all, fields(tenant_id=%self.tenant_shard_id.tenant_id, shard_id=%self.tenant_shard_id.shard_slug()))] pub(crate) fn optimize_attachment( &self, - nodes: &HashMap, + scheduler: &mut Scheduler, schedule_context: &ScheduleContext, ) -> Option { let attached = (*self.intent.get_attached())?; - if self.intent.secondary.is_empty() { - // We can only do useful work if we have both attached and secondary locations: this - // function doesn't schedule new locations, only swaps between attached and secondaries. - return None; - } - let current_affinity_score = schedule_context.get_node_affinity(attached); - let current_attachment_count = schedule_context.get_node_attachments(attached); + let schedule_context = schedule_context.project_detach(self); - // Generate score for each node, dropping any un-schedulable nodes. - let all_pageservers = self.intent.all_pageservers(); - let mut scores = all_pageservers - .iter() - .flat_map(|node_id| { - let node = nodes.get(node_id); - if node.is_none() { - None - } else if matches!( - node.unwrap().get_scheduling(), - NodeSchedulingPolicy::Filling - ) { - // If the node is currently filling, don't count it as a candidate to avoid, - // racing with the background fill. - None - } else if matches!(node.unwrap().may_schedule(), MaySchedule::No) { - None - } else { - let affinity_score = schedule_context.get_node_affinity(*node_id); - let attachment_count = schedule_context.get_node_attachments(*node_id); - Some((*node_id, affinity_score, attachment_count)) - } - }) - .collect::>(); - - // Sort precedence: - // 1st - prefer nodes with the lowest total affinity score - // 2nd - prefer nodes with the lowest number of attachments in this context - // 3rd - if all else is equal, sort by node ID for determinism in tests. - scores.sort_by_key(|i| (i.1, i.2, i.0)); - - if let Some((preferred_node, preferred_affinity_score, preferred_attachment_count)) = - scores.first() - { - if attached != *preferred_node { - // The best alternative must be more than 1 better than us, otherwise we could end - // up flapping back next time we're called (e.g. there's no point migrating from - // a location with score 1 to a score zero, because on next location the situation - // would be the same, but in reverse). - if current_affinity_score > *preferred_affinity_score + AffinityScore(1) - || current_attachment_count > *preferred_attachment_count + 1 - { - tracing::info!( - "Identified optimization: migrate attachment {attached}->{preferred_node} (secondaries {:?})", - self.intent.get_secondary() - ); - return Some(ScheduleOptimization { - sequence: self.sequence, - action: ScheduleOptimizationAction::MigrateAttachment(MigrateAttachment { - old_attached_node_id: attached, - new_attached_node_id: *preferred_node, - }), - }); - } - } else { - tracing::debug!( - "Node {} is already preferred (score {:?})", - preferred_node, - preferred_affinity_score - ); + // If we already have a secondary that is higher-scoring than out current location, + // then simply migrate to it. + for secondary in self.intent.get_secondary() { + if let Some(true) = self.is_better_location::( + scheduler, + &schedule_context, + attached, + *secondary, + ) { + return Some(ScheduleOptimization { + sequence: self.sequence, + action: ScheduleOptimizationAction::MigrateAttachment(MigrateAttachment { + old_attached_node_id: attached, + new_attached_node_id: *secondary, + }), + }); } } - // Fall-through: we didn't find an optimization - None + // Given that none of our current secondaries is a better location than our current + // attached location (checked above), we may trim any secondaries that are not needed + // for the placement policy. + if self.intent.get_secondary().len() > self.policy.want_secondaries() { + // This code path cleans up extra secondaries after migrating, and/or + // trims extra secondaries after a PlacementPolicy::Attached(N) was + // modified to decrease N. + + let secondary_scores = self + .intent + .get_secondary() + .iter() + .map(|node_id| { + ( + *node_id, + scheduler.compute_node_score::( + *node_id, + &self.intent.preferred_az_id, + &schedule_context, + ), + ) + }) + .collect::>(); + + if secondary_scores.iter().any(|score| score.1.is_none()) { + // Don't have full list of scores, so can't make a good decision about which to drop unless + // there is an obvious one in the wrong AZ + for secondary in self.intent.get_secondary() { + if scheduler.get_node_az(secondary) == self.intent.preferred_az_id { + return Some(ScheduleOptimization { + sequence: self.sequence, + action: ScheduleOptimizationAction::RemoveSecondary(*secondary), + }); + } + } + + // Fall through: we didn't identify one to remove. This ought to be rare. + tracing::warn!("Keeping extra secondaries: can't determine which of {:?} to remove (some nodes offline?)", + self.intent.get_secondary() + ); + } else { + let victim = secondary_scores + .iter() + .max_by_key(|score| score.1.unwrap()) + .unwrap() + .0; + return Some(ScheduleOptimization { + sequence: self.sequence, + action: ScheduleOptimizationAction::RemoveSecondary(victim), + }); + } + } + + let replacement = self.find_better_location::( + scheduler, + &schedule_context, + attached, + &[], // Don't exclude secondaries: our preferred attachment location may be a secondary + ); + + // We have found a candidate and confirmed that its score is preferable + // to our current location. See if we have a secondary location in the preferred location already: if not, + // then create one. + if let Some(replacement) = replacement { + // If we are currently in non-preferred AZ, then the scheduler might suggest a location that is better, but still + // not in our preferred AZ. Migration has a cost in resources an impact to the workload, so we want to avoid doing + // multiple hops where we might go to some other AZ before eventually finding a suitable location in our preferred + // AZ: skip this optimization if it is not in our final, preferred AZ. + // + // This should be a transient state, there should always be capacity eventually in our preferred AZ (even if nodes + // there are too overloaded for scheduler to suggest them, more should be provisioned eventually). + if self.intent.preferred_az_id.is_some() + && scheduler.get_node_az(&replacement) != self.intent.preferred_az_id + { + tracing::debug!( + "Candidate node {replacement} is not in preferred AZ {:?}", + self.intent.preferred_az_id + ); + + // This should only happen if our current location is not in the preferred AZ, otherwise + // [`Self::find_better_location`]` should have rejected any other location outside the preferred Az, because + // AZ is the highest priority part of NodeAttachmentSchedulingScore. + debug_assert!(scheduler.get_node_az(&attached) != self.intent.preferred_az_id); + + return None; + } + + if !self.intent.get_secondary().contains(&replacement) { + Some(ScheduleOptimization { + sequence: self.sequence, + action: ScheduleOptimizationAction::CreateSecondary(replacement), + }) + } else { + // We already have a secondary in the preferred location, let's try migrating to it. Our caller + // will check the warmth of the destination before deciding whether to really execute this. + Some(ScheduleOptimization { + sequence: self.sequence, + action: ScheduleOptimizationAction::MigrateAttachment(MigrateAttachment { + old_attached_node_id: attached, + new_attached_node_id: replacement, + }), + }) + } + } else { + // We didn't find somewhere we'd rather be, and we don't have any excess secondaries + // to clean up: no action required. + None + } } #[instrument(skip_all, fields(tenant_id=%self.tenant_shard_id.tenant_id, shard_id=%self.tenant_shard_id.shard_slug()))] @@ -837,50 +1061,40 @@ impl TenantShard { scheduler: &mut Scheduler, schedule_context: &ScheduleContext, ) -> Option { - if self.intent.secondary.is_empty() { - // We can only do useful work if we have both attached and secondary locations: this - // function doesn't schedule new locations, only swaps between attached and secondaries. + if self.intent.get_secondary().len() > self.policy.want_secondaries() { + // We have extra secondaries, perhaps to facilitate a migration of the attached location: + // do nothing, it is up to [`Self::optimize_attachment`] to clean them up. When that's done, + // and we are called again, we will proceed. + tracing::debug!("Too many secondaries: skipping"); return None; } + let schedule_context = schedule_context.project_detach(self); + for secondary in self.intent.get_secondary() { - let Some(affinity_score) = schedule_context.nodes.get(secondary) else { - // We're already on a node unaffected any affinity constraints, - // so we won't change it. - continue; + // Make sure we don't try to migrate a secondary to our attached location: this case happens + // easily in environments without multiple AZs. + let exclude = match self.intent.attached { + Some(attached) => vec![attached], + None => vec![], }; - // Let the scheduler suggest a node, where it would put us if we were scheduling afresh - // This implicitly limits the choice to nodes that are available, and prefers nodes - // with lower utilization. - let Ok(candidate_node) = scheduler.schedule_shard::( - &self.intent.all_pageservers(), - &self.preferred_az_id, - schedule_context, - ) else { - // A scheduling error means we have no possible candidate replacements - continue; - }; - - let candidate_affinity_score = schedule_context - .nodes - .get(&candidate_node) - .unwrap_or(&AffinityScore::FREE); - - // The best alternative must be more than 1 better than us, otherwise we could end - // up flapping back next time we're called. - if *candidate_affinity_score + AffinityScore(1) < *affinity_score { - // If some other node is available and has a lower score than this node, then - // that other node is a good place to migrate to. - tracing::info!( - "Identified optimization: replace secondary {secondary}->{candidate_node} (current secondaries {:?})", - self.intent.get_secondary() - ); + let replacement = self.find_better_location::( + scheduler, + &schedule_context, + *secondary, + &exclude, + ); + assert!(replacement != Some(*secondary)); + if let Some(replacement) = replacement { + // We have found a candidate and confirmed that its score is preferable + // to our current location. See if we have a secondary location in the preferred location already: if not, + // then create one. return Some(ScheduleOptimization { sequence: self.sequence, action: ScheduleOptimizationAction::ReplaceSecondary(ReplaceSecondary { old_node_id: *secondary, - new_node_id: candidate_node, + new_node_id: replacement, }), }); } @@ -921,11 +1135,54 @@ impl TenantShard { self.intent.remove_secondary(scheduler, old_node_id); self.intent.push_secondary(scheduler, new_node_id); } + ScheduleOptimizationAction::CreateSecondary(new_node_id) => { + self.intent.push_secondary(scheduler, new_node_id); + } + ScheduleOptimizationAction::RemoveSecondary(old_secondary) => { + self.intent.remove_secondary(scheduler, old_secondary); + } } true } + /// When a shard has several secondary locations, we need to pick one in situations where + /// we promote one of them to an attached location: + /// - When draining a node for restart + /// - When responding to a node failure + /// + /// In this context, 'preferred' does not mean the node with the best scheduling score: instead + /// we want to pick the node which is best for use _temporarily_ while the previous attached location + /// is unavailable (e.g. because it's down or deploying). That means we prefer to use secondary + /// locations in a non-preferred AZ, as they're more likely to have awarm cache than a temporary + /// secondary in the preferred AZ (which are usually only created for migrations, and if they exist + /// they're probably not warmed up yet). The latter behavior is based oni + /// + /// If the input is empty, or all the nodes are not elegible for scheduling, return None: the + /// caller needs to a pick a node some other way. + pub(crate) fn preferred_secondary(&self, scheduler: &Scheduler) -> Option { + let candidates = scheduler.filter_usable_nodes(&self.intent.secondary); + + // We will sort candidates to prefer nodes which are _not_ in our preferred AZ, i.e. we prefer + // to migrate to a long-lived secondary location (which would have been scheduled in a non-preferred AZ), + // rather than a short-lived secondary location being used for optimization/migration (which would have + // been scheduled in our preferred AZ). + let mut candidates = candidates + .iter() + .map(|(node_id, node_az)| { + if node_az == &self.intent.preferred_az_id { + (1, *node_id) + } else { + (0, *node_id) + } + }) + .collect::>(); + + candidates.sort(); + + candidates.first().map(|i| i.1) + } + /// Query whether the tenant's observed state for attached node matches its intent state, and if so, /// yield the node ID. This is appropriate for emitting compute hook notifications: we are checking that /// the node in question is not only where we intend to attach, but that the tenant is indeed already attached there. @@ -1207,7 +1464,7 @@ impl TenantShard { detach, reconciler_config, config: self.config.clone(), - preferred_az: self.preferred_az_id.clone(), + preferred_az: self.intent.preferred_az_id.clone(), observed: self.observed.clone(), original_observed: self.observed.clone(), compute_hook: compute_hook.clone(), @@ -1428,7 +1685,6 @@ impl TenantShard { pending_compute_notification: false, delayed_reconcile: false, scheduling_policy: serde_json::from_str(&tsp.scheduling_policy).unwrap(), - preferred_az_id: tsp.preferred_az_id.map(AvailabilityZone), }) } @@ -1444,16 +1700,16 @@ impl TenantShard { config: serde_json::to_string(&self.config).unwrap(), splitting: SplitState::default(), scheduling_policy: serde_json::to_string(&self.scheduling_policy).unwrap(), - preferred_az_id: self.preferred_az_id.as_ref().map(|az| az.0.clone()), + preferred_az_id: self.intent.preferred_az_id.as_ref().map(|az| az.0.clone()), } } pub(crate) fn preferred_az(&self) -> Option<&AvailabilityZone> { - self.preferred_az_id.as_ref() + self.intent.preferred_az_id.as_ref() } pub(crate) fn set_preferred_az(&mut self, preferred_az_id: AvailabilityZone) { - self.preferred_az_id = Some(preferred_az_id); + self.intent.preferred_az_id = Some(preferred_az_id); } /// Returns all the nodes to which this tenant shard is attached according to the @@ -1756,65 +2012,90 @@ pub(crate) mod tests { } #[test] - fn optimize_attachment() -> anyhow::Result<()> { - let nodes = make_test_nodes(3, &[]); + /// Simple case: moving attachment to somewhere better where we already have a secondary + fn optimize_attachment_simple() -> anyhow::Result<()> { + let nodes = make_test_nodes( + 3, + &[ + AvailabilityZone("az-a".to_string()), + AvailabilityZone("az-b".to_string()), + AvailabilityZone("az-c".to_string()), + ], + ); let mut scheduler = Scheduler::new(nodes.values()); let mut shard_a = make_test_tenant_shard(PlacementPolicy::Attached(1)); + shard_a.intent.preferred_az_id = Some(AvailabilityZone("az-a".to_string())); let mut shard_b = make_test_tenant_shard(PlacementPolicy::Attached(1)); + shard_b.intent.preferred_az_id = Some(AvailabilityZone("az-a".to_string())); // Initially: both nodes attached on shard 1, and both have secondary locations // on different nodes. - shard_a.intent.set_attached(&mut scheduler, Some(NodeId(1))); - shard_a.intent.push_secondary(&mut scheduler, NodeId(2)); + shard_a.intent.set_attached(&mut scheduler, Some(NodeId(2))); + shard_a.intent.push_secondary(&mut scheduler, NodeId(1)); shard_b.intent.set_attached(&mut scheduler, Some(NodeId(1))); - shard_b.intent.push_secondary(&mut scheduler, NodeId(3)); + shard_b.intent.push_secondary(&mut scheduler, NodeId(2)); - let mut schedule_context = ScheduleContext::default(); - schedule_context.avoid(&shard_a.intent.all_pageservers()); - schedule_context.push_attached(shard_a.intent.get_attached().unwrap()); - schedule_context.avoid(&shard_b.intent.all_pageservers()); - schedule_context.push_attached(shard_b.intent.get_attached().unwrap()); + fn make_schedule_context(shard_a: &TenantShard, shard_b: &TenantShard) -> ScheduleContext { + let mut schedule_context = ScheduleContext::default(); + schedule_context.avoid(&shard_a.intent.all_pageservers()); + schedule_context.avoid(&shard_b.intent.all_pageservers()); + schedule_context + } - let optimization_a = shard_a.optimize_attachment(&nodes, &schedule_context); - - // Either shard should recognize that it has the option to switch to a secondary location where there - // would be no other shards from the same tenant, and request to do so. + let schedule_context = make_schedule_context(&shard_a, &shard_b); + let optimization_a = shard_a.optimize_attachment(&mut scheduler, &schedule_context); assert_eq!( optimization_a, Some(ScheduleOptimization { sequence: shard_a.sequence, action: ScheduleOptimizationAction::MigrateAttachment(MigrateAttachment { - old_attached_node_id: NodeId(1), - new_attached_node_id: NodeId(2) + old_attached_node_id: NodeId(2), + new_attached_node_id: NodeId(1) }) }) ); - - // Note that these optimizing two shards in the same tenant with the same ScheduleContext is - // mutually exclusive (the optimization of one invalidates the stats) -- it is the responsibility - // of [`Service::optimize_all`] to avoid trying - // to do optimizations for multiple shards in the same tenant at the same time. Generating - // both optimizations is just done for test purposes - let optimization_b = shard_b.optimize_attachment(&nodes, &schedule_context); - assert_eq!( - optimization_b, - Some(ScheduleOptimization { - sequence: shard_b.sequence, - action: ScheduleOptimizationAction::MigrateAttachment(MigrateAttachment { - old_attached_node_id: NodeId(1), - new_attached_node_id: NodeId(3) - }) - }) - ); - - // Applying these optimizations should result in the end state proposed shard_a.apply_optimization(&mut scheduler, optimization_a.unwrap()); - assert_eq!(shard_a.intent.get_attached(), &Some(NodeId(2))); - assert_eq!(shard_a.intent.get_secondary(), &vec![NodeId(1)]); - shard_b.apply_optimization(&mut scheduler, optimization_b.unwrap()); - assert_eq!(shard_b.intent.get_attached(), &Some(NodeId(3))); - assert_eq!(shard_b.intent.get_secondary(), &vec![NodeId(1)]); + + // // Either shard should recognize that it has the option to switch to a secondary location where there + // // would be no other shards from the same tenant, and request to do so. + // assert_eq!( + // optimization_a_prepare, + // Some(ScheduleOptimization { + // sequence: shard_a.sequence, + // action: ScheduleOptimizationAction::CreateSecondary(NodeId(2)) + // }) + // ); + // shard_a.apply_optimization(&mut scheduler, optimization_a_prepare.unwrap()); + + // let schedule_context = make_schedule_context(&shard_a, &shard_b); + // let optimization_a_migrate = shard_a.optimize_attachment(&mut scheduler, &schedule_context); + // assert_eq!( + // optimization_a_migrate, + // Some(ScheduleOptimization { + // sequence: shard_a.sequence, + // action: ScheduleOptimizationAction::MigrateAttachment(MigrateAttachment { + // old_attached_node_id: NodeId(1), + // new_attached_node_id: NodeId(2) + // }) + // }) + // ); + // shard_a.apply_optimization(&mut scheduler, optimization_a_migrate.unwrap()); + + // let schedule_context = make_schedule_context(&shard_a, &shard_b); + // let optimization_a_cleanup = shard_a.optimize_attachment(&mut scheduler, &schedule_context); + // assert_eq!( + // optimization_a_cleanup, + // Some(ScheduleOptimization { + // sequence: shard_a.sequence, + // action: ScheduleOptimizationAction::RemoveSecondary(NodeId(1)) + // }) + // ); + // shard_a.apply_optimization(&mut scheduler, optimization_a_cleanup.unwrap()); + + // // Shard B should not be moved anywhere, since the pressure on node 1 was relieved by moving shard A + // let schedule_context = make_schedule_context(&shard_a, &shard_b); + // assert_eq!(shard_b.optimize_attachment(&mut scheduler, &schedule_context), None); shard_a.intent.clear(&mut scheduler); shard_b.intent.clear(&mut scheduler); @@ -1822,6 +2103,190 @@ pub(crate) mod tests { Ok(()) } + #[test] + /// Complicated case: moving attachment to somewhere better where we do not have a secondary + /// already, creating one as needed. + fn optimize_attachment_multistep() -> anyhow::Result<()> { + let nodes = make_test_nodes( + 3, + &[ + AvailabilityZone("az-a".to_string()), + AvailabilityZone("az-b".to_string()), + AvailabilityZone("az-c".to_string()), + ], + ); + let mut scheduler = Scheduler::new(nodes.values()); + + // Two shards of a tenant that wants to be in AZ A + let mut shard_a = make_test_tenant_shard(PlacementPolicy::Attached(1)); + shard_a.intent.preferred_az_id = Some(AvailabilityZone("az-a".to_string())); + let mut shard_b = make_test_tenant_shard(PlacementPolicy::Attached(1)); + shard_b.intent.preferred_az_id = Some(AvailabilityZone("az-a".to_string())); + + // Both shards are initially attached in non-home AZ _and_ have secondaries in non-home AZs + shard_a.intent.set_attached(&mut scheduler, Some(NodeId(2))); + shard_a.intent.push_secondary(&mut scheduler, NodeId(3)); + shard_b.intent.set_attached(&mut scheduler, Some(NodeId(3))); + shard_b.intent.push_secondary(&mut scheduler, NodeId(2)); + + fn make_schedule_context(shard_a: &TenantShard, shard_b: &TenantShard) -> ScheduleContext { + let mut schedule_context = ScheduleContext::default(); + schedule_context.avoid(&shard_a.intent.all_pageservers()); + schedule_context.avoid(&shard_b.intent.all_pageservers()); + schedule_context + } + + let schedule_context = make_schedule_context(&shard_a, &shard_b); + let optimization_a_prepare = shard_a.optimize_attachment(&mut scheduler, &schedule_context); + assert_eq!( + optimization_a_prepare, + Some(ScheduleOptimization { + sequence: shard_a.sequence, + action: ScheduleOptimizationAction::CreateSecondary(NodeId(1)) + }) + ); + shard_a.apply_optimization(&mut scheduler, optimization_a_prepare.unwrap()); + + let schedule_context = make_schedule_context(&shard_a, &shard_b); + let optimization_a_migrate = shard_a.optimize_attachment(&mut scheduler, &schedule_context); + assert_eq!( + optimization_a_migrate, + Some(ScheduleOptimization { + sequence: shard_a.sequence, + action: ScheduleOptimizationAction::MigrateAttachment(MigrateAttachment { + old_attached_node_id: NodeId(2), + new_attached_node_id: NodeId(1) + }) + }) + ); + shard_a.apply_optimization(&mut scheduler, optimization_a_migrate.unwrap()); + + let schedule_context = make_schedule_context(&shard_a, &shard_b); + let optimization_a_cleanup = shard_a.optimize_attachment(&mut scheduler, &schedule_context); + assert_eq!( + optimization_a_cleanup, + Some(ScheduleOptimization { + sequence: shard_a.sequence, + action: ScheduleOptimizationAction::RemoveSecondary(NodeId(3)) + }) + ); + shard_a.apply_optimization(&mut scheduler, optimization_a_cleanup.unwrap()); + + // // Shard B should not be moved anywhere, since the pressure on node 1 was relieved by moving shard A + // let schedule_context = make_schedule_context(&shard_a, &shard_b); + // assert_eq!(shard_b.optimize_attachment(&mut scheduler, &schedule_context), None); + + shard_a.intent.clear(&mut scheduler); + shard_b.intent.clear(&mut scheduler); + + Ok(()) + } + + #[test] + /// Check that multi-step migration works when moving to somewhere that is only better by + /// 1 AffinityScore -- this ensures that we don't have a bug like the intermediate secondary + /// counting toward the affinity score such that it prevents the rest of the migration from happening. + fn optimize_attachment_marginal() -> anyhow::Result<()> { + let nodes = make_test_nodes(2, &[]); + let mut scheduler = Scheduler::new(nodes.values()); + + // Multi-sharded tenant, we will craft a situation where affinity + // scores differ only slightly + let mut shards = make_test_tenant(PlacementPolicy::Attached(0), ShardCount::new(4), None); + + // 1 attached on node 1 + shards[0] + .intent + .set_attached(&mut scheduler, Some(NodeId(1))); + // 3 attached on node 2 + shards[1] + .intent + .set_attached(&mut scheduler, Some(NodeId(2))); + shards[2] + .intent + .set_attached(&mut scheduler, Some(NodeId(2))); + shards[3] + .intent + .set_attached(&mut scheduler, Some(NodeId(2))); + + // The scheduler should figure out that we need to: + // - Create a secondary for shard 3 on node 1 + // - Migrate shard 3 to node 1 + // - Remove shard 3's location on node 2 + + fn make_schedule_context(shards: &Vec) -> ScheduleContext { + let mut schedule_context = ScheduleContext::default(); + for shard in shards { + schedule_context.avoid(&shard.intent.all_pageservers()); + } + schedule_context + } + + let schedule_context = make_schedule_context(&shards); + let optimization_a_prepare = + shards[1].optimize_attachment(&mut scheduler, &schedule_context); + assert_eq!( + optimization_a_prepare, + Some(ScheduleOptimization { + sequence: shards[1].sequence, + action: ScheduleOptimizationAction::CreateSecondary(NodeId(1)) + }) + ); + shards[1].apply_optimization(&mut scheduler, optimization_a_prepare.unwrap()); + + let schedule_context = make_schedule_context(&shards); + let optimization_a_migrate = + shards[1].optimize_attachment(&mut scheduler, &schedule_context); + assert_eq!( + optimization_a_migrate, + Some(ScheduleOptimization { + sequence: shards[1].sequence, + action: ScheduleOptimizationAction::MigrateAttachment(MigrateAttachment { + old_attached_node_id: NodeId(2), + new_attached_node_id: NodeId(1) + }) + }) + ); + shards[1].apply_optimization(&mut scheduler, optimization_a_migrate.unwrap()); + + let schedule_context = make_schedule_context(&shards); + let optimization_a_cleanup = + shards[1].optimize_attachment(&mut scheduler, &schedule_context); + assert_eq!( + optimization_a_cleanup, + Some(ScheduleOptimization { + sequence: shards[1].sequence, + action: ScheduleOptimizationAction::RemoveSecondary(NodeId(2)) + }) + ); + shards[1].apply_optimization(&mut scheduler, optimization_a_cleanup.unwrap()); + + // Everything should be stable now + let schedule_context = make_schedule_context(&shards); + assert_eq!( + shards[0].optimize_attachment(&mut scheduler, &schedule_context), + None + ); + assert_eq!( + shards[1].optimize_attachment(&mut scheduler, &schedule_context), + None + ); + assert_eq!( + shards[2].optimize_attachment(&mut scheduler, &schedule_context), + None + ); + assert_eq!( + shards[3].optimize_attachment(&mut scheduler, &schedule_context), + None + ); + + for mut shard in shards { + shard.intent.clear(&mut scheduler); + } + + Ok(()) + } + #[test] fn optimize_secondary() -> anyhow::Result<()> { let nodes = make_test_nodes(4, &[]); @@ -1839,9 +2304,7 @@ pub(crate) mod tests { let mut schedule_context = ScheduleContext::default(); schedule_context.avoid(&shard_a.intent.all_pageservers()); - schedule_context.push_attached(shard_a.intent.get_attached().unwrap()); schedule_context.avoid(&shard_b.intent.all_pageservers()); - schedule_context.push_attached(shard_b.intent.get_attached().unwrap()); let optimization_a = shard_a.optimize_secondary(&mut scheduler, &schedule_context); @@ -1872,7 +2335,6 @@ pub(crate) mod tests { // called repeatedly in the background. // Returns the applied optimizations fn optimize_til_idle( - nodes: &HashMap, scheduler: &mut Scheduler, shards: &mut [TenantShard], ) -> Vec { @@ -1884,14 +2346,18 @@ pub(crate) mod tests { for shard in shards.iter() { schedule_context.avoid(&shard.intent.all_pageservers()); - if let Some(attached) = shard.intent.get_attached() { - schedule_context.push_attached(*attached); - } } for shard in shards.iter_mut() { - let optimization = shard.optimize_attachment(nodes, &schedule_context); + let optimization = shard.optimize_attachment(scheduler, &schedule_context); + tracing::info!( + "optimize_attachment({})={:?}", + shard.tenant_shard_id, + optimization + ); if let Some(optimization) = optimization { + // Check that maybe_optimizable wouldn't have wrongly claimed this optimization didn't exist + assert!(shard.maybe_optimizable(scheduler, &schedule_context)); optimizations.push(optimization.clone()); shard.apply_optimization(scheduler, optimization); any_changed = true; @@ -1899,7 +2365,15 @@ pub(crate) mod tests { } let optimization = shard.optimize_secondary(scheduler, &schedule_context); + tracing::info!( + "optimize_secondary({})={:?}", + shard.tenant_shard_id, + optimization + ); if let Some(optimization) = optimization { + // Check that maybe_optimizable wouldn't have wrongly claimed this optimization didn't exist + assert!(shard.maybe_optimizable(scheduler, &schedule_context)); + optimizations.push(optimization.clone()); shard.apply_optimization(scheduler, optimization); any_changed = true; @@ -1923,14 +2397,34 @@ pub(crate) mod tests { /// that it converges. #[test] fn optimize_add_nodes() -> anyhow::Result<()> { - let nodes = make_test_nodes(4, &[]); + let nodes = make_test_nodes( + 9, + &[ + // Initial 6 nodes + AvailabilityZone("az-a".to_string()), + AvailabilityZone("az-a".to_string()), + AvailabilityZone("az-b".to_string()), + AvailabilityZone("az-b".to_string()), + AvailabilityZone("az-c".to_string()), + AvailabilityZone("az-c".to_string()), + // Three we will add later + AvailabilityZone("az-a".to_string()), + AvailabilityZone("az-b".to_string()), + AvailabilityZone("az-c".to_string()), + ], + ); - // Only show the scheduler a couple of nodes + // Only show the scheduler two nodes in each AZ to start with let mut scheduler = Scheduler::new([].iter()); - scheduler.node_upsert(nodes.get(&NodeId(1)).unwrap()); - scheduler.node_upsert(nodes.get(&NodeId(2)).unwrap()); + for i in 1..=6 { + scheduler.node_upsert(nodes.get(&NodeId(i)).unwrap()); + } - let mut shards = make_test_tenant(PlacementPolicy::Attached(1), ShardCount::new(4), None); + let mut shards = make_test_tenant( + PlacementPolicy::Attached(1), + ShardCount::new(4), + Some(AvailabilityZone("az-a".to_string())), + ); let mut schedule_context = ScheduleContext::default(); for shard in &mut shards { assert!(shard @@ -1938,30 +2432,50 @@ pub(crate) mod tests { .is_ok()); } - // We should see equal number of locations on the two nodes. - assert_eq!(scheduler.get_node_shard_count(NodeId(1)), 4); + // Initial: attached locations land in the tenant's home AZ. + assert_eq!(scheduler.get_node_shard_count(NodeId(1)), 2); assert_eq!(scheduler.get_node_attached_shard_count(NodeId(1)), 2); - - assert_eq!(scheduler.get_node_shard_count(NodeId(2)), 4); + assert_eq!(scheduler.get_node_shard_count(NodeId(2)), 2); assert_eq!(scheduler.get_node_attached_shard_count(NodeId(2)), 2); - // Add another two nodes: we should see the shards spread out when their optimize - // methods are called - scheduler.node_upsert(nodes.get(&NodeId(3)).unwrap()); - scheduler.node_upsert(nodes.get(&NodeId(4)).unwrap()); - optimize_til_idle(&nodes, &mut scheduler, &mut shards); + // Initial: secondary locations in a remote AZ + assert_eq!(scheduler.get_node_shard_count(NodeId(3)), 1); + assert_eq!(scheduler.get_node_attached_shard_count(NodeId(3)), 0); + assert_eq!(scheduler.get_node_shard_count(NodeId(4)), 1); + assert_eq!(scheduler.get_node_attached_shard_count(NodeId(4)), 0); + assert_eq!(scheduler.get_node_shard_count(NodeId(5)), 1); + assert_eq!(scheduler.get_node_attached_shard_count(NodeId(5)), 0); + assert_eq!(scheduler.get_node_shard_count(NodeId(6)), 1); + assert_eq!(scheduler.get_node_attached_shard_count(NodeId(6)), 0); - assert_eq!(scheduler.get_node_shard_count(NodeId(1)), 2); + // Add another three nodes: we should see the shards spread out when their optimize + // methods are called + scheduler.node_upsert(nodes.get(&NodeId(7)).unwrap()); + scheduler.node_upsert(nodes.get(&NodeId(8)).unwrap()); + scheduler.node_upsert(nodes.get(&NodeId(9)).unwrap()); + optimize_til_idle(&mut scheduler, &mut shards); + + // We expect one attached location was moved to the new node in the tenant's home AZ + assert_eq!(scheduler.get_node_shard_count(NodeId(7)), 1); + assert_eq!(scheduler.get_node_attached_shard_count(NodeId(7)), 1); + // The original node has one less attached shard + assert_eq!(scheduler.get_node_shard_count(NodeId(1)), 1); assert_eq!(scheduler.get_node_attached_shard_count(NodeId(1)), 1); + // One of the original nodes still has two attachments, since there are an odd number of nodes assert_eq!(scheduler.get_node_shard_count(NodeId(2)), 2); - assert_eq!(scheduler.get_node_attached_shard_count(NodeId(2)), 1); + assert_eq!(scheduler.get_node_attached_shard_count(NodeId(2)), 2); - assert_eq!(scheduler.get_node_shard_count(NodeId(3)), 2); - assert_eq!(scheduler.get_node_attached_shard_count(NodeId(3)), 1); - - assert_eq!(scheduler.get_node_shard_count(NodeId(4)), 2); - assert_eq!(scheduler.get_node_attached_shard_count(NodeId(4)), 1); + // None of our secondaries moved, since we already had enough nodes for those to be + // scheduled perfectly + assert_eq!(scheduler.get_node_shard_count(NodeId(3)), 1); + assert_eq!(scheduler.get_node_attached_shard_count(NodeId(3)), 0); + assert_eq!(scheduler.get_node_shard_count(NodeId(4)), 1); + assert_eq!(scheduler.get_node_attached_shard_count(NodeId(4)), 0); + assert_eq!(scheduler.get_node_shard_count(NodeId(5)), 1); + assert_eq!(scheduler.get_node_attached_shard_count(NodeId(5)), 0); + assert_eq!(scheduler.get_node_shard_count(NodeId(6)), 1); + assert_eq!(scheduler.get_node_attached_shard_count(NodeId(6)), 0); for shard in shards.iter_mut() { shard.intent.clear(&mut scheduler); @@ -2001,10 +2515,10 @@ pub(crate) mod tests { shard.schedule(&mut scheduler, context).unwrap(); } - let applied_to_a = optimize_til_idle(&nodes, &mut scheduler, &mut a); + let applied_to_a = optimize_til_idle(&mut scheduler, &mut a); assert_eq!(applied_to_a, vec![]); - let applied_to_b = optimize_til_idle(&nodes, &mut scheduler, &mut b); + let applied_to_b = optimize_til_idle(&mut scheduler, &mut b); assert_eq!(applied_to_b, vec![]); for shard in a.iter_mut().chain(b.iter_mut()) { diff --git a/test_runner/performance/test_sharding_autosplit.py b/test_runner/performance/test_sharding_autosplit.py index caa89955e3..76c3ad01a4 100644 --- a/test_runner/performance/test_sharding_autosplit.py +++ b/test_runner/performance/test_sharding_autosplit.py @@ -2,6 +2,7 @@ from __future__ import annotations import concurrent.futures import re +import threading from pathlib import Path import pytest @@ -188,7 +189,20 @@ def test_sharding_autosplit(neon_env_builder: NeonEnvBuilder, pg_bin: PgBin): check_pgbench_output(out_path) - with concurrent.futures.ThreadPoolExecutor(max_workers=tenant_count) as pgbench_threads: + stop_pump = threading.Event() + + def pump_controller(): + # Run a background loop to force the storage controller to run its + # background work faster than it otherwise would: this helps + # us: + # A) to create a test that runs in a shorter time + # B) to create a test that is more intensive by doing the shard migrations + # after splits happen more rapidly. + while not stop_pump.is_set(): + env.storage_controller.reconcile_all() + stop_pump.wait(0.1) + + with concurrent.futures.ThreadPoolExecutor(max_workers=tenant_count + 1) as pgbench_threads: pgbench_futs = [] for tenant_state in tenants.values(): fut = pgbench_threads.submit(run_pgbench_init, tenant_state.endpoint) @@ -198,6 +212,8 @@ def test_sharding_autosplit(neon_env_builder: NeonEnvBuilder, pg_bin: PgBin): for fut in pgbench_futs: fut.result() + pump_fut = pgbench_threads.submit(pump_controller) + pgbench_futs = [] for tenant_state in tenants.values(): fut = pgbench_threads.submit(run_pgbench_main, tenant_state.endpoint) @@ -207,6 +223,9 @@ def test_sharding_autosplit(neon_env_builder: NeonEnvBuilder, pg_bin: PgBin): for fut in pgbench_futs: fut.result() + stop_pump.set() + pump_fut.result() + def assert_all_split(): for tenant_id in tenants.keys(): shards = tenant_get_shards(env, tenant_id) diff --git a/test_runner/performance/test_storage_controller_scale.py b/test_runner/performance/test_storage_controller_scale.py index 49f41483ec..d45db28c78 100644 --- a/test_runner/performance/test_storage_controller_scale.py +++ b/test_runner/performance/test_storage_controller_scale.py @@ -13,11 +13,13 @@ from fixtures.log_helper import log from fixtures.neon_fixtures import ( NeonEnv, NeonEnvBuilder, + NeonPageserver, PageserverAvailability, PageserverSchedulingPolicy, ) from fixtures.pageserver.http import PageserverApiException, PageserverHttpClient from fixtures.pg_version import PgVersion +from fixtures.utils import wait_until def get_consistent_node_shard_counts(env: NeonEnv, total_shards) -> defaultdict[str, int]: @@ -85,8 +87,12 @@ def test_storage_controller_many_tenants( ) AZS = ["alpha", "bravo", "charlie"] + + def az_selector(node_id): + return f"az-{AZS[(node_id - 1) % len(AZS)]}" + neon_env_builder.pageserver_config_override = lambda ps_cfg: ps_cfg.update( - {"availability_zone": f"az-{AZS[ps_cfg['id'] % len(AZS)]}"} + {"availability_zone": az_selector(ps_cfg["id"])} ) # A small sleep on each call into the notify hook, to simulate the latency of doing a database write @@ -168,6 +174,31 @@ def test_storage_controller_many_tenants( log.info(f"Resident memory: {rss} ({ rss / total_shards} per shard)") assert rss < expect_memory_per_shard * total_shards + def assert_all_tenants_scheduled_in_home_az(): + for tenant_id in tenant_ids: + desc = env.storage_controller.tenant_describe(tenant_id) + preferred_az = None + for shard in desc["shards"]: + # All shards in a tenant should have the same preferred AZ + if preferred_az is None: + preferred_az = shard["preferred_az_id"] + else: + assert preferred_az == shard["preferred_az_id"] + + # Attachment should be in the preferred AZ + assert shard["preferred_az_id"] == az_selector( + shard["node_attached"] + ), f"Shard {shard['tenant_shard_id']} not in {shard['preferred_az_id']}" + + # Secondary locations should not be in the preferred AZ + for node_secondary in shard["node_secondary"]: + assert ( + shard["preferred_az_id"] != az_selector(node_secondary) + ), f"Shard {shard['tenant_shard_id']} secondary should be in {shard['preferred_az_id']}" + + # There should only be one secondary location (i.e. no migrations in flight) + assert len(shard["node_secondary"]) == 1 + # Issue more concurrent operations than the storage controller's reconciler concurrency semaphore # permits, to ensure that we are exercising stressing that. api_concurrency = 135 @@ -242,6 +273,22 @@ def test_storage_controller_many_tenants( f"Created {len(tenants_with_timelines)} timelines in {time.time() - t1}, {len(tenants_with_timelines) / (time.time() - t1)}/s" ) + # Check initial scheduling + assert_all_tenants_scheduled_in_home_az() + az_attached_counts: defaultdict[str, int] = defaultdict(int) + az_secondary_counts: defaultdict[str, int] = defaultdict(int) + node_attached_counts: defaultdict[str, int] = defaultdict(int) + for tenant_id in tenants.keys(): + desc = env.storage_controller.tenant_describe(tenant_id) + for shard in desc["shards"]: + az_attached_counts[az_selector(shard["node_attached"])] += 1 + node_attached_counts[shard["node_attached"]] += 1 + for node_secondary in shard["node_secondary"]: + az_secondary_counts[az_selector(node_secondary)] += 1 + + log.info(f"Initial node attached counts: {node_attached_counts}") + log.info(f"Initial AZ shard counts: {az_attached_counts}, {az_secondary_counts}") + # Plan operations: ensure each tenant with a timeline gets at least # one of each operation type. Then add other tenants to make up the # numbers. @@ -450,11 +497,77 @@ def test_storage_controller_many_tenants( env.storage_controller.reconcile_until_idle(max_interval=0.1, timeout_secs=120) env.storage_controller.consistency_check() + # Since we did `reconcile_until_idle` during the above loop, the system should be left in + # an optimally scheduled state. Validate that this includes all the tenants being scheduled + # in their home AZ. + assert_all_tenants_scheduled_in_home_az() + # Consistency check is safe here: restarting pageservers should not have caused any Reconcilers to spawn, # as they were not offline long enough to trigger any scheduling changes. env.storage_controller.consistency_check() check_memory() + # Simulate loss of an AZ + victim_az = "az-alpha" + killed_pageservers = [] + for ps in env.pageservers: + if az_selector(ps.id) == victim_az: + ps.stop(immediate=True) + killed_pageservers.append(ps) + log.info(f"Killed pageserver {ps.id}") + + assert killed_pageservers + + # Wait for the controller to notice the pageservers are dead + def assert_pageservers_availability( + pageservers: list[NeonPageserver], expected_availability: PageserverAvailability + ): + nodes = env.storage_controller.nodes() + checked_any = False + node_ids = [ps.id for ps in pageservers] + for node in nodes: + if node["id"] in node_ids: + checked_any = True + assert ( + node["availability"] == expected_availability + ), f"Node {node['id']} is not {expected_availability} yet: {node['availability']}" + + assert checked_any + + wait_until( + lambda: assert_pageservers_availability(killed_pageservers, PageserverAvailability.OFFLINE), + timeout=60, + ) + + # Let the controller finish all its rescheduling + env.storage_controller.reconcile_until_idle(max_interval=0.1, timeout_secs=120) + + # Check that all the tenants are rescheduled to the remaining pageservers + for tenant_id in tenant_ids: + desc = env.storage_controller.tenant_describe(tenant_id) + for shard in desc["shards"]: + # Attachment should be outside the AZ where we killed the pageservers + assert ( + az_selector(shard["node_attached"]) != victim_az + ), f"Shard {shard['tenant_shard_id']} still in {victim_az} (node {shard['node_attached']})" + + # Bring back the pageservers + for ps in killed_pageservers: + ps.start() + + wait_until( + lambda: assert_pageservers_availability(killed_pageservers, PageserverAvailability.ACTIVE), + timeout=60, + ) + + # A very long timeout is required: we will be migrating all the tenants on all the pageservers + # in the region that we just restored. Assume it'll take up to twice as long as it took to fill + # a single node + env.storage_controller.reconcile_until_idle( + max_interval=0.1, timeout_secs=DRAIN_FILL_TIMEOUT * 4 + ) + assert_all_tenants_scheduled_in_home_az() + # Stop the storage controller before tearing down fixtures, because it otherwise might log # errors trying to call our `ComputeReconfigure`. env.storage_controller.stop() diff --git a/test_runner/regress/test_sharding.py b/test_runner/regress/test_sharding.py index 673904a1cd..86a6b7428b 100644 --- a/test_runner/regress/test_sharding.py +++ b/test_runner/regress/test_sharding.py @@ -520,14 +520,18 @@ def test_sharding_split_smoke( shard_count = 2 # Shard count we split into split_shard_count = 4 - # We will have 2 shards per pageserver once done (including secondaries) - neon_env_builder.num_pageservers = split_shard_count + # In preferred AZ & other AZ we will end up with one shard per pageserver + neon_env_builder.num_pageservers = split_shard_count * 2 # Two AZs def assign_az(ps_cfg): az = f"az-{(ps_cfg['id'] - 1) % 2}" ps_cfg["availability_zone"] = az + # We will run more pageservers than tests usually do, so give them tiny page caches + # in case we're on a test node under memory pressure. + ps_cfg["page_cache_size"] = 128 + neon_env_builder.pageserver_config_override = assign_az # 1MiB stripes: enable getting some meaningful data distribution without @@ -679,8 +683,8 @@ def test_sharding_split_smoke( # - shard_count reconciles for the original setup of the tenant # - shard_count reconciles for detaching the original secondary locations during split # - split_shard_count reconciles during shard splitting, for setting up secondaries. - # - split_shard_count/2 of the child shards will need to fail over to their secondaries (since we have 8 shards and 4 pageservers, only 4 will move) - expect_reconciles = shard_count * 2 + split_shard_count + split_shard_count / 2 + # - split_shard_count/2 reconciles to migrate shards to their temporary secondaries + expect_reconciles = shard_count * 2 + split_shard_count + 3 * (split_shard_count / 2) reconcile_ok = env.storage_controller.get_metric_value( "storage_controller_reconcile_complete_total", filter={"status": "ok"} @@ -745,10 +749,14 @@ def test_sharding_split_smoke( # dominated by shard count. log.info(f"total: {total}") assert total == { - 1: 2, - 2: 2, - 3: 2, - 4: 2, + 1: 1, + 2: 1, + 3: 1, + 4: 1, + 5: 1, + 6: 1, + 7: 1, + 8: 1, } # The controller is not required to lay out the attached locations in any particular way, but @@ -1387,13 +1395,7 @@ def test_sharding_split_failures( else: attached_count += 1 - if exclude_ps_id is not None: - # For a node failure case, we expect there to be a secondary location - # scheduled on the offline node, so expect one fewer secondary in total - assert secondary_count == initial_shard_count - 1 - else: - assert secondary_count == initial_shard_count - + assert secondary_count == initial_shard_count assert attached_count == initial_shard_count def assert_split_done(exclude_ps_id: int | None = None) -> None: diff --git a/test_runner/regress/test_storage_controller.py b/test_runner/regress/test_storage_controller.py index 3a55e75589..8ffb6ba6b2 100644 --- a/test_runner/regress/test_storage_controller.py +++ b/test_runner/regress/test_storage_controller.py @@ -3213,11 +3213,12 @@ def eq_safekeeper_records(a: dict[str, Any], b: dict[str, Any]) -> bool: @run_only_on_default_postgres("this is like a 'unit test' against storcon db") def test_shard_preferred_azs(neon_env_builder: NeonEnvBuilder): def assign_az(ps_cfg): - az = f"az-{ps_cfg['id']}" + az = f"az-{ps_cfg['id'] % 2}" + log.info("Assigned AZ {az}") ps_cfg["availability_zone"] = az neon_env_builder.pageserver_config_override = assign_az - neon_env_builder.num_pageservers = 2 + neon_env_builder.num_pageservers = 4 env = neon_env_builder.init_configs() env.start() @@ -3232,8 +3233,14 @@ def test_shard_preferred_azs(neon_env_builder: NeonEnvBuilder): assert shards[0]["preferred_az_id"] == expected_az + # When all other schedule scoring parameters are equal, tenants should round-robin on AZs + assert env.storage_controller.tenant_describe(tids[0])["shards"][0]["preferred_az_id"] == "az-0" + assert env.storage_controller.tenant_describe(tids[1])["shards"][0]["preferred_az_id"] == "az-1" + assert env.storage_controller.tenant_describe(tids[2])["shards"][0]["preferred_az_id"] == "az-0" + + # Try modifying preferred AZ updated = env.storage_controller.set_preferred_azs( - {TenantShardId(tid, 0, 0): "foo" for tid in tids} + {TenantShardId(tid, 0, 0): "az-0" for tid in tids} ) assert set(updated) == set([TenantShardId(tid, 0, 0) for tid in tids]) @@ -3241,29 +3248,24 @@ def test_shard_preferred_azs(neon_env_builder: NeonEnvBuilder): for tid in tids: shards = env.storage_controller.tenant_describe(tid)["shards"] assert len(shards) == 1 - assert shards[0]["preferred_az_id"] == "foo" + assert shards[0]["preferred_az_id"] == "az-0" - # Generate a layer to avoid shard split handling on ps from tripping - # up on debug assert. - timeline_id = TimelineId.generate() - env.create_timeline("bar", tids[0], timeline_id) - - workload = Workload(env, tids[0], timeline_id, branch_name="bar") - workload.init() - workload.write_rows(256) - workload.validate() + # Having modified preferred AZ, we should get moved there + env.storage_controller.reconcile_until_idle(max_interval=0.1) + for tid in tids: + shard = env.storage_controller.tenant_describe(tid)["shards"][0] + attached_to = shard["node_attached"] + attached_in_az = env.get_pageserver(attached_to).az_id + assert shard["preferred_az_id"] == attached_in_az == "az-0" env.storage_controller.tenant_shard_split(tids[0], shard_count=2) + env.storage_controller.reconcile_until_idle(max_interval=0.1) shards = env.storage_controller.tenant_describe(tids[0])["shards"] assert len(shards) == 2 for shard in shards: attached_to = shard["node_attached"] - expected_az = env.get_pageserver(attached_to).az_id - - # The scheduling optimization logic is not yet AZ-aware, so doesn't succeed - # in putting the tenant shards in the preferred AZ. - # To be fixed in https://github.com/neondatabase/neon/pull/9916 - # assert shard["preferred_az_id"] == expected_az + attached_in_az = env.get_pageserver(attached_to).az_id + assert shard["preferred_az_id"] == attached_in_az == "az-0" @run_only_on_default_postgres("Postgres version makes no difference here") From 1783501eaa12519cd94c174114d2e79381ffee2d Mon Sep 17 00:00:00 2001 From: Konstantin Knizhnik Date: Mon, 13 Jan 2025 22:01:03 +0200 Subject: [PATCH 066/111] Increase max connection for replica to prevent test flukyness (#10306) ## Problem See https://github.com/neondatabase/neon/issues/10167 Too small number of `max_connections` (2) can cause failures of test_physical_replication_config_mismatch_too_many_known_xids test ## Summary of changes Increase `max_connections` to 5 Co-authored-by: Konstantin Knizhnik --- test_runner/regress/test_physical_replication.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_runner/regress/test_physical_replication.py b/test_runner/regress/test_physical_replication.py index 6cb11b825d..17819fd367 100644 --- a/test_runner/regress/test_physical_replication.py +++ b/test_runner/regress/test_physical_replication.py @@ -187,7 +187,7 @@ def test_physical_replication_config_mismatch_too_many_known_xids(neon_simple_en origin=primary, endpoint_id="secondary", config_lines=[ - "max_connections=2", + "max_connections=5", "autovacuum_max_workers=1", "max_worker_processes=5", "max_wal_senders=1", From 430b556b3472007341f57af57ebdbafb67e6dc85 Mon Sep 17 00:00:00 2001 From: Tristan Partin Date: Mon, 13 Jan 2025 18:44:39 -0600 Subject: [PATCH 067/111] Update postgres-exporter and sql_exporter in computes (#10349) The postgres-exporter was much further out of date, but let's just bump both. Signed-off-by: Tristan Partin --- build-tools.Dockerfile | 2 +- compute/compute-node.Dockerfile | 4 ++-- test_runner/regress/test_compute_metrics.py | 9 +++------ 3 files changed, 6 insertions(+), 9 deletions(-) diff --git a/build-tools.Dockerfile b/build-tools.Dockerfile index cf6439d004..7a2ec9c43e 100644 --- a/build-tools.Dockerfile +++ b/build-tools.Dockerfile @@ -115,7 +115,7 @@ RUN set -e \ # Keep the version the same as in compute/compute-node.Dockerfile and # test_runner/regress/test_compute_metrics.py. -ENV SQL_EXPORTER_VERSION=0.16.0 +ENV SQL_EXPORTER_VERSION=0.17.0 RUN curl -fsSL \ "https://github.com/burningalchemist/sql_exporter/releases/download/${SQL_EXPORTER_VERSION}/sql_exporter-${SQL_EXPORTER_VERSION}.linux-$(case "$(uname -m)" in x86_64) echo amd64;; aarch64) echo arm64;; esac).tar.gz" \ --output sql_exporter.tar.gz \ diff --git a/compute/compute-node.Dockerfile b/compute/compute-node.Dockerfile index 2d38796d77..89cee6761f 100644 --- a/compute/compute-node.Dockerfile +++ b/compute/compute-node.Dockerfile @@ -1274,11 +1274,11 @@ RUN set -e \ # ######################################################################################### -FROM quay.io/prometheuscommunity/postgres-exporter:v0.12.1 AS postgres-exporter +FROM quay.io/prometheuscommunity/postgres-exporter:v0.16.0 AS postgres-exporter # Keep the version the same as in build-tools.Dockerfile and # test_runner/regress/test_compute_metrics.py. -FROM burningalchemist/sql_exporter:0.16.0 AS sql-exporter +FROM burningalchemist/sql_exporter:0.17.0 AS sql-exporter ######################################################################################### # diff --git a/test_runner/regress/test_compute_metrics.py b/test_runner/regress/test_compute_metrics.py index 71963355b7..5dcc93acff 100644 --- a/test_runner/regress/test_compute_metrics.py +++ b/test_runner/regress/test_compute_metrics.py @@ -219,7 +219,7 @@ if SQL_EXPORTER is None: # # The "host" network mode allows sql_exporter to talk to the # endpoint which is running on the host. - super().__init__("docker.io/burningalchemist/sql_exporter:0.16.0", network_mode="host") + super().__init__("docker.io/burningalchemist/sql_exporter:0.17.0", network_mode="host") self.__logs_dir = logs_dir self.__port = port @@ -252,7 +252,7 @@ if SQL_EXPORTER is None: log.info("Waiting for sql_exporter to be ready") wait_for_logs( self, - rf'level=info msg="Listening on" address=\[::\]:{self.__port}', + rf'msg="Listening on" address=\[::\]:{self.__port}', timeout=5, ) @@ -344,10 +344,7 @@ else: time.sleep(0.5) continue - if ( - f'level=info msg="Listening on" address=[::]:{self._sql_exporter_port}' - in line - ): + if f'msg="Listening on" address=[::]:{self._sql_exporter_port}' in line: break @override From a039f8381f83547dec0d774b3598b8e4b75abf3a Mon Sep 17 00:00:00 2001 From: Konstantin Knizhnik Date: Tue, 14 Jan 2025 07:54:30 +0200 Subject: [PATCH 068/111] Optimize vector get last written LSN (#10360) ## Problem See https://github.com/neondatabase/neon/issues/10281 pg17 performs extra lock/unlock operation when fetching LwLSN. ## Summary of changes Perform all lookups under one lock, moving initialization of not found keys to separate loop. Related Postgres PR: https://github.com/neondatabase/postgres/pull/553 --------- Co-authored-by: Konstantin Knizhnik --- vendor/postgres-v17 | 2 +- vendor/revisions.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/vendor/postgres-v17 b/vendor/postgres-v17 index 9c9e9a78a9..0f8da73ed0 160000 --- a/vendor/postgres-v17 +++ b/vendor/postgres-v17 @@ -1 +1 @@ -Subproject commit 9c9e9a78a93aebec2f6a2f54644442d35ffa245c +Subproject commit 0f8da73ed08d4fc4ee58cccea008c75bfb20baa8 diff --git a/vendor/revisions.json b/vendor/revisions.json index d182b88008..b4d57ab709 100644 --- a/vendor/revisions.json +++ b/vendor/revisions.json @@ -1,7 +1,7 @@ { "v17": [ "17.2", - "9c9e9a78a93aebec2f6a2f54644442d35ffa245c" + "0f8da73ed08d4fc4ee58cccea008c75bfb20baa8" ], "v16": [ "16.6", From 6007a94f91f75cdda0cc1d030e09888a108e4c20 Mon Sep 17 00:00:00 2001 From: Christian Schwarz Date: Tue, 14 Jan 2025 11:29:30 +0100 Subject: [PATCH 069/111] it compiles --- pageserver/src/page_service.rs | 4 +- pageserver/src/tenant/timeline/handle.rs | 138 +++++++++-------------- 2 files changed, 55 insertions(+), 87 deletions(-) diff --git a/pageserver/src/page_service.rs b/pageserver/src/page_service.rs index cf2276f65e..714410baf4 100644 --- a/pageserver/src/page_service.rs +++ b/pageserver/src/page_service.rs @@ -907,9 +907,7 @@ impl PageServerHandler { assert_eq!(accum_pages.len(), max_batch_size.get()); return false; } - if (accum_shard.tenant_shard_id, accum_shard.timeline_id) - != (this_shard.tenant_shard_id, this_shard.timeline_id) - { + if !Arc::ptr_eq(accum_shard.timeline(), this_shard.timeline()) { trace!(%accum_lsn, %this_lsn, "stopping batching because timeline object mismatch"); // TODO: we _could_ batch & execute each shard seperately (and in parallel). // But the current logic for keeping responses in order does not support that. diff --git a/pageserver/src/tenant/timeline/handle.rs b/pageserver/src/tenant/timeline/handle.rs index 983eae89ca..2d48890aa9 100644 --- a/pageserver/src/tenant/timeline/handle.rs +++ b/pageserver/src/tenant/timeline/handle.rs @@ -113,8 +113,8 @@ use std::sync::Arc; use std::sync::Mutex; use std::sync::Weak; -use futures::lock::OwnedMutexGuard; use pageserver_api::shard::ShardIdentity; +use tokio::sync::OwnedMutexGuard; use tracing::instrument; use tracing::trace; use utils::id::TimelineId; @@ -172,11 +172,10 @@ pub(crate) struct ShardTimelineId { } /// See module-level comment. -pub(crate) struct Handle(tokio::sync::OwnedMutexGuard>); -pub(crate) struct WeakHandle(Arc>>); -enum HandleInner { +pub(crate) struct Handle(T::Timeline, tokio::sync::OwnedMutexGuard); +pub(crate) struct WeakHandle(T::Timeline, Arc>); +enum HandleInner { KeepingTimelineGateOpen { - timeline: T::Timeline, gate_guard: utils::sync::gate::GateGuard, }, ShutDown, @@ -187,7 +186,7 @@ enum HandleInner { /// See module-level comment for details. pub struct PerTimelineState { // None = shutting down - handles: Mutex>>>>>, + handles: Mutex>)>>>, } impl Default for PerTimelineState { @@ -247,24 +246,8 @@ impl Cache { shard_selector: ShardSelector, tenant_manager: &T::TenantManager, ) -> Result, GetError> { - // terminates because each iteration removes an element from the map - loop { - let handle = self - .get_impl(timeline_id, shard_selector, tenant_manager) - .await?; - if handle.0.check_shut_down().is_none() { - let removed = self - .map - .remove(&handle.0.timeline.shard_timeline_id()) - .expect("invariant of get_impl is that the returned handle is in the map"); - assert!( - Weak::ptr_eq(&removed, &Arc::downgrade(&handle.0)), - "shard_timeline_id() incorrect?" - ); - } else { - return Ok(handle); - } - } + self.get_impl(timeline_id, shard_selector, tenant_manager) + .await } #[instrument(level = "trace", skip_all)] @@ -275,13 +258,14 @@ impl Cache { tenant_manager: &T::TenantManager, ) -> Result, GetError> { let miss: ShardSelector = { - let routing_state = self.shard_routing(timeline_id, shard_selector); + let routing_state = self.shard_routing(timeline_id, shard_selector).await; match routing_state { RoutingResult::FastPath(handle) => return Ok(handle), RoutingResult::SlowPath(key) => match self.map.get(&key) { - Some(cached) => match cached.upgrade() { - Some(upgraded) => return Ok(Handle(upgraded)), - None => { + Some(cached) => match cached.upgrade().await { + Ok(upgraded) => return Ok(upgraded), + Err(HandleUpgradeError::ShutDown) => { + // TODO: dedup with shard_routing() trace!("handle cache stale"); self.map.remove(&key).unwrap(); ShardSelector::Known(key.shard_index) @@ -296,7 +280,7 @@ impl Cache { } #[inline(always)] - fn shard_routing( + async fn shard_routing( &mut self, timeline_id: TimelineId, shard_selector: ShardSelector, @@ -306,15 +290,15 @@ impl Cache { let Some((first_key, first_handle)) = self.map.iter().next() else { return RoutingResult::NeedConsultTenantManager; }; - let Some(first_handle) = first_handle.upgrade() else { - // TODO: dedup with get() + let Ok(first_handle) = first_handle.upgrade().await else { + // TODO: dedup with get_impl() trace!("handle cache stale"); let first_key_owned = *first_key; self.map.remove(&first_key_owned).unwrap(); continue; }; - let first_handle_shard_identity = first_handle.timeline.get_shard_identity(); + let first_handle_shard_identity = first_handle.get_shard_identity(); let make_shard_index = |shard_num: ShardNumber| ShardIndex { shard_number: shard_num, shard_count: first_handle_shard_identity.count, @@ -333,11 +317,11 @@ impl Cache { }; let first_handle_shard_timeline_id = ShardTimelineId { shard_index: first_handle_shard_identity.shard_index(), - timeline_id: first_handle.timeline.shard_timeline_id().timeline_id, + timeline_id: first_handle.shard_timeline_id().timeline_id, }; if need_shard_timeline_id == first_handle_shard_timeline_id { - return RoutingResult::FastPath(Handle(first_handle)); + return RoutingResult::FastPath(first_handle); } else { return RoutingResult::SlowPath(need_shard_timeline_id); } @@ -368,15 +352,13 @@ impl Cache { } }; trace!("creating new HandleInner"); - let handle = Arc::new( + let handle_inner_arc = Arc::new(tokio::sync::Mutex::new( // TODO: global metric that keeps track of the number of live HandlerTimeline instances // so we can identify reference cycle bugs. - HandleInner::KeepingTimelineGateOpen { - timeline: timeline.clone(), - gate_guard, - }, - ); - let handle = { + HandleInner::KeepingTimelineGateOpen { gate_guard }, + )); + let handle_locked = handle_inner_arc.clone().try_lock_owned().unwrap(); + { let mut lock_guard = timeline .per_timeline_state() .handles @@ -384,7 +366,8 @@ impl Cache { .expect("mutex poisoned"); match &mut *lock_guard { Some(per_timeline_state) => { - let replaced = per_timeline_state.insert(self.id, Arc::clone(&handle)); + let replaced = per_timeline_state + .insert(self.id, (timeline.clone(), Arc::clone(&handle_inner_arc))); assert!(replaced.is_none(), "some earlier code left a stale handle"); match self.map.entry(key) { hash_map::Entry::Occupied(_o) => { @@ -395,8 +378,10 @@ impl Cache { unreachable!() } hash_map::Entry::Vacant(v) => { - v.insert(Arc::downgrade(&handle)); - handle + v.insert(WeakHandle( + timeline.clone(), + Arc::clone(&handle_inner_arc), + )); } } } @@ -404,8 +389,8 @@ impl Cache { return Err(GetError::PerTimelineStateShutDown); } } - }; - Ok(Handle(handle)) + } + Ok(Handle(timeline.clone(), handle_locked)) } Err(e) => Err(GetError::TenantManager(e)), } @@ -418,31 +403,41 @@ pub(crate) enum HandleUpgradeError { impl WeakHandle { pub(crate) async fn upgrade(&self) -> Result, HandleUpgradeError> { - let lock_guard = self.0.lock_owned().await; + let lock_guard = self.1.clone().lock_owned().await; match &*lock_guard { - HandleInner::KeepingTimelineGateOpen { - timeline, - gate_guard, - } => Ok(Handle(lock_guard)), + HandleInner::KeepingTimelineGateOpen { gate_guard } => { + Ok(Handle(self.0.clone(), lock_guard)) + } HandleInner::ShutDown => Err(HandleUpgradeError::ShutDown), } } + pub(crate) fn timeline(&self) -> &T::Timeline { + &self.0 + } } impl Deref for Handle { type Target = T::Timeline; fn deref(&self) -> &Self::Target { - match &*self.0 { - HandleInner::KeepingTimelineGateOpen { timeline, .. } => timeline, - HandleInner::ShutDown => unreachable!(), - } + &self.0 } } impl Handle { pub(crate) fn downgrade(&self) -> WeakHandle { - WeakHandle(Arc::clone(&self.0.mutex())) + WeakHandle(self.0.clone(), Arc::clone(OwnedMutexGuard::mutex(&self.1))) + } +} + +impl HandleInner { + pub(crate) fn shutdown(&mut self) { + match self { + HandleInner::KeepingTimelineGateOpen { .. } => *self = HandleInner::ShutDown, + HandleInner::ShutDown => { + unreachable!("handles are only shut down once in their lifetime"); + } + } } } @@ -465,40 +460,15 @@ impl PerTimelineState { trace!("already shut down"); return; }; - for handle in handles.values() { + for (_, state) in handles.values().map(|h| &*h) { // Make further cache hits - let mut lock_guard = handle.lock().await; - match std::mem::replace(&mut *lock_guard, HandleInner::ShutDown) { - HandleInner::KeepingTimelineGateOpen { gate_guard, .. } => { - drop(gate_guard); - } - HandleInner::ShutDown => unreachable!(), - } + let mut lock_guard = state.lock().await; + lock_guard.shutdown(); } drop(handles); } } -// When dropping a [`Cache`], prune its handles in the [`PerTimelineState`] to break the reference cycle. -impl Drop for Cache { - fn drop(&mut self) { - for (_, weak) in self.map.drain() { - if let Some(strong) = weak.upgrade() { - // handle is still being kept alive in PerTimelineState - let timeline = strong.timeline.per_timeline_state(); - let mut handles = timeline.handles.lock().expect("mutex poisoned"); - if let Some(handles) = &mut *handles { - let Some(removed) = handles.remove(&self.id) else { - // There could have been a shutdown inbetween us upgrading the weak and locking the mutex. - continue; - }; - assert!(Arc::ptr_eq(&removed, &strong)); - } - } - } - } -} - #[cfg(test)] mod tests { use pageserver_api::{ From 3fefa5b415c6134f72e201931c019f16ba0ddfc8 Mon Sep 17 00:00:00 2001 From: Christian Schwarz Date: Tue, 14 Jan 2025 11:31:58 +0100 Subject: [PATCH 070/111] fix warnings --- pageserver/src/tenant/timeline/handle.rs | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/pageserver/src/tenant/timeline/handle.rs b/pageserver/src/tenant/timeline/handle.rs index 2d48890aa9..1e8def0aa1 100644 --- a/pageserver/src/tenant/timeline/handle.rs +++ b/pageserver/src/tenant/timeline/handle.rs @@ -108,10 +108,8 @@ use std::collections::hash_map; use std::collections::HashMap; use std::ops::Deref; -use std::sync::atomic::Ordering; use std::sync::Arc; use std::sync::Mutex; -use std::sync::Weak; use pageserver_api::shard::ShardIdentity; use tokio::sync::OwnedMutexGuard; @@ -121,7 +119,6 @@ use utils::id::TimelineId; use utils::shard::ShardIndex; use utils::shard::ShardNumber; -use crate::page_service::GetActiveTimelineError; use crate::tenant::mgr::ShardSelector; /// The requirement for Debug is so that #[derive(Debug)] works in some places. @@ -176,6 +173,7 @@ pub(crate) struct Handle(T::Timeline, tokio::sync::OwnedMutexGuard(T::Timeline, Arc>); enum HandleInner { KeepingTimelineGateOpen { + #[allow(dead_code)] gate_guard: utils::sync::gate::GateGuard, }, ShutDown, @@ -405,9 +403,7 @@ impl WeakHandle { pub(crate) async fn upgrade(&self) -> Result, HandleUpgradeError> { let lock_guard = self.1.clone().lock_owned().await; match &*lock_guard { - HandleInner::KeepingTimelineGateOpen { gate_guard } => { - Ok(Handle(self.0.clone(), lock_guard)) - } + HandleInner::KeepingTimelineGateOpen { .. } => Ok(Handle(self.0.clone(), lock_guard)), HandleInner::ShutDown => Err(HandleUpgradeError::ShutDown), } } @@ -471,6 +467,8 @@ impl PerTimelineState { #[cfg(test)] mod tests { + use std::sync::Weak; + use pageserver_api::{ key::{rel_block_to_key, Key, DBDIR_KEY}, models::ShardParameters, From 5b45f03aa255e3fd45f460b2d8f44a318fe80abe Mon Sep 17 00:00:00 2001 From: Christian Schwarz Date: Tue, 14 Jan 2025 11:39:19 +0100 Subject: [PATCH 071/111] tests: comment out the strong_count / weak_count assertions, --- pageserver/src/tenant/timeline/handle.rs | 84 ++++++++++++------------ 1 file changed, 42 insertions(+), 42 deletions(-) diff --git a/pageserver/src/tenant/timeline/handle.rs b/pageserver/src/tenant/timeline/handle.rs index 1e8def0aa1..565f7c0bdd 100644 --- a/pageserver/src/tenant/timeline/handle.rs +++ b/pageserver/src/tenant/timeline/handle.rs @@ -590,27 +590,27 @@ mod tests { .expect("we have the timeline"); let handle_inner_weak = Arc::downgrade(&handle.0); assert!(Weak::ptr_eq(&handle.myself, &shard0.myself)); - assert_eq!( - ( - Weak::strong_count(&handle_inner_weak), - Weak::weak_count(&handle_inner_weak) - ), - (2, 2), - "strong: handle, per_timeline_state, weak: handle_inner_weak, cache" - ); + // assert_eq!( + // ( + // Weak::strong_count(&handle_inner_weak), + // Weak::weak_count(&handle_inner_weak) + // ), + // (2, 2), + // "strong: handle, per_timeline_state, weak: handle_inner_weak, cache" + // ); assert_eq!(cache.map.len(), 1); - assert_eq!( - (Arc::strong_count(&shard0), Arc::weak_count(&shard0)), - (3, 1), - "strong: handleinner(per_timeline_state), shard0, mgr; weak: myself" - ); + // assert_eq!( + // (Arc::strong_count(&shard0), Arc::weak_count(&shard0)), + // (3, 1), + // "strong: handleinner(per_timeline_state), shard0, mgr; weak: myself" + // ); drop(handle); - assert_eq!( - (Arc::strong_count(&shard0), Arc::weak_count(&shard0)), - (3, 1), - "strong: handleinner(per_timeline_state), shard0, mgr; weak: myself" - ); + // assert_eq!( + // (Arc::strong_count(&shard0), Arc::weak_count(&shard0)), + // (3, 1), + // "strong: handleinner(per_timeline_state), shard0, mgr; weak: myself" + // ); // // demonstrate that Handle holds up gate closure @@ -635,21 +635,21 @@ mod tests { // SHUTDOWN shard0.per_timeline_state.shutdown().await; // keeping handle alive across shutdown - assert_eq!( - 1, - Weak::strong_count(&handle_inner_weak), - "through local var handle" - ); + // assert_eq!( + // 1, + // Weak::strong_count(&handle_inner_weak), + // "through local var handle" + // ); assert_eq!( cache.map.len(), 1, "this is an implementation detail but worth pointing out: we can't clear the cache from shutdown(), it's cleared on first access after" ); - assert_eq!( - (Arc::strong_count(&shard0), Arc::weak_count(&shard0)), - (3, 1), - "strong: handleinner(via handle), shard0, mgr; weak: myself" - ); + // assert_eq!( + // (Arc::strong_count(&shard0), Arc::weak_count(&shard0)), + // (3, 1), + // "strong: handleinner(via handle), shard0, mgr; weak: myself" + // ); // this handle is perfectly usable handle.getpage(); @@ -673,16 +673,16 @@ mod tests { } drop(handle); - assert_eq!( - 0, - Weak::strong_count(&handle_inner_weak), - "the HandleInner destructor already ran" - ); - assert_eq!( - (Arc::strong_count(&shard0), Arc::weak_count(&shard0)), - (2, 1), - "strong: shard0, mgr; weak: myself" - ); + // assert_eq!( + // 0, + // Weak::strong_count(&handle_inner_weak), + // "the HandleInner destructor already ran" + // ); + // assert_eq!( + // (Arc::strong_count(&shard0), Arc::weak_count(&shard0)), + // (2, 1), + // "strong: shard0, mgr; weak: myself" + // ); // closing gate succeeds after dropping handle tokio::select! { @@ -704,7 +704,7 @@ mod tests { let myself = Weak::clone(&shard0.myself); drop(shard0); drop(mgr); - assert_eq!(Weak::strong_count(&myself), 0); + // assert_eq!(Weak::strong_count(&myself), 0); } #[tokio::test] @@ -947,9 +947,9 @@ mod tests { } // No handles exist, thus gates are closed and don't require shutdown - assert!(used_handles - .iter() - .all(|weak| Weak::strong_count(weak) == 0)); + // assert!(used_handles + // .iter() + // .all(|weak| Weak::strong_count(weak) == 0)); // ... thus the gate should close immediately, even without shutdown tokio::select! { From 5762fdbf684b9f2258e04b51591c0b115dd3149e Mon Sep 17 00:00:00 2001 From: Christian Schwarz Date: Tue, 14 Jan 2025 11:39:46 +0100 Subject: [PATCH 072/111] test failure at: https://github.com/neondatabase/neon/blob/5b45f03aa255e3fd45f460b2d8f44a318fe80abe/pageserver/src/tenant/timeline/handle.rs#L958 From 62f63275b2deeac0b1c081b685043b0bf25ef032 Mon Sep 17 00:00:00 2001 From: Christian Schwarz Date: Tue, 14 Jan 2025 11:55:45 +0100 Subject: [PATCH 073/111] fix test_connection_handler_exit --- pageserver/src/tenant/timeline/handle.rs | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/pageserver/src/tenant/timeline/handle.rs b/pageserver/src/tenant/timeline/handle.rs index 565f7c0bdd..dfc721ba15 100644 --- a/pageserver/src/tenant/timeline/handle.rs +++ b/pageserver/src/tenant/timeline/handle.rs @@ -465,6 +465,27 @@ impl PerTimelineState { } } +// When dropping a [`Cache`], prune its handles in the [`PerTimelineState`] to break the reference cycle. +impl Drop for Cache { + fn drop(&mut self) { + for (_, WeakHandle(timeline, handle_inner_arc)) in self.map.drain() { + // handle is still being kept alive in PerTimelineState + let per_timeline_state = timeline.per_timeline_state(); + let mut handles = per_timeline_state.handles.lock().expect("mutex poisoned"); + let Some(handles) = &mut *handles else { + continue; + }; + let Some(removed) = handles.remove(&self.id) else { + // There could have been a shutdown inbetween us upgrading the weak and locking the mutex. + continue; + }; + let (_removed_timeline, removed_handle_inner_arc) = removed; + // TODO (extend ArcTimeline trait): assert!(Arc::ptr_eq(&removed_timeline, &timeline)); + assert!(Arc::ptr_eq(&removed_handle_inner_arc, &handle_inner_arc)); + } + } +} + #[cfg(test)] mod tests { use std::sync::Weak; From bfe9efefb823624b3ec9738e20ed96d177e8d3a9 Mon Sep 17 00:00:00 2001 From: Christian Schwarz Date: Tue, 14 Jan 2025 11:56:08 +0100 Subject: [PATCH 074/111] test tenant::timeline::handle::tests::test_timeline_shutdown hangs From e82aa9419e3f19bf6e6df20570c1b5d78f931843 Mon Sep 17 00:00:00 2001 From: Christian Schwarz Date: Tue, 14 Jan 2025 12:01:00 +0100 Subject: [PATCH 075/111] convert handles to named structs --- pageserver/src/tenant/timeline/handle.rs | 52 +++++++++++++++++------- 1 file changed, 37 insertions(+), 15 deletions(-) diff --git a/pageserver/src/tenant/timeline/handle.rs b/pageserver/src/tenant/timeline/handle.rs index dfc721ba15..4f7cd7fbe9 100644 --- a/pageserver/src/tenant/timeline/handle.rs +++ b/pageserver/src/tenant/timeline/handle.rs @@ -169,8 +169,14 @@ pub(crate) struct ShardTimelineId { } /// See module-level comment. -pub(crate) struct Handle(T::Timeline, tokio::sync::OwnedMutexGuard); -pub(crate) struct WeakHandle(T::Timeline, Arc>); +pub(crate) struct Handle { + timeline: T::Timeline, + inner: tokio::sync::OwnedMutexGuard, +} +pub(crate) struct WeakHandle { + timeline: T::Timeline, + inner: Arc>, +} enum HandleInner { KeepingTimelineGateOpen { #[allow(dead_code)] @@ -376,10 +382,10 @@ impl Cache { unreachable!() } hash_map::Entry::Vacant(v) => { - v.insert(WeakHandle( - timeline.clone(), - Arc::clone(&handle_inner_arc), - )); + v.insert(WeakHandle { + timeline: timeline.clone(), + inner: Arc::clone(&handle_inner_arc), + }); } } } @@ -388,7 +394,10 @@ impl Cache { } } } - Ok(Handle(timeline.clone(), handle_locked)) + Ok(Handle { + timeline: timeline.clone(), + inner: handle_locked, + }) } Err(e) => Err(GetError::TenantManager(e)), } @@ -401,14 +410,17 @@ pub(crate) enum HandleUpgradeError { impl WeakHandle { pub(crate) async fn upgrade(&self) -> Result, HandleUpgradeError> { - let lock_guard = self.1.clone().lock_owned().await; + let lock_guard = self.inner.clone().lock_owned().await; match &*lock_guard { - HandleInner::KeepingTimelineGateOpen { .. } => Ok(Handle(self.0.clone(), lock_guard)), + HandleInner::KeepingTimelineGateOpen { .. } => Ok(Handle { + timeline: self.timeline.clone(), + inner: lock_guard, + }), HandleInner::ShutDown => Err(HandleUpgradeError::ShutDown), } } pub(crate) fn timeline(&self) -> &T::Timeline { - &self.0 + &self.timeline } } @@ -416,13 +428,16 @@ impl Deref for Handle { type Target = T::Timeline; fn deref(&self) -> &Self::Target { - &self.0 + &self.timeline } } impl Handle { pub(crate) fn downgrade(&self) -> WeakHandle { - WeakHandle(self.0.clone(), Arc::clone(OwnedMutexGuard::mutex(&self.1))) + WeakHandle { + timeline: self.timeline.clone(), + inner: Arc::clone(OwnedMutexGuard::mutex(&self.inner)), + } } } @@ -468,7 +483,14 @@ impl PerTimelineState { // When dropping a [`Cache`], prune its handles in the [`PerTimelineState`] to break the reference cycle. impl Drop for Cache { fn drop(&mut self) { - for (_, WeakHandle(timeline, handle_inner_arc)) in self.map.drain() { + for ( + _, + WeakHandle { + timeline, + inner: handle_inner_arc, + }, + ) in self.map.drain() + { // handle is still being kept alive in PerTimelineState let per_timeline_state = timeline.per_timeline_state(); let mut handles = per_timeline_state.handles.lock().expect("mutex poisoned"); @@ -609,7 +631,7 @@ mod tests { .get(timeline_id, ShardSelector::Page(key), &mgr) .await .expect("we have the timeline"); - let handle_inner_weak = Arc::downgrade(&handle.0); + let handle_inner_weak = Arc::downgrade(&handle.timeline); assert!(Weak::ptr_eq(&handle.myself, &shard0.myself)); // assert_eq!( // ( @@ -964,7 +986,7 @@ mod tests { handle }; handle.getpage(); - used_handles.push(Arc::downgrade(&handle.0)); + used_handles.push(Arc::downgrade(&handle.timeline)); } // No handles exist, thus gates are closed and don't require shutdown From c868ceded0ebc3dc8635f822fbf072348440ae66 Mon Sep 17 00:00:00 2001 From: Christian Schwarz Date: Tue, 14 Jan 2025 12:08:53 +0100 Subject: [PATCH 076/111] WeakHandle should store weak ref to the GateGuard --- pageserver/src/tenant/timeline/handle.rs | 25 +++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/pageserver/src/tenant/timeline/handle.rs b/pageserver/src/tenant/timeline/handle.rs index 4f7cd7fbe9..fedc64c307 100644 --- a/pageserver/src/tenant/timeline/handle.rs +++ b/pageserver/src/tenant/timeline/handle.rs @@ -110,6 +110,7 @@ use std::collections::HashMap; use std::ops::Deref; use std::sync::Arc; use std::sync::Mutex; +use std::sync::Weak; use pageserver_api::shard::ShardIdentity; use tokio::sync::OwnedMutexGuard; @@ -175,7 +176,7 @@ pub(crate) struct Handle { } pub(crate) struct WeakHandle { timeline: T::Timeline, - inner: Arc>, + inner: Weak>, } enum HandleInner { KeepingTimelineGateOpen { @@ -384,7 +385,7 @@ impl Cache { hash_map::Entry::Vacant(v) => { v.insert(WeakHandle { timeline: timeline.clone(), - inner: Arc::clone(&handle_inner_arc), + inner: Arc::downgrade(&handle_inner_arc), }); } } @@ -410,7 +411,10 @@ pub(crate) enum HandleUpgradeError { impl WeakHandle { pub(crate) async fn upgrade(&self) -> Result, HandleUpgradeError> { - let lock_guard = self.inner.clone().lock_owned().await; + let Some(inner) = self.inner.upgrade() else { + return Err(HandleUpgradeError::ShutDown); + }; + let lock_guard = inner.clone().lock_owned().await; match &*lock_guard { HandleInner::KeepingTimelineGateOpen { .. } => Ok(Handle { timeline: self.timeline.clone(), @@ -435,8 +439,8 @@ impl Deref for Handle { impl Handle { pub(crate) fn downgrade(&self) -> WeakHandle { WeakHandle { - timeline: self.timeline.clone(), - inner: Arc::clone(OwnedMutexGuard::mutex(&self.inner)), + timeline: self.timeline.clone(), // FIXME: avoid this clone, it hits shared cache line + inner: Arc::downgrade(OwnedMutexGuard::mutex(&self.inner)), } } } @@ -486,13 +490,13 @@ impl Drop for Cache { for ( _, WeakHandle { - timeline, - inner: handle_inner_arc, + timeline: handle_timeline, + inner: handle_inner_weak, }, ) in self.map.drain() { // handle is still being kept alive in PerTimelineState - let per_timeline_state = timeline.per_timeline_state(); + let per_timeline_state = handle_timeline.per_timeline_state(); let mut handles = per_timeline_state.handles.lock().expect("mutex poisoned"); let Some(handles) = &mut *handles else { continue; @@ -503,7 +507,10 @@ impl Drop for Cache { }; let (_removed_timeline, removed_handle_inner_arc) = removed; // TODO (extend ArcTimeline trait): assert!(Arc::ptr_eq(&removed_timeline, &timeline)); - assert!(Arc::ptr_eq(&removed_handle_inner_arc, &handle_inner_arc)); + assert!(Weak::ptr_eq( + &Arc::downgrade(&removed_handle_inner_arc), + &handle_inner_weak + )); } } } From f6eb6fff9f67cf6303077dba023edd59b8f6e9e6 Mon Sep 17 00:00:00 2001 From: Christian Schwarz Date: Tue, 14 Jan 2025 12:14:31 +0100 Subject: [PATCH 077/111] WIP experiment: avoid upgrading --- pageserver/src/tenant/timeline/handle.rs | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/pageserver/src/tenant/timeline/handle.rs b/pageserver/src/tenant/timeline/handle.rs index fedc64c307..f3ccb5ad9f 100644 --- a/pageserver/src/tenant/timeline/handle.rs +++ b/pageserver/src/tenant/timeline/handle.rs @@ -292,17 +292,10 @@ impl Cache { ) -> RoutingResult { loop { // terminates because when every iteration we remove an element from the map - let Some((first_key, first_handle)) = self.map.iter().next() else { + let Some((first_key, WeakHandle { timeline, inner })) = self.map.iter().next() else { return RoutingResult::NeedConsultTenantManager; }; - let Ok(first_handle) = first_handle.upgrade().await else { - // TODO: dedup with get_impl() - trace!("handle cache stale"); - let first_key_owned = *first_key; - self.map.remove(&first_key_owned).unwrap(); - continue; - }; - + let first_handle = timeline; let first_handle_shard_identity = first_handle.get_shard_identity(); let make_shard_index = |shard_num: ShardNumber| ShardIndex { shard_number: shard_num, From a8d096b72c0af4cf776585088f28519c975d422c Mon Sep 17 00:00:00 2001 From: Christian Schwarz Date: Tue, 14 Jan 2025 12:18:02 +0100 Subject: [PATCH 078/111] Revert "WIP experiment: avoid upgrading" This reverts commit f6eb6fff9f67cf6303077dba023edd59b8f6e9e6. --- pageserver/src/tenant/timeline/handle.rs | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/pageserver/src/tenant/timeline/handle.rs b/pageserver/src/tenant/timeline/handle.rs index f3ccb5ad9f..fedc64c307 100644 --- a/pageserver/src/tenant/timeline/handle.rs +++ b/pageserver/src/tenant/timeline/handle.rs @@ -292,10 +292,17 @@ impl Cache { ) -> RoutingResult { loop { // terminates because when every iteration we remove an element from the map - let Some((first_key, WeakHandle { timeline, inner })) = self.map.iter().next() else { + let Some((first_key, first_handle)) = self.map.iter().next() else { return RoutingResult::NeedConsultTenantManager; }; - let first_handle = timeline; + let Ok(first_handle) = first_handle.upgrade().await else { + // TODO: dedup with get_impl() + trace!("handle cache stale"); + let first_key_owned = *first_key; + self.map.remove(&first_key_owned).unwrap(); + continue; + }; + let first_handle_shard_identity = first_handle.get_shard_identity(); let make_shard_index = |shard_num: ShardNumber| ShardIndex { shard_number: shard_num, From 22058d17d1cad65e5d467d19b5b4aa8a8e3ac80c Mon Sep 17 00:00:00 2001 From: Christian Schwarz Date: Tue, 14 Jan 2025 12:22:00 +0100 Subject: [PATCH 079/111] it turns out PerTimelineState need not store a Types::Timeline at all --- pageserver/src/page_service.rs | 2 +- pageserver/src/tenant/timeline.rs | 2 +- pageserver/src/tenant/timeline/handle.rs | 27 ++++++++++-------------- 3 files changed, 13 insertions(+), 18 deletions(-) diff --git a/pageserver/src/page_service.rs b/pageserver/src/page_service.rs index 714410baf4..ba410e4bd5 100644 --- a/pageserver/src/page_service.rs +++ b/pageserver/src/page_service.rs @@ -403,7 +403,7 @@ impl timeline::handle::ArcTimeline for Arc { Timeline::shard_timeline_id(self) } - fn per_timeline_state(&self) -> &timeline::handle::PerTimelineState { + fn per_timeline_state(&self) -> &timeline::handle::PerTimelineState { &self.handles } diff --git a/pageserver/src/tenant/timeline.rs b/pageserver/src/tenant/timeline.rs index 75dab4e7dd..d0223c2488 100644 --- a/pageserver/src/tenant/timeline.rs +++ b/pageserver/src/tenant/timeline.rs @@ -427,7 +427,7 @@ pub struct Timeline { pub(crate) l0_flush_global_state: L0FlushGlobalState, - pub(crate) handles: handle::PerTimelineState, + pub(crate) handles: handle::PerTimelineState, pub(crate) attach_wal_lag_cooldown: Arc>, diff --git a/pageserver/src/tenant/timeline/handle.rs b/pageserver/src/tenant/timeline/handle.rs index fedc64c307..743b3c0204 100644 --- a/pageserver/src/tenant/timeline/handle.rs +++ b/pageserver/src/tenant/timeline/handle.rs @@ -189,12 +189,12 @@ enum HandleInner { /// Embedded in each [`Types::Timeline`] as the anchor for the only long-lived strong ref to `HandleInner`. /// /// See module-level comment for details. -pub struct PerTimelineState { +pub struct PerTimelineState { // None = shutting down - handles: Mutex>)>>>, + handles: Mutex>>>>, } -impl Default for PerTimelineState { +impl Default for PerTimelineState { fn default() -> Self { Self { handles: Mutex::new(Some(Default::default())), @@ -218,7 +218,7 @@ pub(crate) trait ArcTimeline: Clone { fn gate(&self) -> &utils::sync::gate::Gate; fn shard_timeline_id(&self) -> ShardTimelineId; fn get_shard_identity(&self) -> &ShardIdentity; - fn per_timeline_state(&self) -> &PerTimelineState; + fn per_timeline_state(&self) -> &PerTimelineState; } /// Errors returned by [`Cache::get`]. @@ -371,8 +371,8 @@ impl Cache { .expect("mutex poisoned"); match &mut *lock_guard { Some(per_timeline_state) => { - let replaced = per_timeline_state - .insert(self.id, (timeline.clone(), Arc::clone(&handle_inner_arc))); + let replaced = + per_timeline_state.insert(self.id, Arc::clone(&handle_inner_arc)); assert!(replaced.is_none(), "some earlier code left a stale handle"); match self.map.entry(key) { hash_map::Entry::Occupied(_o) => { @@ -456,7 +456,7 @@ impl HandleInner { } } -impl PerTimelineState { +impl PerTimelineState { /// Invalidate all handles to this timeline in all [`Cache`]s. /// /// After this method returns, all subsequent [`Handle::upgrade`] will fail @@ -475,7 +475,7 @@ impl PerTimelineState { trace!("already shut down"); return; }; - for (_, state) in handles.values().map(|h| &*h) { + for state in handles.values().map(|h| &*h) { // Make further cache hits let mut lock_guard = state.lock().await; lock_guard.shutdown(); @@ -505,12 +505,7 @@ impl Drop for Cache { // There could have been a shutdown inbetween us upgrading the weak and locking the mutex. continue; }; - let (_removed_timeline, removed_handle_inner_arc) = removed; - // TODO (extend ArcTimeline trait): assert!(Arc::ptr_eq(&removed_timeline, &timeline)); - assert!(Weak::ptr_eq( - &Arc::downgrade(&removed_handle_inner_arc), - &handle_inner_weak - )); + assert!(Weak::ptr_eq(&Arc::downgrade(&removed), &handle_inner_weak)); } } } @@ -547,7 +542,7 @@ mod tests { gate: utils::sync::gate::Gate, id: TimelineId, shard: ShardIdentity, - per_timeline_state: PerTimelineState, + per_timeline_state: PerTimelineState, myself: Weak, } @@ -573,7 +568,7 @@ mod tests { &self.shard } - fn per_timeline_state(&self) -> &PerTimelineState { + fn per_timeline_state(&self) -> &PerTimelineState { &self.per_timeline_state } } From 6b22acba9bdc399a69a81c0b002c12d7194fef1e Mon Sep 17 00:00:00 2001 From: Christian Schwarz Date: Tue, 14 Jan 2025 12:28:37 +0100 Subject: [PATCH 080/111] avoid cloning the Arc on every handle upgrade/downgade, by wrapping it in yet another Arc --- pageserver/src/tenant/timeline/handle.rs | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/pageserver/src/tenant/timeline/handle.rs b/pageserver/src/tenant/timeline/handle.rs index 743b3c0204..7ef237734f 100644 --- a/pageserver/src/tenant/timeline/handle.rs +++ b/pageserver/src/tenant/timeline/handle.rs @@ -171,11 +171,11 @@ pub(crate) struct ShardTimelineId { /// See module-level comment. pub(crate) struct Handle { - timeline: T::Timeline, + timeline: Arc, inner: tokio::sync::OwnedMutexGuard, } pub(crate) struct WeakHandle { - timeline: T::Timeline, + timeline: Arc, inner: Weak>, } enum HandleInner { @@ -363,7 +363,7 @@ impl Cache { HandleInner::KeepingTimelineGateOpen { gate_guard }, )); let handle_locked = handle_inner_arc.clone().try_lock_owned().unwrap(); - { + let timeline = { let mut lock_guard = timeline .per_timeline_state() .handles @@ -383,10 +383,12 @@ impl Cache { unreachable!() } hash_map::Entry::Vacant(v) => { + let timeline = Arc::new(timeline.clone()); v.insert(WeakHandle { - timeline: timeline.clone(), + timeline: Arc::clone(&timeline), inner: Arc::downgrade(&handle_inner_arc), }); + timeline } } } @@ -394,9 +396,9 @@ impl Cache { return Err(GetError::PerTimelineStateShutDown); } } - } + }; Ok(Handle { - timeline: timeline.clone(), + timeline, inner: handle_locked, }) } @@ -417,7 +419,7 @@ impl WeakHandle { let lock_guard = inner.clone().lock_owned().await; match &*lock_guard { HandleInner::KeepingTimelineGateOpen { .. } => Ok(Handle { - timeline: self.timeline.clone(), + timeline: Arc::clone(&self.timeline), inner: lock_guard, }), HandleInner::ShutDown => Err(HandleUpgradeError::ShutDown), @@ -439,7 +441,7 @@ impl Deref for Handle { impl Handle { pub(crate) fn downgrade(&self) -> WeakHandle { WeakHandle { - timeline: self.timeline.clone(), // FIXME: avoid this clone, it hits shared cache line + timeline: Arc::clone(&self.timeline), inner: Arc::downgrade(OwnedMutexGuard::mutex(&self.inner)), } } From b44070d0c78552c75dcd119e5599ad54a08c8468 Mon Sep 17 00:00:00 2001 From: Christian Schwarz Date: Tue, 14 Jan 2025 13:30:48 +0100 Subject: [PATCH 081/111] experiment: what if we make Handle !Send so it can't be held across await points Result: the whole point of having a Handle at hand is to be holding a GateGuard while performing a Timeline operation. Dont' do it. --- pageserver/src/tenant/timeline/handle.rs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/pageserver/src/tenant/timeline/handle.rs b/pageserver/src/tenant/timeline/handle.rs index 7ef237734f..278bbdb863 100644 --- a/pageserver/src/tenant/timeline/handle.rs +++ b/pageserver/src/tenant/timeline/handle.rs @@ -173,6 +173,7 @@ pub(crate) struct ShardTimelineId { pub(crate) struct Handle { timeline: Arc, inner: tokio::sync::OwnedMutexGuard, + _dont_hold_across_await_points: std::marker::PhantomData<*const u8>, } pub(crate) struct WeakHandle { timeline: Arc, @@ -400,6 +401,7 @@ impl Cache { Ok(Handle { timeline, inner: handle_locked, + _dont_hold_across_await_points: Default::default(), }) } Err(e) => Err(GetError::TenantManager(e)), @@ -421,6 +423,7 @@ impl WeakHandle { HandleInner::KeepingTimelineGateOpen { .. } => Ok(Handle { timeline: Arc::clone(&self.timeline), inner: lock_guard, + _dont_hold_across_await_points: Default::default(), }), HandleInner::ShutDown => Err(HandleUpgradeError::ShutDown), } @@ -439,6 +442,9 @@ impl Deref for Handle { } impl Handle { + pub(crate) fn timeline(&self) -> &T::Timeline { + &self.timeline + } pub(crate) fn downgrade(&self) -> WeakHandle { WeakHandle { timeline: Arc::clone(&self.timeline), From b95365b45d4778e5d85a2eda3d6e1509558a7a3c Mon Sep 17 00:00:00 2001 From: Christian Schwarz Date: Tue, 14 Jan 2025 13:32:12 +0100 Subject: [PATCH 082/111] Revert "experiment: what if we make Handle !Send so it can't be held across await points" This reverts commit b44070d0c78552c75dcd119e5599ad54a08c8468. --- pageserver/src/tenant/timeline/handle.rs | 6 ------ 1 file changed, 6 deletions(-) diff --git a/pageserver/src/tenant/timeline/handle.rs b/pageserver/src/tenant/timeline/handle.rs index 278bbdb863..7ef237734f 100644 --- a/pageserver/src/tenant/timeline/handle.rs +++ b/pageserver/src/tenant/timeline/handle.rs @@ -173,7 +173,6 @@ pub(crate) struct ShardTimelineId { pub(crate) struct Handle { timeline: Arc, inner: tokio::sync::OwnedMutexGuard, - _dont_hold_across_await_points: std::marker::PhantomData<*const u8>, } pub(crate) struct WeakHandle { timeline: Arc, @@ -401,7 +400,6 @@ impl Cache { Ok(Handle { timeline, inner: handle_locked, - _dont_hold_across_await_points: Default::default(), }) } Err(e) => Err(GetError::TenantManager(e)), @@ -423,7 +421,6 @@ impl WeakHandle { HandleInner::KeepingTimelineGateOpen { .. } => Ok(Handle { timeline: Arc::clone(&self.timeline), inner: lock_guard, - _dont_hold_across_await_points: Default::default(), }), HandleInner::ShutDown => Err(HandleUpgradeError::ShutDown), } @@ -442,9 +439,6 @@ impl Deref for Handle { } impl Handle { - pub(crate) fn timeline(&self) -> &T::Timeline { - &self.timeline - } pub(crate) fn downgrade(&self) -> WeakHandle { WeakHandle { timeline: Arc::clone(&self.timeline), From df4abd8b14ff823f060f66acde34713da013d062 Mon Sep 17 00:00:00 2001 From: Conrad Ludgate Date: Tue, 14 Jan 2025 12:53:32 +0000 Subject: [PATCH 083/111] fix: force-refresh azure identity token (#10378) ## Problem Because of https://github.com/Azure/azure-sdk-for-rust/issues/1739, our identity token file was not being refreshed. This caused our uploads to start failing when the storage token expired. ## Summary of changes Drop and recreate the remote storage config every time we upload in order to force reload the identity token file. --- proxy/src/context/parquet.rs | 73 +++++++++++++++++++++++------------- 1 file changed, 46 insertions(+), 27 deletions(-) diff --git a/proxy/src/context/parquet.rs b/proxy/src/context/parquet.rs index 5f65b17374..d7ffff0483 100644 --- a/proxy/src/context/parquet.rs +++ b/proxy/src/context/parquet.rs @@ -187,10 +187,6 @@ pub async fn worker( let rx = futures::stream::poll_fn(move |cx| rx.poll_recv(cx)); let rx = rx.map(RequestData::from); - let storage = GenericRemoteStorage::from_config(&remote_storage_config) - .await - .context("remote storage init")?; - let properties = WriterProperties::builder() .set_data_page_size_limit(config.parquet_upload_page_size) .set_compression(config.parquet_upload_compression); @@ -224,18 +220,18 @@ pub async fn worker( let rx_disconnect = futures::stream::poll_fn(move |cx| rx_disconnect.poll_recv(cx)); let rx_disconnect = rx_disconnect.map(RequestData::from); - let storage_disconnect = - GenericRemoteStorage::from_config(&disconnect_events_storage_config) - .await - .context("remote storage for disconnect events init")?; let parquet_config_disconnect = parquet_config.clone(); tokio::try_join!( - worker_inner(storage, rx, parquet_config), - worker_inner(storage_disconnect, rx_disconnect, parquet_config_disconnect) + worker_inner(remote_storage_config, rx, parquet_config), + worker_inner( + disconnect_events_storage_config, + rx_disconnect, + parquet_config_disconnect + ) ) .map(|_| ()) } else { - worker_inner(storage, rx, parquet_config).await + worker_inner(remote_storage_config, rx, parquet_config).await } } @@ -251,18 +247,32 @@ struct ParquetConfig { test_remote_failures: u64, } +impl ParquetConfig { + async fn storage( + &self, + storage_config: &RemoteStorageConfig, + ) -> anyhow::Result { + let storage = GenericRemoteStorage::from_config(storage_config) + .await + .context("remote storage init")?; + + #[cfg(any(test, feature = "testing"))] + if self.test_remote_failures > 0 { + return Ok(GenericRemoteStorage::unreliable_wrapper( + storage, + self.test_remote_failures, + )); + } + + Ok(storage) + } +} + async fn worker_inner( - storage: GenericRemoteStorage, + storage_config: RemoteStorageConfig, rx: impl Stream, config: ParquetConfig, ) -> anyhow::Result<()> { - #[cfg(any(test, feature = "testing"))] - let storage = if config.test_remote_failures > 0 { - GenericRemoteStorage::unreliable_wrapper(storage, config.test_remote_failures) - } else { - storage - }; - let mut rx = std::pin::pin!(rx); let mut rows = Vec::with_capacity(config.rows_per_group); @@ -285,7 +295,7 @@ async fn worker_inner( } if len > config.file_size || force { last_upload = time::Instant::now(); - let file = upload_parquet(w, len, &storage).await?; + let file = upload_parquet(w, len, &storage_config, &config).await?; w = SerializedFileWriter::new(file, schema.clone(), config.propeties.clone())?; len = 0; } @@ -298,7 +308,7 @@ async fn worker_inner( } if !w.flushed_row_groups().is_empty() { - let _rtchk: Writer = upload_parquet(w, len, &storage).await?; + let _rtchk: Writer = upload_parquet(w, len, &storage_config, &config).await?; } Ok(()) @@ -340,7 +350,8 @@ where async fn upload_parquet( mut w: SerializedFileWriter>, len: i64, - storage: &GenericRemoteStorage, + storage_config: &RemoteStorageConfig, + config: &ParquetConfig, ) -> anyhow::Result> { let len_uncompressed = w .flushed_row_groups() @@ -377,6 +388,15 @@ async fn upload_parquet( size, compression, "uploading request parquet file" ); + // A bug in azure-sdk means that the identity-token-file that expires after + // 1 hour is not refreshed. This identity-token is used to fetch the actual azure storage + // tokens that last for 24 hours. After this 24 hour period, azure-sdk tries to refresh + // the storage token, but the identity token has now expired. + // + // + // To work around this, we recreate the storage every time. + let storage = config.storage(storage_config).await?; + let year = now.year(); let month = now.month(); let day = now.day(); @@ -431,8 +451,8 @@ mod tests { use rand::rngs::StdRng; use rand::{Rng, SeedableRng}; use remote_storage::{ - GenericRemoteStorage, RemoteStorageConfig, RemoteStorageKind, S3Config, - DEFAULT_MAX_KEYS_PER_LIST_RESPONSE, DEFAULT_REMOTE_STORAGE_S3_CONCURRENCY_LIMIT, + RemoteStorageConfig, RemoteStorageKind, S3Config, DEFAULT_MAX_KEYS_PER_LIST_RESPONSE, + DEFAULT_REMOTE_STORAGE_S3_CONCURRENCY_LIMIT, }; use tokio::sync::mpsc; use tokio::time; @@ -559,12 +579,11 @@ mod tests { timeout: std::time::Duration::from_secs(120), small_timeout: std::time::Duration::from_secs(30), }; - let storage = GenericRemoteStorage::from_config(&remote_storage_config) + + worker_inner(remote_storage_config, rx, config) .await .unwrap(); - worker_inner(storage, rx, config).await.unwrap(); - let mut files = WalkDir::new(tmpdir.as_std_path()) .into_iter() .filter_map(|entry| entry.ok()) From d68c5ddf7eeb1c2694e53a372493baf12227360e Mon Sep 17 00:00:00 2001 From: Christian Schwarz Date: Tue, 14 Jan 2025 14:23:28 +0100 Subject: [PATCH 084/111] avoid the tokio::sync::Mutex by wrapping the GateGuard into an Arc --- pageserver/src/tenant/timeline/handle.rs | 65 ++++++++++++++---------- 1 file changed, 37 insertions(+), 28 deletions(-) diff --git a/pageserver/src/tenant/timeline/handle.rs b/pageserver/src/tenant/timeline/handle.rs index 7ef237734f..3263697eb7 100644 --- a/pageserver/src/tenant/timeline/handle.rs +++ b/pageserver/src/tenant/timeline/handle.rs @@ -113,7 +113,6 @@ use std::sync::Mutex; use std::sync::Weak; use pageserver_api::shard::ShardIdentity; -use tokio::sync::OwnedMutexGuard; use tracing::instrument; use tracing::trace; use utils::id::TimelineId; @@ -172,16 +171,18 @@ pub(crate) struct ShardTimelineId { /// See module-level comment. pub(crate) struct Handle { timeline: Arc, - inner: tokio::sync::OwnedMutexGuard, + #[allow(dead_code)] // the field exists to keep the gate open + gate_guard: Arc, + inner: Arc>, } pub(crate) struct WeakHandle { timeline: Arc, - inner: Weak>, + inner: Weak>, } enum HandleInner { KeepingTimelineGateOpen { #[allow(dead_code)] - gate_guard: utils::sync::gate::GateGuard, + gate_guard: Arc, }, ShutDown, } @@ -191,7 +192,7 @@ enum HandleInner { /// See module-level comment for details. pub struct PerTimelineState { // None = shutting down - handles: Mutex>>>>, + handles: Mutex>>>>, } impl Default for PerTimelineState { @@ -356,13 +357,15 @@ impl Cache { return Err(GetError::TimelineGateClosed); } }; + let gate_guard = Arc::new(gate_guard); trace!("creating new HandleInner"); - let handle_inner_arc = Arc::new(tokio::sync::Mutex::new( + let handle_inner_arc = Arc::new(Mutex::new( // TODO: global metric that keeps track of the number of live HandlerTimeline instances // so we can identify reference cycle bugs. - HandleInner::KeepingTimelineGateOpen { gate_guard }, + HandleInner::KeepingTimelineGateOpen { + gate_guard: Arc::clone(&gate_guard), + }, )); - let handle_locked = handle_inner_arc.clone().try_lock_owned().unwrap(); let timeline = { let mut lock_guard = timeline .per_timeline_state() @@ -399,7 +402,8 @@ impl Cache { }; Ok(Handle { timeline, - inner: handle_locked, + inner: handle_inner_arc, + gate_guard, }) } Err(e) => Err(GetError::TenantManager(e)), @@ -413,15 +417,20 @@ pub(crate) enum HandleUpgradeError { impl WeakHandle { pub(crate) async fn upgrade(&self) -> Result, HandleUpgradeError> { - let Some(inner) = self.inner.upgrade() else { + let Some(inner) = Weak::upgrade(&self.inner) else { return Err(HandleUpgradeError::ShutDown); }; - let lock_guard = inner.clone().lock_owned().await; + let lock_guard = inner.lock().expect("poisoned"); match &*lock_guard { - HandleInner::KeepingTimelineGateOpen { .. } => Ok(Handle { - timeline: Arc::clone(&self.timeline), - inner: lock_guard, - }), + HandleInner::KeepingTimelineGateOpen { gate_guard } => { + let gate_guard = Arc::clone(gate_guard); + drop(lock_guard); + Ok(Handle { + timeline: Arc::clone(&self.timeline), + gate_guard, + inner, + }) + } HandleInner::ShutDown => Err(HandleUpgradeError::ShutDown), } } @@ -442,18 +451,7 @@ impl Handle { pub(crate) fn downgrade(&self) -> WeakHandle { WeakHandle { timeline: Arc::clone(&self.timeline), - inner: Arc::downgrade(OwnedMutexGuard::mutex(&self.inner)), - } - } -} - -impl HandleInner { - pub(crate) fn shutdown(&mut self) { - match self { - HandleInner::KeepingTimelineGateOpen { .. } => *self = HandleInner::ShutDown, - HandleInner::ShutDown => { - unreachable!("handles are only shut down once in their lifetime"); - } + inner: Arc::downgrade(&self.inner), } } } @@ -479,13 +477,24 @@ impl PerTimelineState { }; for state in handles.values().map(|h| &*h) { // Make further cache hits - let mut lock_guard = state.lock().await; + let mut lock_guard = state.lock().expect("poisoned"); lock_guard.shutdown(); } drop(handles); } } +impl HandleInner { + pub(crate) fn shutdown(&mut self) { + match self { + HandleInner::KeepingTimelineGateOpen { .. } => *self = HandleInner::ShutDown, + HandleInner::ShutDown => { + unreachable!("handles are only shut down once in their lifetime"); + } + } + } +} + // When dropping a [`Cache`], prune its handles in the [`PerTimelineState`] to break the reference cycle. impl Drop for Cache { fn drop(&mut self) { From 7034f54a9ed499d6ae6d63109dcc9e792fbf5397 Mon Sep 17 00:00:00 2001 From: Christian Schwarz Date: Tue, 14 Jan 2025 14:35:43 +0100 Subject: [PATCH 085/111] remove the earlier-commented-out assertions on arc reference counts, they were too whiteboxy to begin with --- pageserver/src/tenant/timeline/handle.rs | 56 +----------------------- 1 file changed, 2 insertions(+), 54 deletions(-) diff --git a/pageserver/src/tenant/timeline/handle.rs b/pageserver/src/tenant/timeline/handle.rs index 3263697eb7..77e496acff 100644 --- a/pageserver/src/tenant/timeline/handle.rs +++ b/pageserver/src/tenant/timeline/handle.rs @@ -634,39 +634,13 @@ mod tests { // // fill the cache // - assert_eq!( - (Arc::strong_count(&shard0), Arc::weak_count(&shard0)), - (2, 1), - "strong: shard0, mgr; weak: myself" - ); - let handle: Handle<_> = cache .get(timeline_id, ShardSelector::Page(key), &mgr) .await .expect("we have the timeline"); - let handle_inner_weak = Arc::downgrade(&handle.timeline); assert!(Weak::ptr_eq(&handle.myself, &shard0.myself)); - // assert_eq!( - // ( - // Weak::strong_count(&handle_inner_weak), - // Weak::weak_count(&handle_inner_weak) - // ), - // (2, 2), - // "strong: handle, per_timeline_state, weak: handle_inner_weak, cache" - // ); assert_eq!(cache.map.len(), 1); - - // assert_eq!( - // (Arc::strong_count(&shard0), Arc::weak_count(&shard0)), - // (3, 1), - // "strong: handleinner(per_timeline_state), shard0, mgr; weak: myself" - // ); drop(handle); - // assert_eq!( - // (Arc::strong_count(&shard0), Arc::weak_count(&shard0)), - // (3, 1), - // "strong: handleinner(per_timeline_state), shard0, mgr; weak: myself" - // ); // // demonstrate that Handle holds up gate closure @@ -691,21 +665,11 @@ mod tests { // SHUTDOWN shard0.per_timeline_state.shutdown().await; // keeping handle alive across shutdown - // assert_eq!( - // 1, - // Weak::strong_count(&handle_inner_weak), - // "through local var handle" - // ); assert_eq!( cache.map.len(), 1, "this is an implementation detail but worth pointing out: we can't clear the cache from shutdown(), it's cleared on first access after" ); - // assert_eq!( - // (Arc::strong_count(&shard0), Arc::weak_count(&shard0)), - // (3, 1), - // "strong: handleinner(via handle), shard0, mgr; weak: myself" - // ); // this handle is perfectly usable handle.getpage(); @@ -729,16 +693,6 @@ mod tests { } drop(handle); - // assert_eq!( - // 0, - // Weak::strong_count(&handle_inner_weak), - // "the HandleInner destructor already ran" - // ); - // assert_eq!( - // (Arc::strong_count(&shard0), Arc::weak_count(&shard0)), - // (2, 1), - // "strong: shard0, mgr; weak: myself" - // ); // closing gate succeeds after dropping handle tokio::select! { @@ -757,10 +711,8 @@ mod tests { assert_eq!(cache.map.len(), 0); // ensure all refs to shard0 are gone and we're not leaking anything - let myself = Weak::clone(&shard0.myself); drop(shard0); drop(mgr); - // assert_eq!(Weak::strong_count(&myself), 0); } #[tokio::test] @@ -1002,12 +954,8 @@ mod tests { used_handles.push(Arc::downgrade(&handle.timeline)); } - // No handles exist, thus gates are closed and don't require shutdown - // assert!(used_handles - // .iter() - // .all(|weak| Weak::strong_count(weak) == 0)); - - // ... thus the gate should close immediately, even without shutdown + // No handles exist, thus gates are closed and don't require shutdown. + // Thus the gate should close immediately, even without shutdown. tokio::select! { _ = shard0.gate.close() => { } _ = tokio::time::sleep(FOREVER) => { From e4ea706424b8d3589cddf933139e68d7c7447485 Mon Sep 17 00:00:00 2001 From: Christian Schwarz Date: Tue, 14 Jan 2025 14:48:03 +0100 Subject: [PATCH 086/111] turns out PerTimelineState::shutdown() doesn't need to be async --- pageserver/src/tenant/timeline.rs | 2 +- pageserver/src/tenant/timeline/handle.rs | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/pageserver/src/tenant/timeline.rs b/pageserver/src/tenant/timeline.rs index d0223c2488..d6fb641a90 100644 --- a/pageserver/src/tenant/timeline.rs +++ b/pageserver/src/tenant/timeline.rs @@ -1826,7 +1826,7 @@ impl Timeline { self.cancel.cancel(); // Ensure Prevent new page service requests from starting. - self.handles.shutdown().await; + self.handles.shutdown(); // Transition the remote_client into a state where it's only useful for timeline deletion. // (The deletion use case is why we can't just hook up remote_client to Self::cancel).) diff --git a/pageserver/src/tenant/timeline/handle.rs b/pageserver/src/tenant/timeline/handle.rs index 77e496acff..c1b5091f77 100644 --- a/pageserver/src/tenant/timeline/handle.rs +++ b/pageserver/src/tenant/timeline/handle.rs @@ -462,7 +462,7 @@ impl PerTimelineState { /// After this method returns, all subsequent [`Handle::upgrade`] will fail /// and they will not be holding the [`ArcTimeline`]'s gate open. #[instrument(level = "trace", skip_all)] - pub(super) async fn shutdown(&self) { + pub(super) fn shutdown(&self) { let handles = self .handles .lock() @@ -475,7 +475,7 @@ impl PerTimelineState { trace!("already shut down"); return; }; - for state in handles.values().map(|h| &*h) { + for state in handles.values() { // Make further cache hits let mut lock_guard = state.lock().expect("poisoned"); lock_guard.shutdown(); @@ -663,7 +663,7 @@ mod tests { assert!(Weak::ptr_eq(&handle.myself, &shard0.myself)); // SHUTDOWN - shard0.per_timeline_state.shutdown().await; // keeping handle alive across shutdown + shard0.per_timeline_state.shutdown(); // keeping handle alive across shutdown assert_eq!( cache.map.len(), @@ -754,7 +754,7 @@ mod tests { assert_eq!(cache.map.len(), 2); // delete timeline A - timeline_a.per_timeline_state.shutdown().await; + timeline_a.per_timeline_state.shutdown(); mgr.shards.retain(|t| t.id != timeline_a.id); assert!( mgr.resolve(timeline_a.id, ShardSelector::Page(key)) @@ -882,7 +882,7 @@ mod tests { assert!(Weak::ptr_eq(&parent_handle.myself, &parent.myself)); // invalidate the cache - parent.per_timeline_state.shutdown().await; + parent.per_timeline_state.shutdown(); // the cache will now return the child, even though the parent handle still exists for i in 0..2 { From 3b65465e106c2c60d39acc1491fcb09d870827c7 Mon Sep 17 00:00:00 2001 From: Christian Schwarz Date: Tue, 14 Jan 2025 14:53:41 +0100 Subject: [PATCH 087/111] turns out with the switch to sync Mutex there's no reason for upgrade() to be async either --- pageserver/src/page_service.rs | 17 +++++------------ pageserver/src/tenant/timeline/handle.rs | 10 +++++----- 2 files changed, 10 insertions(+), 17 deletions(-) diff --git a/pageserver/src/page_service.rs b/pageserver/src/page_service.rs index ba410e4bd5..d75444472f 100644 --- a/pageserver/src/page_service.rs +++ b/pageserver/src/page_service.rs @@ -967,14 +967,7 @@ impl PageServerHandler { fail::fail_point!("ps::handle-pagerequest-message::exists"); ( vec![self - .handle_get_rel_exists_request( - &*shard - .upgrade() - // TODO: need sensitive to cancel() - .await?, - &req, - ctx, - ) + .handle_get_rel_exists_request(&*shard.upgrade()?, &req, ctx) .instrument(span.clone()) .await .map(|msg| (msg, timer)) @@ -991,7 +984,7 @@ impl PageServerHandler { fail::fail_point!("ps::handle-pagerequest-message::nblocks"); ( vec![self - .handle_get_nblocks_request(&*shard.upgrade().await?, &req, ctx) + .handle_get_nblocks_request(&*shard.upgrade()?, &req, ctx) .instrument(span.clone()) .await .map(|msg| (msg, timer)) @@ -1012,7 +1005,7 @@ impl PageServerHandler { trace!(npages, "handling getpage request"); let res = self .handle_get_page_at_lsn_request_batched( - &*shard.upgrade().await?, + &*shard.upgrade()?, effective_request_lsn, pages, ctx, @@ -1034,7 +1027,7 @@ impl PageServerHandler { fail::fail_point!("ps::handle-pagerequest-message::dbsize"); ( vec![self - .handle_db_size_request(&*shard.upgrade().await?, &req, ctx) + .handle_db_size_request(&*shard.upgrade()?, &req, ctx) .instrument(span.clone()) .await .map(|msg| (msg, timer)) @@ -1051,7 +1044,7 @@ impl PageServerHandler { fail::fail_point!("ps::handle-pagerequest-message::slrusegment"); ( vec![self - .handle_get_slru_segment_request(&*shard.upgrade().await?, &req, ctx) + .handle_get_slru_segment_request(&*shard.upgrade()?, &req, ctx) .instrument(span.clone()) .await .map(|msg| (msg, timer)) diff --git a/pageserver/src/tenant/timeline/handle.rs b/pageserver/src/tenant/timeline/handle.rs index c1b5091f77..a2c38d5a2d 100644 --- a/pageserver/src/tenant/timeline/handle.rs +++ b/pageserver/src/tenant/timeline/handle.rs @@ -264,11 +264,11 @@ impl Cache { tenant_manager: &T::TenantManager, ) -> Result, GetError> { let miss: ShardSelector = { - let routing_state = self.shard_routing(timeline_id, shard_selector).await; + let routing_state = self.shard_routing(timeline_id, shard_selector); match routing_state { RoutingResult::FastPath(handle) => return Ok(handle), RoutingResult::SlowPath(key) => match self.map.get(&key) { - Some(cached) => match cached.upgrade().await { + Some(cached) => match cached.upgrade() { Ok(upgraded) => return Ok(upgraded), Err(HandleUpgradeError::ShutDown) => { // TODO: dedup with shard_routing() @@ -286,7 +286,7 @@ impl Cache { } #[inline(always)] - async fn shard_routing( + fn shard_routing( &mut self, timeline_id: TimelineId, shard_selector: ShardSelector, @@ -296,7 +296,7 @@ impl Cache { let Some((first_key, first_handle)) = self.map.iter().next() else { return RoutingResult::NeedConsultTenantManager; }; - let Ok(first_handle) = first_handle.upgrade().await else { + let Ok(first_handle) = first_handle.upgrade() else { // TODO: dedup with get_impl() trace!("handle cache stale"); let first_key_owned = *first_key; @@ -416,7 +416,7 @@ pub(crate) enum HandleUpgradeError { } impl WeakHandle { - pub(crate) async fn upgrade(&self) -> Result, HandleUpgradeError> { + pub(crate) fn upgrade(&self) -> Result, HandleUpgradeError> { let Some(inner) = Weak::upgrade(&self.inner) else { return Err(HandleUpgradeError::ShutDown); }; From 768a867dcf83a1db76edd7423f43c859312d2457 Mon Sep 17 00:00:00 2001 From: Christian Schwarz Date: Tue, 14 Jan 2025 14:54:15 +0100 Subject: [PATCH 088/111] doc comment fix --- pageserver/src/tenant/timeline/handle.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pageserver/src/tenant/timeline/handle.rs b/pageserver/src/tenant/timeline/handle.rs index a2c38d5a2d..9f46df264a 100644 --- a/pageserver/src/tenant/timeline/handle.rs +++ b/pageserver/src/tenant/timeline/handle.rs @@ -459,7 +459,7 @@ impl Handle { impl PerTimelineState { /// Invalidate all handles to this timeline in all [`Cache`]s. /// - /// After this method returns, all subsequent [`Handle::upgrade`] will fail + /// After this method returns, all subsequent [`WeakHandle::upgrade`] will fail /// and they will not be holding the [`ArcTimeline`]'s gate open. #[instrument(level = "trace", skip_all)] pub(super) fn shutdown(&self) { From c8bee865868c9995d46857db4e9664ef5c178a3d Mon Sep 17 00:00:00 2001 From: Christian Schwarz Date: Tue, 14 Jan 2025 15:12:03 +0100 Subject: [PATCH 089/111] in some early WIP commit we had removed the loop{} inside get(); re-establish it one level down --- pageserver/src/tenant/timeline/handle.rs | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/pageserver/src/tenant/timeline/handle.rs b/pageserver/src/tenant/timeline/handle.rs index 9f46df264a..8ea10a1d45 100644 --- a/pageserver/src/tenant/timeline/handle.rs +++ b/pageserver/src/tenant/timeline/handle.rs @@ -263,7 +263,8 @@ impl Cache { shard_selector: ShardSelector, tenant_manager: &T::TenantManager, ) -> Result, GetError> { - let miss: ShardSelector = { + // terminates because when every iteration we remove an element from the map + let miss: ShardSelector = loop { let routing_state = self.shard_routing(timeline_id, shard_selector); match routing_state { RoutingResult::FastPath(handle) => return Ok(handle), @@ -274,12 +275,12 @@ impl Cache { // TODO: dedup with shard_routing() trace!("handle cache stale"); self.map.remove(&key).unwrap(); - ShardSelector::Known(key.shard_index) + continue; } }, - None => ShardSelector::Known(key.shard_index), + None => break ShardSelector::Known(key.shard_index), }, - RoutingResult::NeedConsultTenantManager => shard_selector, + RoutingResult::NeedConsultTenantManager => break shard_selector, } }; self.get_miss(timeline_id, miss, tenant_manager).await From 4e094d9638c7355c24d83951a619a2f3640e79d4 Mon Sep 17 00:00:00 2001 From: Christian Schwarz Date: Tue, 14 Jan 2025 15:12:46 +0100 Subject: [PATCH 090/111] rearrange code & inline HandleInner::shutdown() to minimize the diff --- pageserver/src/tenant/timeline/handle.rs | 58 +++++++++++------------- 1 file changed, 26 insertions(+), 32 deletions(-) diff --git a/pageserver/src/tenant/timeline/handle.rs b/pageserver/src/tenant/timeline/handle.rs index 8ea10a1d45..80a4ec06e5 100644 --- a/pageserver/src/tenant/timeline/handle.rs +++ b/pageserver/src/tenant/timeline/handle.rs @@ -107,7 +107,6 @@ use std::collections::hash_map; use std::collections::HashMap; -use std::ops::Deref; use std::sync::Arc; use std::sync::Mutex; use std::sync::Weak; @@ -440,28 +439,13 @@ impl WeakHandle { } } -impl Deref for Handle { - type Target = T::Timeline; - - fn deref(&self) -> &Self::Target { - &self.timeline - } -} - -impl Handle { - pub(crate) fn downgrade(&self) -> WeakHandle { - WeakHandle { - timeline: Arc::clone(&self.timeline), - inner: Arc::downgrade(&self.inner), - } - } -} - impl PerTimelineState { - /// Invalidate all handles to this timeline in all [`Cache`]s. + /// After this method returns, [`Cache::get`] will never again return a [`Handle`] + /// to the [`Types::Timeline`] that embeds this per-timeline state. + /// Even if [`TenantManager::resolve`] would still resolve to it. /// - /// After this method returns, all subsequent [`WeakHandle::upgrade`] will fail - /// and they will not be holding the [`ArcTimeline`]'s gate open. + /// Already-alive [`Handle`]s for will remain open, usable, and keeping the [`ArcTimeline`] alive. + /// That's ok because they're short-lived. See module-level comment for details. #[instrument(level = "trace", skip_all)] pub(super) fn shutdown(&self) { let handles = self @@ -476,22 +460,32 @@ impl PerTimelineState { trace!("already shut down"); return; }; - for state in handles.values() { - // Make further cache hits - let mut lock_guard = state.lock().expect("poisoned"); - lock_guard.shutdown(); + for handle_inner_arc in handles.values() { + // Make hits fail. + let mut lock_guard = handle_inner_arc.lock().expect("poisoned"); + match &mut *lock_guard { + HandleInner::KeepingTimelineGateOpen { .. } => *lock_guard = HandleInner::ShutDown, + HandleInner::ShutDown => { + unreachable!("handles are only shut down once in their lifetime"); + } + } } drop(handles); } } -impl HandleInner { - pub(crate) fn shutdown(&mut self) { - match self { - HandleInner::KeepingTimelineGateOpen { .. } => *self = HandleInner::ShutDown, - HandleInner::ShutDown => { - unreachable!("handles are only shut down once in their lifetime"); - } +impl std::ops::Deref for Handle { + type Target = T::Timeline; + fn deref(&self) -> &Self::Target { + &self.timeline + } +} + +impl Handle { + pub(crate) fn downgrade(&self) -> WeakHandle { + WeakHandle { + timeline: Arc::clone(&self.timeline), + inner: Arc::downgrade(&self.inner), } } } From 47544dcc0b16403dd750cc976644345d5d8e457e Mon Sep 17 00:00:00 2001 From: Christian Schwarz Date: Tue, 14 Jan 2025 15:28:09 +0100 Subject: [PATCH 091/111] simplify SmgrOpTimerState variant names, and add some doc comments; https://github.com/neondatabase/neon/pull/10353#discussion_r1913676569 and https://github.com/neondatabase/neon/pull/10353#discussion_r1913676824 --- pageserver/src/metrics.rs | 46 +++++++++++++++++++++++++++++---------- 1 file changed, 34 insertions(+), 12 deletions(-) diff --git a/pageserver/src/metrics.rs b/pageserver/src/metrics.rs index 38c8d8aba6..fe4d3b7b06 100644 --- a/pageserver/src/metrics.rs +++ b/pageserver/src/metrics.rs @@ -1238,6 +1238,16 @@ pub(crate) struct SmgrOpTimerInner { timings: SmgrOpTimerState, } +/// The stages of request processing are represented by the enum variants. +/// Used as part of [`SmgrOpTimerInner::timings`]. +/// +/// Request processing calls into the `SmgrOpTimer::observe_*` methods at the +/// transition points. +/// These methods bump relevant counters and then update [`SmgrOpTimerInner::timings`] +/// to the next state. +/// +/// Each request goes through every stage, in all configurations. +/// #[derive(Debug)] enum SmgrOpTimerState { Received { @@ -1247,13 +1257,13 @@ enum SmgrOpTimerState { #[allow(dead_code)] received_at: Instant, }, - ParsedRoutedNowThrottling { + Throttling { throttle_started_at: Instant, }, - ParsedRoutedThrottledNowBatching { + Batching { throttle_done_at: Instant, }, - BatchedNowExecuting { + Executing { execution_started_at: Instant, }, Flushing, @@ -1262,6 +1272,7 @@ enum SmgrOpTimerState { // NB: when adding observation points, remember to update the Drop impl. impl SmgrOpTimer { + /// See [`SmgrOpTimerState`] for more context. pub(crate) fn observe_throttle_start(&mut self, at: Instant) { let Some(inner) = self.0.as_mut() else { return; @@ -1270,15 +1281,17 @@ impl SmgrOpTimer { return; }; inner.throttling.count_accounted_start.inc(); - inner.timings = SmgrOpTimerState::ParsedRoutedNowThrottling { + inner.timings = SmgrOpTimerState::Throttling { throttle_started_at: at, }; } + + /// See [`SmgrOpTimerState`] for more context. pub(crate) fn observe_throttle_done(&mut self, throttle: ThrottleResult) { let Some(inner) = self.0.as_mut() else { return; }; - let SmgrOpTimerState::ParsedRoutedNowThrottling { + let SmgrOpTimerState::Throttling { throttle_started_at, } = &mut inner.timings else { @@ -1287,7 +1300,7 @@ impl SmgrOpTimer { inner.throttling.count_accounted_finish.inc(); match throttle { ThrottleResult::NotThrottled { end } => { - inner.timings = SmgrOpTimerState::ParsedRoutedThrottledNowBatching { + inner.timings = SmgrOpTimerState::Batching { throttle_done_at: end, }; } @@ -1299,20 +1312,19 @@ impl SmgrOpTimer { .wait_time .inc_by((end - *throttle_started_at).as_micros().try_into().unwrap()); // state transition - inner.timings = SmgrOpTimerState::ParsedRoutedThrottledNowBatching { + inner.timings = SmgrOpTimerState::Batching { throttle_done_at: end, }; } } } + /// See [`SmgrOpTimerState`] for more context. pub(crate) fn observe_execution_start(&mut self, at: Instant) { let Some(inner) = self.0.as_mut() else { return; }; - let SmgrOpTimerState::ParsedRoutedThrottledNowBatching { throttle_done_at } = - &mut inner.timings - else { + let SmgrOpTimerState::Batching { throttle_done_at } = &mut inner.timings else { return; }; // update metrics @@ -1322,13 +1334,15 @@ impl SmgrOpTimer { .per_timeline_batch_wait_time .observe(batch.as_secs_f64()); // state transition - inner.timings = SmgrOpTimerState::BatchedNowExecuting { + inner.timings = SmgrOpTimerState::Executing { execution_started_at: at, } } /// For all but the first caller, this is a no-op. /// The first callers receives Some, subsequent ones None. + /// + /// See [`SmgrOpTimerState`] for more context. pub(crate) fn observe_execution_end_flush_start( &mut self, at: Instant, @@ -1338,7 +1352,7 @@ impl SmgrOpTimer { let Some(mut inner) = self.0.take() else { return None; }; - let SmgrOpTimerState::BatchedNowExecuting { + let SmgrOpTimerState::Executing { execution_started_at, } = &mut inner.timings else { @@ -1373,6 +1387,14 @@ impl SmgrOpTimer { } } +/// The last stage of request processing is serializing and flushing the request +/// into the TCP connection. We want to make slow flushes observable +/// _while they are occuring_, so this struct provides a wrapper method [`Self::measure`] +/// to periodically bump the metric. +/// +/// If in the future we decide that we're not interested in live updates, we can +/// add another `observe_*` method to [`SmgrOpTimer`], follow the existing pattern there, +/// and remove this struct from the code base. pub(crate) struct SmgrOpFlushInProgress { flush_started_at: Instant, global_micros: IntCounter, From 9e1cd986d77511f76408d8906dac5f650e867194 Mon Sep 17 00:00:00 2001 From: Christian Schwarz Date: Tue, 14 Jan 2025 15:30:33 +0100 Subject: [PATCH 092/111] address "why mut" nits; https://github.com/neondatabase/neon/pull/10353#discussion_r1913685991 https://github.com/neondatabase/neon/pull/10353#discussion_r1913683065 https://github.com/neondatabase/neon/pull/10353#discussion_r1913683392 --- pageserver/src/metrics.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pageserver/src/metrics.rs b/pageserver/src/metrics.rs index fe4d3b7b06..d478aa87fa 100644 --- a/pageserver/src/metrics.rs +++ b/pageserver/src/metrics.rs @@ -1293,7 +1293,7 @@ impl SmgrOpTimer { }; let SmgrOpTimerState::Throttling { throttle_started_at, - } = &mut inner.timings + } = &inner.timings else { return; }; @@ -1324,7 +1324,7 @@ impl SmgrOpTimer { let Some(inner) = self.0.as_mut() else { return; }; - let SmgrOpTimerState::Batching { throttle_done_at } = &mut inner.timings else { + let SmgrOpTimerState::Batching { throttle_done_at } = &inner.timings else { return; }; // update metrics @@ -1354,7 +1354,7 @@ impl SmgrOpTimer { }; let SmgrOpTimerState::Executing { execution_started_at, - } = &mut inner.timings + } = &inner.timings else { return None; }; From 9bdb14c1c0b3ef1082ce68f1c54d4547393da362 Mon Sep 17 00:00:00 2001 From: "Alex Chi Z." <4198311+skyzh@users.noreply.github.com> Date: Tue, 14 Jan 2025 10:27:48 -0500 Subject: [PATCH 093/111] fix(pageserver): ensure initial image layers have correct key ranges (#10374) ## Problem Discovered during the relation dir refactor work. If we do not create images as in this patch, we would get two set of image layers: ``` 0000...METADATA_KEYS 0000...REL_KEYS ``` They overlap at the same LSN and would cause data loss for relation keys. This doesn't happen in prod because initial image layer generation is never called, but better to be fixed to avoid future issues with the reldir refactors. ## Summary of changes * Consolidate create_image_layers call into a single one. --------- Signed-off-by: Alex Chi Z --- pageserver/src/tenant/timeline.rs | 43 +++++++++++++++---------------- 1 file changed, 21 insertions(+), 22 deletions(-) diff --git a/pageserver/src/tenant/timeline.rs b/pageserver/src/tenant/timeline.rs index f7227efeba..741b214a73 100644 --- a/pageserver/src/tenant/timeline.rs +++ b/pageserver/src/tenant/timeline.rs @@ -3781,36 +3781,35 @@ impl Timeline { return Err(FlushLayerError::Cancelled); } - let mut layers_to_upload = Vec::new(); - layers_to_upload.extend( - self.create_image_layers( - &rel_partition, - self.initdb_lsn, - ImageLayerCreationMode::Initial, - ctx, - ) - .await?, - ); + // Ensure that we have a single call to `create_image_layers` with a combined dense keyspace. + // So that the key ranges don't overlap. + let mut partitions = KeyPartitioning::default(); + partitions.parts.extend(rel_partition.parts); if !metadata_partition.parts.is_empty() { assert_eq!( metadata_partition.parts.len(), 1, "currently sparse keyspace should only contain a single metadata keyspace" ); - layers_to_upload.extend( - self.create_image_layers( - // Safety: create_image_layers treat sparse keyspaces differently that it does not scan - // every single key within the keyspace, and therefore, it's safe to force converting it - // into a dense keyspace before calling this function. - &metadata_partition.into_dense(), - self.initdb_lsn, - ImageLayerCreationMode::Initial, - ctx, - ) - .await?, - ); + // Safety: create_image_layers treat sparse keyspaces differently that it does not scan + // every single key within the keyspace, and therefore, it's safe to force converting it + // into a dense keyspace before calling this function. + partitions + .parts + .extend(metadata_partition.into_dense().parts); } + let mut layers_to_upload = Vec::new(); + layers_to_upload.extend( + self.create_image_layers( + &partitions, + self.initdb_lsn, + ImageLayerCreationMode::Initial, + ctx, + ) + .await?, + ); + (layers_to_upload, None) } else { // Normal case, write out a L0 delta layer file. From 2466a2f97763aff5b84253a242909fc19bc995a2 Mon Sep 17 00:00:00 2001 From: Christian Schwarz Date: Tue, 14 Jan 2025 16:28:01 +0100 Subject: [PATCH 094/111] page_service: throttle individual requests instead of the batched request (#10353) ## Problem Before this PR, the pagestream throttle was applied weighted on a per-batch basis. This had several problems: 1. The throttle occurence counters were only bumped by `1` instead of `batch_size`. 2. The throttle wait time aggregator metric only counted one wait time, irrespective of `batch_size`. That makes sense in some ways of looking at it but not in others. 3. If the last request in the batch runs into the throttle, the other requests in the batch are also throttled, i.e., over-throttling happens (theoretical, didn't measure it in practice). ## Solution It occured to me that we can simply push the throttling upwards into `pagestream_read_message`. This has the added benefit that in pipeline mode, the `executor` stage will, if it is idle, steal whatever requests already made it into the `spsc_fold` and execute them; before this change, that was not the case - the throttling happened in the `executor` stage instead of the `batcher` stage. ## Code Changes There are two changes in this PR: 1. Lifting up the throttling into the `pagestream_read_message` method. 2. Move the throttling metrics out of the `Throttle` type into `SmgrOpMetrics`. Unlike the other smgr metrics, throttling is per-tenant, hence the Arc. 3. Refactor the `SmgrOpTimer` implementation to account for the new observation states, and simplify its design. 4. Drive-by-fix flush time metrics. It was using the same `now` in the `observe_guard` every time. The `SmgrOpTimer` is now a state machine. Each observation point moves the state machine forward. If a timer object is dropped early some "pair"-like metrics still require an increment or observation. That's done in the Drop implementation, by driving the state machine to completion. --- pageserver/src/metrics.rs | 367 +++++++++++------------ pageserver/src/page_service.rs | 139 +++++---- pageserver/src/tenant.rs | 14 +- pageserver/src/tenant/throttle.rs | 45 +-- pageserver/src/tenant/timeline.rs | 8 +- pageserver/src/tenant/timeline/delete.rs | 1 + 6 files changed, 284 insertions(+), 290 deletions(-) diff --git a/pageserver/src/metrics.rs b/pageserver/src/metrics.rs index 5b8419fda9..5b1cbbad63 100644 --- a/pageserver/src/metrics.rs +++ b/pageserver/src/metrics.rs @@ -1224,117 +1224,189 @@ pub(crate) struct SmgrOpTimerInner { global_flush_in_progress_micros: IntCounter, per_timeline_flush_in_progress_micros: IntCounter, + throttling: Arc, + timings: SmgrOpTimerState, } +/// The stages of request processing are represented by the enum variants. +/// Used as part of [`SmgrOpTimerInner::timings`]. +/// +/// Request processing calls into the `SmgrOpTimer::observe_*` methods at the +/// transition points. +/// These methods bump relevant counters and then update [`SmgrOpTimerInner::timings`] +/// to the next state. +/// +/// Each request goes through every stage, in all configurations. +/// #[derive(Debug)] enum SmgrOpTimerState { Received { + // In the future, we may want to track the full time the request spent + // inside pageserver process (time spent in kernel buffers can't be tracked). + // `received_at` would be used for that. + #[allow(dead_code)] received_at: Instant, }, - ThrottleDoneExecutionStarting { - received_at: Instant, + Throttling { throttle_started_at: Instant, - started_execution_at: Instant, }, + Batching { + throttle_done_at: Instant, + }, + Executing { + execution_started_at: Instant, + }, + Flushing, + // NB: when adding observation points, remember to update the Drop impl. } +// NB: when adding observation points, remember to update the Drop impl. +impl SmgrOpTimer { + /// See [`SmgrOpTimerState`] for more context. + pub(crate) fn observe_throttle_start(&mut self, at: Instant) { + let Some(inner) = self.0.as_mut() else { + return; + }; + let SmgrOpTimerState::Received { received_at: _ } = &mut inner.timings else { + return; + }; + inner.throttling.count_accounted_start.inc(); + inner.timings = SmgrOpTimerState::Throttling { + throttle_started_at: at, + }; + } + + /// See [`SmgrOpTimerState`] for more context. + pub(crate) fn observe_throttle_done(&mut self, throttle: ThrottleResult) { + let Some(inner) = self.0.as_mut() else { + return; + }; + let SmgrOpTimerState::Throttling { + throttle_started_at, + } = &inner.timings + else { + return; + }; + inner.throttling.count_accounted_finish.inc(); + match throttle { + ThrottleResult::NotThrottled { end } => { + inner.timings = SmgrOpTimerState::Batching { + throttle_done_at: end, + }; + } + ThrottleResult::Throttled { end } => { + // update metrics + inner.throttling.count_throttled.inc(); + inner + .throttling + .wait_time + .inc_by((end - *throttle_started_at).as_micros().try_into().unwrap()); + // state transition + inner.timings = SmgrOpTimerState::Batching { + throttle_done_at: end, + }; + } + } + } + + /// See [`SmgrOpTimerState`] for more context. + pub(crate) fn observe_execution_start(&mut self, at: Instant) { + let Some(inner) = self.0.as_mut() else { + return; + }; + let SmgrOpTimerState::Batching { throttle_done_at } = &inner.timings else { + return; + }; + // update metrics + let batch = at - *throttle_done_at; + inner.global_batch_wait_time.observe(batch.as_secs_f64()); + inner + .per_timeline_batch_wait_time + .observe(batch.as_secs_f64()); + // state transition + inner.timings = SmgrOpTimerState::Executing { + execution_started_at: at, + } + } + + /// For all but the first caller, this is a no-op. + /// The first callers receives Some, subsequent ones None. + /// + /// See [`SmgrOpTimerState`] for more context. + pub(crate) fn observe_execution_end_flush_start( + &mut self, + at: Instant, + ) -> Option { + // NB: unlike the other observe_* methods, this one take()s. + #[allow(clippy::question_mark)] // maintain similar code pattern. + let Some(mut inner) = self.0.take() else { + return None; + }; + let SmgrOpTimerState::Executing { + execution_started_at, + } = &inner.timings + else { + return None; + }; + // update metrics + let execution = at - *execution_started_at; + inner + .global_execution_latency_histo + .observe(execution.as_secs_f64()); + if let Some(per_timeline_execution_latency_histo) = + &inner.per_timeline_execution_latency_histo + { + per_timeline_execution_latency_histo.observe(execution.as_secs_f64()); + } + + // state transition + inner.timings = SmgrOpTimerState::Flushing; + + // return the flush in progress object which + // will do the remaining metrics updates + let SmgrOpTimerInner { + global_flush_in_progress_micros, + per_timeline_flush_in_progress_micros, + .. + } = inner; + Some(SmgrOpFlushInProgress { + flush_started_at: at, + global_micros: global_flush_in_progress_micros, + per_timeline_micros: per_timeline_flush_in_progress_micros, + }) + } +} + +/// The last stage of request processing is serializing and flushing the request +/// into the TCP connection. We want to make slow flushes observable +/// _while they are occuring_, so this struct provides a wrapper method [`Self::measure`] +/// to periodically bump the metric. +/// +/// If in the future we decide that we're not interested in live updates, we can +/// add another `observe_*` method to [`SmgrOpTimer`], follow the existing pattern there, +/// and remove this struct from the code base. pub(crate) struct SmgrOpFlushInProgress { flush_started_at: Instant, global_micros: IntCounter, per_timeline_micros: IntCounter, } -impl SmgrOpTimer { - pub(crate) fn observe_throttle_done_execution_starting(&mut self, throttle: &ThrottleResult) { - let inner = self.0.as_mut().expect("other public methods consume self"); - match (&mut inner.timings, throttle) { - (SmgrOpTimerState::Received { received_at }, throttle) => match throttle { - ThrottleResult::NotThrottled { start } => { - inner.timings = SmgrOpTimerState::ThrottleDoneExecutionStarting { - received_at: *received_at, - throttle_started_at: *start, - started_execution_at: *start, - }; - } - ThrottleResult::Throttled { start, end } => { - inner.timings = SmgrOpTimerState::ThrottleDoneExecutionStarting { - received_at: *start, - throttle_started_at: *start, - started_execution_at: *end, - }; - } - }, - (x, _) => panic!("called in unexpected state: {x:?}"), - } - } - - pub(crate) fn observe_smgr_op_completion_and_start_flushing(mut self) -> SmgrOpFlushInProgress { - let (flush_start, inner) = self - .smgr_op_end() - .expect("this method consume self, and the only other caller is drop handler"); - let SmgrOpTimerInner { - global_flush_in_progress_micros, - per_timeline_flush_in_progress_micros, - .. - } = inner; - SmgrOpFlushInProgress { - flush_started_at: flush_start, - global_micros: global_flush_in_progress_micros, - per_timeline_micros: per_timeline_flush_in_progress_micros, - } - } - - /// Returns `None`` if this method has already been called, `Some` otherwise. - fn smgr_op_end(&mut self) -> Option<(Instant, SmgrOpTimerInner)> { - let inner = self.0.take()?; - - let now = Instant::now(); - - let batch; - let execution; - let throttle; - match inner.timings { - SmgrOpTimerState::Received { received_at } => { - batch = (now - received_at).as_secs_f64(); - // TODO: use label for dropped requests. - // This is quite rare in practice, only during tenant/pageservers shutdown. - throttle = Duration::ZERO; - execution = Duration::ZERO.as_secs_f64(); - } - SmgrOpTimerState::ThrottleDoneExecutionStarting { - received_at, - throttle_started_at, - started_execution_at, - } => { - batch = (throttle_started_at - received_at).as_secs_f64(); - throttle = started_execution_at - throttle_started_at; - execution = (now - started_execution_at).as_secs_f64(); - } - } - - // update time spent in batching - inner.global_batch_wait_time.observe(batch); - inner.per_timeline_batch_wait_time.observe(batch); - - // time spent in throttle metric is updated by throttle impl - let _ = throttle; - - // update metrics for execution latency - inner.global_execution_latency_histo.observe(execution); - if let Some(per_timeline_execution_latency_histo) = - &inner.per_timeline_execution_latency_histo - { - per_timeline_execution_latency_histo.observe(execution); - } - - Some((now, inner)) - } -} - impl Drop for SmgrOpTimer { fn drop(&mut self) { - self.smgr_op_end(); + // In case of early drop, update any of the remaining metrics with + // observations so that (started,finished) counter pairs balance out + // and all counters on the latency path have the the same number of + // observations. + // It's technically lying and it would be better if each metric had + // a separate label or similar for cancelled requests. + // But we don't have that right now and counter pairs balancing + // out is useful when using the metrics in panels and whatnot. + let now = Instant::now(); + self.observe_throttle_start(now); + self.observe_throttle_done(ThrottleResult::NotThrottled { end: now }); + self.observe_execution_start(now); + self.observe_execution_end_flush_start(now); } } @@ -1345,12 +1417,12 @@ impl SmgrOpFlushInProgress { { let mut fut = std::pin::pin!(fut); - let now = Instant::now(); // Whenever observe_guard gets called, or dropped, // it adds the time elapsed since its last call to metrics. // Last call is tracked in `now`. let mut observe_guard = scopeguard::guard( || { + let now = Instant::now(); let elapsed = now - self.flush_started_at; self.global_micros .inc_by(u64::try_from(elapsed.as_micros()).unwrap()); @@ -1393,7 +1465,6 @@ pub enum SmgrQueryType { GetSlruSegment, } -#[derive(Debug)] pub(crate) struct SmgrQueryTimePerTimeline { global_started: [IntCounter; SmgrQueryType::COUNT], global_latency: [Histogram; SmgrQueryType::COUNT], @@ -1405,6 +1476,7 @@ pub(crate) struct SmgrQueryTimePerTimeline { per_timeline_flush_in_progress_micros: IntCounter, global_batch_wait_time: Histogram, per_timeline_batch_wait_time: Histogram, + throttling: Arc, } static SMGR_QUERY_STARTED_GLOBAL: Lazy = Lazy::new(|| { @@ -1610,7 +1682,11 @@ static PAGE_SERVICE_SMGR_BATCH_WAIT_TIME_GLOBAL: Lazy = Lazy::new(|| }); impl SmgrQueryTimePerTimeline { - pub(crate) fn new(tenant_shard_id: &TenantShardId, timeline_id: &TimelineId) -> Self { + pub(crate) fn new( + tenant_shard_id: &TenantShardId, + timeline_id: &TimelineId, + pagestream_throttle_metrics: Arc, + ) -> Self { let tenant_id = tenant_shard_id.tenant_id.to_string(); let shard_slug = format!("{}", tenant_shard_id.shard_slug()); let timeline_id = timeline_id.to_string(); @@ -1671,6 +1747,7 @@ impl SmgrQueryTimePerTimeline { per_timeline_flush_in_progress_micros, global_batch_wait_time, per_timeline_batch_wait_time, + throttling: pagestream_throttle_metrics, } } pub(crate) fn start_smgr_op(&self, op: SmgrQueryType, received_at: Instant) -> SmgrOpTimer { @@ -1686,88 +1763,24 @@ impl SmgrQueryTimePerTimeline { SmgrOpTimer(Some(SmgrOpTimerInner { global_execution_latency_histo: self.global_latency[op as usize].clone(), per_timeline_execution_latency_histo: per_timeline_latency_histo, - timings: SmgrOpTimerState::Received { received_at }, global_flush_in_progress_micros: self.global_flush_in_progress_micros.clone(), per_timeline_flush_in_progress_micros: self .per_timeline_flush_in_progress_micros .clone(), global_batch_wait_time: self.global_batch_wait_time.clone(), per_timeline_batch_wait_time: self.per_timeline_batch_wait_time.clone(), + throttling: self.throttling.clone(), + timings: SmgrOpTimerState::Received { received_at }, })) } + /// TODO: do something about this? seems odd, we have a similar call on SmgrOpTimer pub(crate) fn observe_getpage_batch_start(&self, batch_size: usize) { self.global_batch_size.observe(batch_size as f64); self.per_timeline_batch_size.observe(batch_size as f64); } } -#[cfg(test)] -mod smgr_query_time_tests { - use std::time::Instant; - - use pageserver_api::shard::TenantShardId; - use strum::IntoEnumIterator; - use utils::id::{TenantId, TimelineId}; - - // Regression test, we used hard-coded string constants before using an enum. - #[test] - fn op_label_name() { - use super::SmgrQueryType::*; - let expect: [(super::SmgrQueryType, &'static str); 5] = [ - (GetRelExists, "get_rel_exists"), - (GetRelSize, "get_rel_size"), - (GetPageAtLsn, "get_page_at_lsn"), - (GetDbSize, "get_db_size"), - (GetSlruSegment, "get_slru_segment"), - ]; - for (op, expect) in expect { - let actual: &'static str = op.into(); - assert_eq!(actual, expect); - } - } - - #[test] - fn basic() { - let ops: Vec<_> = super::SmgrQueryType::iter().collect(); - - for op in &ops { - let tenant_id = TenantId::generate(); - let timeline_id = TimelineId::generate(); - let metrics = super::SmgrQueryTimePerTimeline::new( - &TenantShardId::unsharded(tenant_id), - &timeline_id, - ); - - let get_counts = || { - let global: u64 = ops - .iter() - .map(|op| metrics.global_latency[*op as usize].get_sample_count()) - .sum(); - ( - global, - metrics.per_timeline_getpage_latency.get_sample_count(), - ) - }; - - let (pre_global, pre_per_tenant_timeline) = get_counts(); - assert_eq!(pre_per_tenant_timeline, 0); - - let timer = metrics.start_smgr_op(*op, Instant::now()); - drop(timer); - - let (post_global, post_per_tenant_timeline) = get_counts(); - if matches!(op, super::SmgrQueryType::GetPageAtLsn) { - // getpage ops are tracked per-timeline, others aren't - assert_eq!(post_per_tenant_timeline, 1); - } else { - assert_eq!(post_per_tenant_timeline, 0); - } - assert!(post_global > pre_global); - } - } -} - // keep in sync with control plane Go code so that we can validate // compute's basebackup_ms metric with our perspective in the context of SLI/SLO. static COMPUTE_STARTUP_BUCKETS: Lazy<[f64; 28]> = Lazy::new(|| { @@ -3563,9 +3576,7 @@ pub(crate) mod tenant_throttling { use once_cell::sync::Lazy; use utils::shard::TenantShardId; - use crate::tenant::{self}; - - struct GlobalAndPerTenantIntCounter { + pub(crate) struct GlobalAndPerTenantIntCounter { global: IntCounter, per_tenant: IntCounter, } @@ -3583,10 +3594,10 @@ pub(crate) mod tenant_throttling { } pub(crate) struct Metrics { - count_accounted_start: GlobalAndPerTenantIntCounter, - count_accounted_finish: GlobalAndPerTenantIntCounter, - wait_time: GlobalAndPerTenantIntCounter, - count_throttled: GlobalAndPerTenantIntCounter, + pub(super) count_accounted_start: GlobalAndPerTenantIntCounter, + pub(super) count_accounted_finish: GlobalAndPerTenantIntCounter, + pub(super) wait_time: GlobalAndPerTenantIntCounter, + pub(super) count_throttled: GlobalAndPerTenantIntCounter, } static COUNT_ACCOUNTED_START: Lazy = Lazy::new(|| { @@ -3721,26 +3732,6 @@ pub(crate) mod tenant_throttling { } } } - - impl tenant::throttle::Metric for Metrics { - #[inline(always)] - fn accounting_start(&self) { - self.count_accounted_start.inc(); - } - #[inline(always)] - fn accounting_finish(&self) { - self.count_accounted_finish.inc(); - } - #[inline(always)] - fn observe_throttling( - &self, - tenant::throttle::Observation { wait_time }: &tenant::throttle::Observation, - ) { - let val = u64::try_from(wait_time.as_micros()).unwrap(); - self.wait_time.inc_by(val); - self.count_throttled.inc(); - } - } } pub(crate) mod disk_usage_based_eviction { diff --git a/pageserver/src/page_service.rs b/pageserver/src/page_service.rs index f6504bd3b5..b3e18fed99 100644 --- a/pageserver/src/page_service.rs +++ b/pageserver/src/page_service.rs @@ -592,43 +592,21 @@ enum BatchedFeMessage { } impl BatchedFeMessage { - async fn throttle_and_record_start_processing( - &mut self, - cancel: &CancellationToken, - ) -> Result<(), QueryError> { - let (shard, tokens, timers) = match self { - BatchedFeMessage::Exists { shard, timer, .. } - | BatchedFeMessage::Nblocks { shard, timer, .. } - | BatchedFeMessage::DbSize { shard, timer, .. } - | BatchedFeMessage::GetSlruSegment { shard, timer, .. } => { - ( - shard, - // 1 token is probably under-estimating because these - // request handlers typically do several Timeline::get calls. - 1, - itertools::Either::Left(std::iter::once(timer)), - ) + fn observe_execution_start(&mut self, at: Instant) { + match self { + BatchedFeMessage::Exists { timer, .. } + | BatchedFeMessage::Nblocks { timer, .. } + | BatchedFeMessage::DbSize { timer, .. } + | BatchedFeMessage::GetSlruSegment { timer, .. } => { + timer.observe_execution_start(at); } - BatchedFeMessage::GetPage { shard, pages, .. } => ( - shard, - pages.len(), - itertools::Either::Right(pages.iter_mut().map(|p| &mut p.timer)), - ), - BatchedFeMessage::RespondError { .. } => return Ok(()), - }; - let throttled = tokio::select! { - throttled = shard.pagestream_throttle.throttle(tokens) => { throttled } - _ = shard.cancel.cancelled() => { - return Err(QueryError::Shutdown); + BatchedFeMessage::GetPage { pages, .. } => { + for page in pages { + page.timer.observe_execution_start(at); + } } - _ = cancel.cancelled() => { - return Err(QueryError::Shutdown); - } - }; - for timer in timers { - timer.observe_throttle_done_execution_starting(&throttled); + BatchedFeMessage::RespondError { .. } => {} } - Ok(()) } } @@ -720,6 +698,26 @@ impl PageServerHandler { let neon_fe_msg = PagestreamFeMessage::parse(&mut copy_data_bytes.reader(), protocol_version)?; + // TODO: turn in to async closure once available to avoid repeating received_at + async fn record_op_start_and_throttle( + shard: &timeline::handle::Handle, + op: metrics::SmgrQueryType, + received_at: Instant, + ) -> Result { + // It's important to start the smgr op metric recorder as early as possible + // so that the _started counters are incremented before we do + // any serious waiting, e.g., for throttle, batching, or actual request handling. + let mut timer = shard.query_metrics.start_smgr_op(op, received_at); + let now = Instant::now(); + timer.observe_throttle_start(now); + let throttled = tokio::select! { + res = shard.pagestream_throttle.throttle(1, now) => res, + _ = shard.cancel.cancelled() => return Err(QueryError::Shutdown), + }; + timer.observe_throttle_done(throttled); + Ok(timer) + } + let batched_msg = match neon_fe_msg { PagestreamFeMessage::Exists(req) => { let span = tracing::info_span!(parent: parent_span, "handle_get_rel_exists_request", rel = %req.rel, req_lsn = %req.hdr.request_lsn); @@ -727,9 +725,12 @@ impl PageServerHandler { .get(tenant_id, timeline_id, ShardSelector::Zero) .instrument(span.clone()) // sets `shard_id` field .await?; - let timer = shard - .query_metrics - .start_smgr_op(metrics::SmgrQueryType::GetRelExists, received_at); + let timer = record_op_start_and_throttle( + &shard, + metrics::SmgrQueryType::GetRelExists, + received_at, + ) + .await?; BatchedFeMessage::Exists { span, timer, @@ -743,9 +744,12 @@ impl PageServerHandler { .get(tenant_id, timeline_id, ShardSelector::Zero) .instrument(span.clone()) // sets `shard_id` field .await?; - let timer = shard - .query_metrics - .start_smgr_op(metrics::SmgrQueryType::GetRelSize, received_at); + let timer = record_op_start_and_throttle( + &shard, + metrics::SmgrQueryType::GetRelSize, + received_at, + ) + .await?; BatchedFeMessage::Nblocks { span, timer, @@ -759,9 +763,12 @@ impl PageServerHandler { .get(tenant_id, timeline_id, ShardSelector::Zero) .instrument(span.clone()) // sets `shard_id` field .await?; - let timer = shard - .query_metrics - .start_smgr_op(metrics::SmgrQueryType::GetDbSize, received_at); + let timer = record_op_start_and_throttle( + &shard, + metrics::SmgrQueryType::GetDbSize, + received_at, + ) + .await?; BatchedFeMessage::DbSize { span, timer, @@ -775,9 +782,12 @@ impl PageServerHandler { .get(tenant_id, timeline_id, ShardSelector::Zero) .instrument(span.clone()) // sets `shard_id` field .await?; - let timer = shard - .query_metrics - .start_smgr_op(metrics::SmgrQueryType::GetSlruSegment, received_at); + let timer = record_op_start_and_throttle( + &shard, + metrics::SmgrQueryType::GetSlruSegment, + received_at, + ) + .await?; BatchedFeMessage::GetSlruSegment { span, timer, @@ -826,12 +836,12 @@ impl PageServerHandler { } }; - // It's important to start the timer before waiting for the LSN - // so that the _started counters are incremented before we do - // any serious waiting, e.g., for LSNs. - let timer = shard - .query_metrics - .start_smgr_op(metrics::SmgrQueryType::GetPageAtLsn, received_at); + let timer = record_op_start_and_throttle( + &shard, + metrics::SmgrQueryType::GetPageAtLsn, + received_at, + ) + .await?; let effective_request_lsn = match Self::wait_or_get_last_lsn( &shard, @@ -937,6 +947,13 @@ impl PageServerHandler { where IO: AsyncRead + AsyncWrite + Send + Sync + Unpin, { + let started_at = Instant::now(); + let batch = { + let mut batch = batch; + batch.observe_execution_start(started_at); + batch + }; + // invoke handler function let (handler_results, span): ( Vec>, @@ -1103,8 +1120,11 @@ impl PageServerHandler { // The timer's underlying metric is used for a storage-internal latency SLO and // we don't want to include latency in it that we can't control. // And as pointed out above, in this case, we don't control the time that flush will take. - let flushing_timer = - timer.map(|timer| timer.observe_smgr_op_completion_and_start_flushing()); + let flushing_timer = timer.map(|mut timer| { + timer + .observe_execution_end_flush_start(Instant::now()) + .expect("we are the first caller") + }); // what we want to do let flush_fut = pgb_writer.flush(); @@ -1258,7 +1278,7 @@ impl PageServerHandler { Ok(msg) => msg, Err(e) => break e, }; - let mut msg = match msg { + let msg = match msg { Some(msg) => msg, None => { debug!("pagestream subprotocol end observed"); @@ -1266,10 +1286,6 @@ impl PageServerHandler { } }; - if let Err(cancelled) = msg.throttle_and_record_start_processing(&self.cancel).await { - break cancelled; - } - let err = self .pagesteam_handle_batched_message(pgb_writer, msg, &cancel, protocol_version, ctx) .await; @@ -1429,15 +1445,12 @@ impl PageServerHandler { return Ok(()); } }; - let mut batch = match batch { + let batch = match batch { Ok(batch) => batch, Err(e) => { return Err(e); } }; - batch - .throttle_and_record_start_processing(&self.cancel) - .await?; self.pagesteam_handle_batched_message( pgb_writer, batch, diff --git a/pageserver/src/tenant.rs b/pageserver/src/tenant.rs index 070593b104..f6d758ad22 100644 --- a/pageserver/src/tenant.rs +++ b/pageserver/src/tenant.rs @@ -365,8 +365,9 @@ pub struct Tenant { /// Throttle applied at the top of [`Timeline::get`]. /// All [`Tenant::timelines`] of a given [`Tenant`] instance share the same [`throttle::Throttle`] instance. - pub(crate) pagestream_throttle: - Arc>, + pub(crate) pagestream_throttle: Arc, + + pub(crate) pagestream_throttle_metrics: Arc, /// An ongoing timeline detach concurrency limiter. /// @@ -1687,6 +1688,7 @@ impl Tenant { TimelineResources { remote_client, pagestream_throttle: self.pagestream_throttle.clone(), + pagestream_throttle_metrics: self.pagestream_throttle_metrics.clone(), l0_flush_global_state: self.l0_flush_global_state.clone(), }, LoadTimelineCause::Attach, @@ -3992,6 +3994,9 @@ impl Tenant { Ok(timeline) } + /// [`Tenant::shutdown`] must be called before dropping the returned [`Tenant`] object + /// to ensure proper cleanup of background tasks and metrics. + // // Allow too_many_arguments because a constructor's argument list naturally grows with the // number of attributes in the struct: breaking these out into a builder wouldn't be helpful. #[allow(clippy::too_many_arguments)] @@ -4100,8 +4105,10 @@ impl Tenant { gate: Gate::default(), pagestream_throttle: Arc::new(throttle::Throttle::new( Tenant::get_pagestream_throttle_config(conf, &attached_conf.tenant_conf), - crate::metrics::tenant_throttling::Metrics::new(&tenant_shard_id), )), + pagestream_throttle_metrics: Arc::new( + crate::metrics::tenant_throttling::Pagestream::new(&tenant_shard_id), + ), tenant_conf: Arc::new(ArcSwap::from_pointee(attached_conf)), ongoing_timeline_detach: std::sync::Mutex::default(), gc_block: Default::default(), @@ -5008,6 +5015,7 @@ impl Tenant { TimelineResources { remote_client: self.build_timeline_remote_client(timeline_id), pagestream_throttle: self.pagestream_throttle.clone(), + pagestream_throttle_metrics: self.pagestream_throttle_metrics.clone(), l0_flush_global_state: self.l0_flush_global_state.clone(), } } diff --git a/pageserver/src/tenant/throttle.rs b/pageserver/src/tenant/throttle.rs index 8ab6a0e060..300d779125 100644 --- a/pageserver/src/tenant/throttle.rs +++ b/pageserver/src/tenant/throttle.rs @@ -3,7 +3,7 @@ use std::{ atomic::{AtomicU64, Ordering}, Arc, }, - time::{Duration, Instant}, + time::Instant, }; use arc_swap::ArcSwap; @@ -16,9 +16,8 @@ use utils::leaky_bucket::{LeakyBucketConfig, RateLimiter}; /// To share a throttle among multiple entities, wrap it in an [`Arc`]. /// /// The intial use case for this is tenant-wide throttling of getpage@lsn requests. -pub struct Throttle { +pub struct Throttle { inner: ArcSwap, - metric: M, /// will be turned into [`Stats::count_accounted_start`] count_accounted_start: AtomicU64, /// will be turned into [`Stats::count_accounted_finish`] @@ -36,15 +35,6 @@ pub struct Inner { pub type Config = pageserver_api::models::ThrottleConfig; -pub struct Observation { - pub wait_time: Duration, -} -pub trait Metric { - fn accounting_start(&self); - fn accounting_finish(&self); - fn observe_throttling(&self, observation: &Observation); -} - /// See [`Throttle::reset_stats`]. pub struct Stats { /// Number of requests that started [`Throttle::throttle`] calls. @@ -59,18 +49,14 @@ pub struct Stats { } pub enum ThrottleResult { - NotThrottled { start: Instant }, - Throttled { start: Instant, end: Instant }, + NotThrottled { end: Instant }, + Throttled { end: Instant }, } -impl Throttle -where - M: Metric, -{ - pub fn new(config: Config, metric: M) -> Self { +impl Throttle { + pub fn new(config: Config) -> Self { Self { inner: ArcSwap::new(Arc::new(Self::new_inner(config))), - metric, count_accounted_start: AtomicU64::new(0), count_accounted_finish: AtomicU64::new(0), count_throttled: AtomicU64::new(0), @@ -127,32 +113,27 @@ where self.inner.load().rate_limiter.steady_rps() } - pub async fn throttle(&self, key_count: usize) -> ThrottleResult { + /// `start` must be [`Instant::now`] or earlier. + pub async fn throttle(&self, key_count: usize, start: Instant) -> ThrottleResult { let inner = self.inner.load_full(); // clones the `Inner` Arc - let start = std::time::Instant::now(); - if !inner.enabled { - return ThrottleResult::NotThrottled { start }; + return ThrottleResult::NotThrottled { end: start }; } - self.metric.accounting_start(); self.count_accounted_start.fetch_add(1, Ordering::Relaxed); let did_throttle = inner.rate_limiter.acquire(key_count).await; self.count_accounted_finish.fetch_add(1, Ordering::Relaxed); - self.metric.accounting_finish(); if did_throttle { self.count_throttled.fetch_add(1, Ordering::Relaxed); - let now = Instant::now(); - let wait_time = now - start; + let end = Instant::now(); + let wait_time = end - start; self.sum_throttled_usecs .fetch_add(wait_time.as_micros() as u64, Ordering::Relaxed); - let observation = Observation { wait_time }; - self.metric.observe_throttling(&observation); - ThrottleResult::Throttled { start, end: now } + ThrottleResult::Throttled { end } } else { - ThrottleResult::NotThrottled { start } + ThrottleResult::NotThrottled { end: start } } } } diff --git a/pageserver/src/tenant/timeline.rs b/pageserver/src/tenant/timeline.rs index 741b214a73..4aa6b7a05a 100644 --- a/pageserver/src/tenant/timeline.rs +++ b/pageserver/src/tenant/timeline.rs @@ -208,8 +208,8 @@ fn drop_wlock(rlock: tokio::sync::RwLockWriteGuard<'_, T>) { /// The outward-facing resources required to build a Timeline pub struct TimelineResources { pub remote_client: RemoteTimelineClient, - pub pagestream_throttle: - Arc>, + pub pagestream_throttle: Arc, + pub pagestream_throttle_metrics: Arc, pub l0_flush_global_state: l0_flush::L0FlushGlobalState, } @@ -412,8 +412,7 @@ pub struct Timeline { gc_lock: tokio::sync::Mutex<()>, /// Cloned from [`super::Tenant::pagestream_throttle`] on construction. - pub(crate) pagestream_throttle: - Arc>, + pub(crate) pagestream_throttle: Arc, /// Size estimator for aux file v2 pub(crate) aux_file_size_estimator: AuxFileSizeEstimator, @@ -2310,6 +2309,7 @@ impl Timeline { query_metrics: crate::metrics::SmgrQueryTimePerTimeline::new( &tenant_shard_id, &timeline_id, + resources.pagestream_throttle_metrics, ), directory_metrics: array::from_fn(|_| AtomicU64::new(0)), diff --git a/pageserver/src/tenant/timeline/delete.rs b/pageserver/src/tenant/timeline/delete.rs index ae44af3fad..bdc315d985 100644 --- a/pageserver/src/tenant/timeline/delete.rs +++ b/pageserver/src/tenant/timeline/delete.rs @@ -301,6 +301,7 @@ impl DeleteTimelineFlow { TimelineResources { remote_client, pagestream_throttle: tenant.pagestream_throttle.clone(), + pagestream_throttle_metrics: tenant.pagestream_throttle_metrics.clone(), l0_flush_global_state: tenant.l0_flush_global_state.clone(), }, // Important. We dont pass ancestor above because it can be missing. From aa7323a384dd60b8c176ba6bed6a3c2e5f9bd95e Mon Sep 17 00:00:00 2001 From: John Spray Date: Tue, 14 Jan 2025 15:30:43 +0000 Subject: [PATCH 095/111] storage controller: quality of life improvements for AZ handling (#10379) ## Problem Since https://github.com/neondatabase/neon/pull/9916, the preferred AZ of a tenant is much more impactful, and we would like to make it more visible in tooling. ## Summary of changes - Include AZ in node describe API - Include AZ info in node & tenant outputs in CLI - Add metrics for per-node shard counts, labelled by AZ - Add a CLI for setting preferred AZ on a tenant - Extend AZ-setting API+CLI to handle None for clearing preferred AZ --- control_plane/storcon_cli/src/main.rs | 120 ++++++++++++++++++++-- libs/pageserver_api/src/controller_api.rs | 4 +- storage_controller/src/metrics.rs | 19 ++++ storage_controller/src/node.rs | 1 + storage_controller/src/persistence.rs | 7 +- storage_controller/src/scheduler.rs | 29 +++++- storage_controller/src/service.rs | 9 +- storage_controller/src/tenant_shard.rs | 4 +- 8 files changed, 175 insertions(+), 18 deletions(-) diff --git a/control_plane/storcon_cli/src/main.rs b/control_plane/storcon_cli/src/main.rs index 1653b3c845..9d133e4af1 100644 --- a/control_plane/storcon_cli/src/main.rs +++ b/control_plane/storcon_cli/src/main.rs @@ -1,12 +1,16 @@ use futures::StreamExt; -use std::{str::FromStr, time::Duration}; +use std::{ + collections::{HashMap, HashSet}, + str::FromStr, + time::Duration, +}; use clap::{Parser, Subcommand}; use pageserver_api::{ controller_api::{ AvailabilityZone, NodeAvailabilityWrapper, NodeDescribeResponse, NodeShardResponse, - SafekeeperDescribeResponse, ShardSchedulingPolicy, TenantCreateRequest, - TenantDescribeResponse, TenantPolicyRequest, + SafekeeperDescribeResponse, ShardSchedulingPolicy, ShardsPreferredAzsRequest, + TenantCreateRequest, TenantDescribeResponse, TenantPolicyRequest, }, models::{ EvictionPolicy, EvictionPolicyLayerAccessThreshold, LocationConfigSecondary, @@ -153,6 +157,12 @@ enum Command { #[arg(long)] tenant_id: TenantId, }, + TenantSetPreferredAz { + #[arg(long)] + tenant_id: TenantId, + #[arg(long)] + preferred_az: Option, + }, /// Uncleanly drop a tenant from the storage controller: this doesn't delete anything from pageservers. Appropriate /// if you e.g. used `tenant-warmup` by mistake on a tenant ID that doesn't really exist, or is in some other region. TenantDrop { @@ -402,11 +412,12 @@ async fn main() -> anyhow::Result<()> { resp.sort_by(|a, b| a.listen_http_addr.cmp(&b.listen_http_addr)); let mut table = comfy_table::Table::new(); - table.set_header(["Id", "Hostname", "Scheduling", "Availability"]); + table.set_header(["Id", "Hostname", "AZ", "Scheduling", "Availability"]); for node in resp { table.add_row([ format!("{}", node.id), node.listen_http_addr, + node.availability_zone_id, format!("{:?}", node.scheduling), format!("{:?}", node.availability), ]); @@ -479,6 +490,7 @@ async fn main() -> anyhow::Result<()> { let mut table = comfy_table::Table::new(); table.set_header([ "TenantId", + "Preferred AZ", "ShardCount", "StripeSize", "Placement", @@ -488,6 +500,11 @@ async fn main() -> anyhow::Result<()> { let shard_zero = tenant.shards.into_iter().next().unwrap(); table.add_row([ format!("{}", tenant.tenant_id), + shard_zero + .preferred_az_id + .as_ref() + .cloned() + .unwrap_or("".to_string()), format!("{}", shard_zero.tenant_shard_id.shard_count.literal()), format!("{:?}", tenant.stripe_size), format!("{:?}", tenant.policy), @@ -614,6 +631,19 @@ async fn main() -> anyhow::Result<()> { None, ) .await?; + + let nodes = storcon_client + .dispatch::<(), Vec>( + Method::GET, + "control/v1/node".to_string(), + None, + ) + .await?; + let nodes = nodes + .into_iter() + .map(|n| (n.id, n)) + .collect::>(); + println!("Tenant {tenant_id}"); let mut table = comfy_table::Table::new(); table.add_row(["Policy", &format!("{:?}", policy)]); @@ -622,7 +652,14 @@ async fn main() -> anyhow::Result<()> { println!("{table}"); println!("Shards:"); let mut table = comfy_table::Table::new(); - table.set_header(["Shard", "Attached", "Secondary", "Last error", "status"]); + table.set_header([ + "Shard", + "Attached", + "Attached AZ", + "Secondary", + "Last error", + "status", + ]); for shard in shards { let secondary = shard .node_secondary @@ -645,11 +682,18 @@ async fn main() -> anyhow::Result<()> { } let status = status_parts.join(","); + let attached_node = shard + .node_attached + .as_ref() + .map(|id| nodes.get(id).expect("Shard references nonexistent node")); + table.add_row([ format!("{}", shard.tenant_shard_id), - shard - .node_attached - .map(|n| format!("{}", n)) + attached_node + .map(|n| format!("{} ({})", n.listen_http_addr, n.id)) + .unwrap_or(String::new()), + attached_node + .map(|n| n.availability_zone_id.clone()) .unwrap_or(String::new()), secondary, shard.last_error, @@ -658,6 +702,66 @@ async fn main() -> anyhow::Result<()> { } println!("{table}"); } + Command::TenantSetPreferredAz { + tenant_id, + preferred_az, + } => { + // First learn about the tenant's shards + let describe_response = storcon_client + .dispatch::<(), TenantDescribeResponse>( + Method::GET, + format!("control/v1/tenant/{tenant_id}"), + None, + ) + .await?; + + // Learn about nodes to validate the AZ ID + let nodes = storcon_client + .dispatch::<(), Vec>( + Method::GET, + "control/v1/node".to_string(), + None, + ) + .await?; + + if let Some(preferred_az) = &preferred_az { + let azs = nodes + .into_iter() + .map(|n| (n.availability_zone_id)) + .collect::>(); + if !azs.contains(preferred_az) { + anyhow::bail!( + "AZ {} not found on any node: known AZs are: {:?}", + preferred_az, + azs + ); + } + } else { + // Make it obvious to the user that since they've omitted an AZ, we're clearing it + eprintln!("Clearing preferred AZ for tenant {}", tenant_id); + } + + // Construct a request that modifies all the tenant's shards + let req = ShardsPreferredAzsRequest { + preferred_az_ids: describe_response + .shards + .into_iter() + .map(|s| { + ( + s.tenant_shard_id, + preferred_az.clone().map(AvailabilityZone), + ) + }) + .collect(), + }; + storcon_client + .dispatch::( + Method::PUT, + "control/v1/preferred_azs".to_string(), + Some(req), + ) + .await?; + } Command::TenantWarmup { tenant_id } => { let describe_response = storcon_client .dispatch::<(), TenantDescribeResponse>( diff --git a/libs/pageserver_api/src/controller_api.rs b/libs/pageserver_api/src/controller_api.rs index f3aefc6df9..f3880cb766 100644 --- a/libs/pageserver_api/src/controller_api.rs +++ b/libs/pageserver_api/src/controller_api.rs @@ -87,7 +87,7 @@ impl Display for AvailabilityZone { #[derive(Serialize, Deserialize)] pub struct ShardsPreferredAzsRequest { #[serde(flatten)] - pub preferred_az_ids: HashMap, + pub preferred_az_ids: HashMap>, } #[derive(Serialize, Deserialize)] @@ -144,6 +144,8 @@ pub struct NodeDescribeResponse { pub availability: NodeAvailabilityWrapper, pub scheduling: NodeSchedulingPolicy, + pub availability_zone_id: String, + pub listen_http_addr: String, pub listen_http_port: u16, diff --git a/storage_controller/src/metrics.rs b/storage_controller/src/metrics.rs index 6d5885eba6..4164e3dc2b 100644 --- a/storage_controller/src/metrics.rs +++ b/storage_controller/src/metrics.rs @@ -53,6 +53,16 @@ pub(crate) struct StorageControllerMetricGroup { /// How many shards are not scheduled into their preferred AZ pub(crate) storage_controller_schedule_az_violation: measured::Gauge, + /// How many shard locations (secondary or attached) on each node + pub(crate) storage_controller_node_shards: measured::GaugeVec, + + /// How many _attached_ shard locations on each node + pub(crate) storage_controller_node_attached_shards: measured::GaugeVec, + + /// How many _home_ shard locations on each node (i.e. the node's AZ matches the shard's + /// preferred AZ) + pub(crate) storage_controller_node_home_shards: measured::GaugeVec, + /// How many shards would like to reconcile but were blocked by concurrency limits pub(crate) storage_controller_pending_reconciles: measured::Gauge, @@ -132,6 +142,15 @@ impl Default for StorageControllerMetrics { } } +#[derive(measured::LabelGroup, Clone)] +#[label(set = NodeLabelGroupSet)] +pub(crate) struct NodeLabelGroup<'a> { + #[label(dynamic_with = lasso::ThreadedRodeo, default)] + pub(crate) az: &'a str, + #[label(dynamic_with = lasso::ThreadedRodeo, default)] + pub(crate) node_id: &'a str, +} + #[derive(measured::LabelGroup)] #[label(set = ReconcileCompleteLabelGroupSet)] pub(crate) struct ReconcileCompleteLabelGroup { diff --git a/storage_controller/src/node.rs b/storage_controller/src/node.rs index 4cc9b0070d..f5c2d329e0 100644 --- a/storage_controller/src/node.rs +++ b/storage_controller/src/node.rs @@ -299,6 +299,7 @@ impl Node { id: self.id, availability: self.availability.clone().into(), scheduling: self.scheduling, + availability_zone_id: self.availability_zone_id.0.clone(), listen_http_addr: self.listen_http_addr.clone(), listen_http_port: self.listen_http_port, listen_pg_addr: self.listen_pg_addr.clone(), diff --git a/storage_controller/src/persistence.rs b/storage_controller/src/persistence.rs index beb014f0a8..eb0bfc879e 100644 --- a/storage_controller/src/persistence.rs +++ b/storage_controller/src/persistence.rs @@ -708,10 +708,11 @@ impl Persistence { Ok(()) } + /// Note that passing None for a shard clears the preferred AZ (rather than leaving it unmodified) pub(crate) async fn set_tenant_shard_preferred_azs( &self, - preferred_azs: Vec<(TenantShardId, AvailabilityZone)>, - ) -> DatabaseResult> { + preferred_azs: Vec<(TenantShardId, Option)>, + ) -> DatabaseResult)>> { use crate::schema::tenant_shards::dsl::*; self.with_measured_conn(DatabaseOperation::SetPreferredAzs, move |conn| { @@ -722,7 +723,7 @@ impl Persistence { .filter(tenant_id.eq(tenant_shard_id.tenant_id.to_string())) .filter(shard_number.eq(tenant_shard_id.shard_number.0 as i32)) .filter(shard_count.eq(tenant_shard_id.shard_count.literal() as i32)) - .set(preferred_az_id.eq(preferred_az.0.clone())) + .set(preferred_az_id.eq(preferred_az.as_ref().map(|az| az.0.clone()))) .execute(conn)?; if updated == 1 { diff --git a/storage_controller/src/scheduler.rs b/storage_controller/src/scheduler.rs index 04a594dcac..f5cab9dd57 100644 --- a/storage_controller/src/scheduler.rs +++ b/storage_controller/src/scheduler.rs @@ -1,4 +1,4 @@ -use crate::{node::Node, tenant_shard::TenantShard}; +use crate::{metrics::NodeLabelGroup, node::Node, tenant_shard::TenantShard}; use itertools::Itertools; use pageserver_api::{controller_api::AvailabilityZone, models::PageserverUtilization}; use serde::Serialize; @@ -872,6 +872,33 @@ impl Scheduler { pub(crate) fn get_node_attached_shard_count(&self, node_id: NodeId) -> usize { self.nodes.get(&node_id).unwrap().attached_shard_count } + + /// Some metrics that we only calculate periodically: this is simpler than + /// rigorously updating them on every change. + pub(crate) fn update_metrics(&self) { + for (node_id, node) in &self.nodes { + let node_id_str = format!("{}", node_id); + let label_group = NodeLabelGroup { + az: &node.az.0, + node_id: &node_id_str, + }; + + crate::metrics::METRICS_REGISTRY + .metrics_group + .storage_controller_node_shards + .set(label_group.clone(), node.shard_count as i64); + + crate::metrics::METRICS_REGISTRY + .metrics_group + .storage_controller_node_attached_shards + .set(label_group.clone(), node.attached_shard_count as i64); + + crate::metrics::METRICS_REGISTRY + .metrics_group + .storage_controller_node_home_shards + .set(label_group.clone(), node.home_shard_count as i64); + } + } } #[cfg(test)] diff --git a/storage_controller/src/service.rs b/storage_controller/src/service.rs index dadcc44cfb..cbb9103880 100644 --- a/storage_controller/src/service.rs +++ b/storage_controller/src/service.rs @@ -2517,7 +2517,7 @@ impl Service { .map(|t| { ( t.get_tenant_shard_id().expect("Corrupt shard in database"), - load_in_az.clone(), + Some(load_in_az.clone()), ) }) .collect(), @@ -6390,7 +6390,7 @@ impl Service { /// available. A return value of 0 indicates that everything is fully reconciled already. fn reconcile_all(&self) -> usize { let mut locked = self.inner.write().unwrap(); - let (nodes, tenants, _scheduler) = locked.parts_mut(); + let (nodes, tenants, scheduler) = locked.parts_mut(); let pageservers = nodes.clone(); // This function is an efficient place to update lazy statistics, since we are walking @@ -6451,6 +6451,9 @@ impl Service { } } + // Some metrics are calculated from SchedulerNode state, update these periodically + scheduler.update_metrics(); + // Process any deferred tenant drops for (tenant_id, guard) in drop_detached_tenants { self.maybe_drop_tenant(tenant_id, &mut locked, &guard); @@ -6509,7 +6512,7 @@ impl Service { // Shard was dropped between planning and execution; continue; }; - tracing::info!("Applying optimization: {optimization:?}"); + tracing::info!(tenant_shard_id=%tenant_shard_id, "Applying optimization: {optimization:?}"); if shard.apply_optimization(scheduler, optimization) { optimizations_applied += 1; if self.maybe_reconcile_shard(shard, nodes).is_some() { diff --git a/storage_controller/src/tenant_shard.rs b/storage_controller/src/tenant_shard.rs index 2ba2a57eba..79ed628c25 100644 --- a/storage_controller/src/tenant_shard.rs +++ b/storage_controller/src/tenant_shard.rs @@ -1708,8 +1708,8 @@ impl TenantShard { self.intent.preferred_az_id.as_ref() } - pub(crate) fn set_preferred_az(&mut self, preferred_az_id: AvailabilityZone) { - self.intent.preferred_az_id = Some(preferred_az_id); + pub(crate) fn set_preferred_az(&mut self, preferred_az_id: Option) { + self.intent.preferred_az_id = preferred_az_id; } /// Returns all the nodes to which this tenant shard is attached according to the From ffaa52ff5d3f36becaf70be2b6053c7423deba61 Mon Sep 17 00:00:00 2001 From: Erik Grinaker Date: Tue, 14 Jan 2025 17:31:59 +0100 Subject: [PATCH 096/111] pageserver: reorder upload queue when possible (#10218) ## Problem The upload queue currently sees significant head-of-line blocking. For example, index uploads act as upload barriers, and for every layer flush we schedule a layer and index upload, which effectively serializes layer uploads. Resolves #10096. ## Summary of changes Allow upload queue operations to bypass the queue if they don't conflict with preceding operations, increasing parallelism. NB: the upload queue currently schedules an explicit barrier after every layer flush as well (see #8550). This must be removed to enable parallelism. This will require a better mechanism for compaction backpressure, see e.g. #8390 or #5415. --- Cargo.lock | 1 + pageserver/Cargo.toml | 5 + pageserver/benches/upload_queue.rs | 86 ++ pageserver/src/tenant/metadata.rs | 1 - .../src/tenant/remote_timeline_client.rs | 101 +- .../tenant/remote_timeline_client/index.rs | 2 +- pageserver/src/tenant/storage_layer/layer.rs | 2 +- pageserver/src/tenant/upload_queue.rs | 957 +++++++++++++++++- 8 files changed, 1029 insertions(+), 126 deletions(-) create mode 100644 pageserver/benches/upload_queue.rs diff --git a/Cargo.lock b/Cargo.lock index 08453120c7..1e29f4fc08 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4044,6 +4044,7 @@ dependencies = [ "postgres_connection", "postgres_ffi", "postgres_initdb", + "pprof", "pq_proto", "procfs", "rand 0.8.5", diff --git a/pageserver/Cargo.toml b/pageserver/Cargo.toml index 140b287ccc..8547746d94 100644 --- a/pageserver/Cargo.toml +++ b/pageserver/Cargo.toml @@ -44,6 +44,7 @@ postgres_backend.workspace = true postgres-protocol.workspace = true postgres-types.workspace = true postgres_initdb.workspace = true +pprof.workspace = true rand.workspace = true range-set-blaze = { version = "0.1.16", features = ["alloc"] } regex.workspace = true @@ -108,3 +109,7 @@ harness = false [[bench]] name = "bench_ingest" harness = false + +[[bench]] +name = "upload_queue" +harness = false diff --git a/pageserver/benches/upload_queue.rs b/pageserver/benches/upload_queue.rs new file mode 100644 index 0000000000..528b3d5490 --- /dev/null +++ b/pageserver/benches/upload_queue.rs @@ -0,0 +1,86 @@ +//! Upload queue benchmarks. + +use std::str::FromStr as _; +use std::sync::atomic::AtomicU32; +use std::sync::Arc; + +use criterion::{criterion_group, criterion_main, Bencher, Criterion}; +use pageserver::tenant::metadata::TimelineMetadata; +use pageserver::tenant::remote_timeline_client::index::LayerFileMetadata; +use pageserver::tenant::storage_layer::LayerName; +use pageserver::tenant::upload_queue::{Delete, UploadOp, UploadQueue, UploadTask}; +use pageserver::tenant::IndexPart; +use pprof::criterion::{Output, PProfProfiler}; +use utils::generation::Generation; +use utils::shard::{ShardCount, ShardIndex, ShardNumber}; + +// Register benchmarks with Criterion. +criterion_group!( + name = benches; + config = Criterion::default().with_profiler(PProfProfiler::new(100, Output::Flamegraph(None))); + targets = bench_upload_queue_next_ready, +); +criterion_main!(benches); + +/// Benchmarks the cost of UploadQueue::next_ready() with the given number of in-progress tasks +/// (which is equivalent to tasks ahead of it in the queue). This has linear cost, and the upload +/// queue as a whole is thus quadratic. +/// +/// UploadOp::UploadLayer requires an entire tenant and timeline to construct, so we just test +/// Delete and UploadMetadata instead. This is incidentally the most expensive case. +fn bench_upload_queue_next_ready(c: &mut Criterion) { + let mut g = c.benchmark_group("upload_queue_next_ready"); + for inprogress in [0, 1, 10, 100, 1_000, 10_000, 100_000, 1_000_000] { + g.bench_function(format!("inprogress={inprogress}"), |b| { + run_bench(b, inprogress).unwrap() + }); + } + + fn run_bench(b: &mut Bencher, inprogress: usize) -> anyhow::Result<()> { + // Construct two layers. layer0 is in the indexes, layer1 will be deleted. + let layer0 = LayerName::from_str("000000000000000000000000000000000000-100000000000000000000000000000000000__00000000016B59D8-00000000016B5A51").expect("invalid name"); + let layer1 = LayerName::from_str("100000000000000000000000000000000001-200000000000000000000000000000000000__00000000016B59D8-00000000016B5A51").expect("invalid name"); + + let metadata = LayerFileMetadata { + shard: ShardIndex::new(ShardNumber(1), ShardCount(2)), + generation: Generation::Valid(1), + file_size: 0, + }; + + // Construct the (initial and uploaded) index with layer0. + let mut index = IndexPart::empty(TimelineMetadata::example()); + index.layer_metadata.insert(layer0, metadata.clone()); + + // Construct the queue. + let mut queue = UploadQueue::Uninitialized; + let queue = queue.initialize_with_current_remote_index_part(&index)?; + + // Populate inprogress_tasks with a bunch of layer1 deletions. + let delete = UploadOp::Delete(Delete { + layers: vec![(layer1, metadata)], + }); + + for task_id in 0..(inprogress as u64) { + queue.inprogress_tasks.insert( + task_id, + Arc::new(UploadTask { + task_id, + retries: AtomicU32::new(0), + op: delete.clone(), + }), + ); + } + + // Benchmark index upload scheduling. + let index_upload = UploadOp::UploadMetadata { + uploaded: Box::new(index), + }; + + b.iter(|| { + queue.queued_operations.push_front(index_upload.clone()); + assert!(queue.next_ready().is_some()); + }); + + Ok(()) + } +} diff --git a/pageserver/src/tenant/metadata.rs b/pageserver/src/tenant/metadata.rs index 24440d4b35..d281eb305f 100644 --- a/pageserver/src/tenant/metadata.rs +++ b/pageserver/src/tenant/metadata.rs @@ -320,7 +320,6 @@ impl TimelineMetadata { // Checksums make it awkward to build a valid instance by hand. This helper // provides a TimelineMetadata with a valid checksum in its header. - #[cfg(test)] pub fn example() -> Self { let instance = Self::new( "0/16960E8".parse::().unwrap(), diff --git a/pageserver/src/tenant/remote_timeline_client.rs b/pageserver/src/tenant/remote_timeline_client.rs index 813111245d..75e8da496d 100644 --- a/pageserver/src/tenant/remote_timeline_client.rs +++ b/pageserver/src/tenant/remote_timeline_client.rs @@ -63,22 +63,18 @@ //! The contract between client and its user is that the user is responsible of //! scheduling operations in an order that keeps the remote consistent as //! described above. +//! //! From the user's perspective, the operations are executed sequentially. //! Internally, the client knows which operations can be performed in parallel, //! and which operations act like a "barrier" that require preceding operations //! to finish. The calling code just needs to call the schedule-functions in the //! correct order, and the client will parallelize the operations in a way that -//! is safe. -//! -//! The caller should be careful with deletion, though. They should not delete -//! local files that have been scheduled for upload but not yet finished uploading. -//! Otherwise the upload will fail. To wait for an upload to finish, use -//! the 'wait_completion' function (more on that later.) +//! is safe. For more details, see `UploadOp::can_bypass`. //! //! All of this relies on the following invariants: //! //! - We rely on read-after write consistency in the remote storage. -//! - Layer files are immutable +//! - Layer files are immutable. //! //! NB: Pageserver assumes that it has exclusive write access to the tenant in remote //! storage. Different tenants can be attached to different pageservers, but if the @@ -1855,57 +1851,17 @@ impl RemoteTimelineClient { Ok(()) } - /// /// Pick next tasks from the queue, and start as many of them as possible without violating /// the ordering constraints. /// - /// The caller needs to already hold the `upload_queue` lock. + /// TODO: consider limiting the number of in-progress tasks, beyond what remote_storage does. + /// This can launch an unbounded number of queued tasks. `UploadQueue::next_ready()` also has + /// worst-case quadratic cost in the number of tasks, and may struggle beyond 10,000 tasks. fn launch_queued_tasks(self: &Arc, upload_queue: &mut UploadQueueInitialized) { - while let Some(next_op) = upload_queue.queued_operations.front() { - // Can we run this task now? - let can_run_now = match next_op { - UploadOp::UploadLayer(..) => { - // Can always be scheduled. - true - } - UploadOp::UploadMetadata { .. } => { - // These can only be performed after all the preceding operations - // have finished. - upload_queue.inprogress_tasks.is_empty() - } - UploadOp::Delete(..) => { - // Wait for preceding uploads to finish. Concurrent deletions are OK, though. - upload_queue.num_inprogress_deletions == upload_queue.inprogress_tasks.len() - } + while let Some(mut next_op) = upload_queue.next_ready() { + debug!("starting op: {next_op}"); - UploadOp::Barrier(_) | UploadOp::Shutdown => { - upload_queue.inprogress_tasks.is_empty() - } - }; - - // If we cannot launch this task, don't look any further. - // - // In some cases, we could let some non-frontmost tasks to "jump the queue" and launch - // them now, but we don't try to do that currently. For example, if the frontmost task - // is an index-file upload that cannot proceed until preceding uploads have finished, we - // could still start layer uploads that were scheduled later. - if !can_run_now { - break; - } - - if let UploadOp::Shutdown = next_op { - // leave the op in the queue but do not start more tasks; it will be dropped when - // the stop is called. - upload_queue.shutdown_ready.close(); - break; - } - - // We can launch this task. Remove it from the queue first. - let mut next_op = upload_queue.queued_operations.pop_front().unwrap(); - - debug!("starting op: {}", next_op); - - // Update the counters and prepare + // Prepare upload. match &mut next_op { UploadOp::UploadLayer(layer, meta, mode) => { if upload_queue @@ -1916,18 +1872,14 @@ impl RemoteTimelineClient { } else { *mode = Some(OpType::MayReorder) } - upload_queue.num_inprogress_layer_uploads += 1; - } - UploadOp::UploadMetadata { .. } => { - upload_queue.num_inprogress_metadata_uploads += 1; } + UploadOp::UploadMetadata { .. } => {} UploadOp::Delete(Delete { layers }) => { for (name, meta) in layers { upload_queue .recently_deleted .insert((name.clone(), meta.generation)); } - upload_queue.num_inprogress_deletions += 1; } UploadOp::Barrier(sender) => { sender.send_replace(()); @@ -2027,6 +1979,8 @@ impl RemoteTimelineClient { let upload_result: anyhow::Result<()> = match &task.op { UploadOp::UploadLayer(ref layer, ref layer_metadata, mode) => { + // TODO: check if this mechanism can be removed now that can_bypass() performs + // conflict checks during scheduling. if let Some(OpType::FlushDeletion) = mode { if self.config.read().unwrap().block_deletions { // Of course, this is not efficient... but usually the queue should be empty. @@ -2249,13 +2203,8 @@ impl RemoteTimelineClient { upload_queue.inprogress_tasks.remove(&task.task_id); let lsn_update = match task.op { - UploadOp::UploadLayer(_, _, _) => { - upload_queue.num_inprogress_layer_uploads -= 1; - None - } + UploadOp::UploadLayer(_, _, _) => None, UploadOp::UploadMetadata { ref uploaded } => { - upload_queue.num_inprogress_metadata_uploads -= 1; - // the task id is reused as a monotonicity check for storing the "clean" // IndexPart. let last_updater = upload_queue.clean.1; @@ -2289,10 +2238,7 @@ impl RemoteTimelineClient { None } } - UploadOp::Delete(_) => { - upload_queue.num_inprogress_deletions -= 1; - None - } + UploadOp::Delete(_) => None, UploadOp::Barrier(..) | UploadOp::Shutdown => unreachable!(), }; @@ -2416,9 +2362,6 @@ impl RemoteTimelineClient { visible_remote_consistent_lsn: initialized .visible_remote_consistent_lsn .clone(), - num_inprogress_layer_uploads: 0, - num_inprogress_metadata_uploads: 0, - num_inprogress_deletions: 0, inprogress_tasks: HashMap::default(), queued_operations: VecDeque::default(), #[cfg(feature = "testing")] @@ -2445,14 +2388,6 @@ impl RemoteTimelineClient { } }; - // consistency check - assert_eq!( - qi.num_inprogress_layer_uploads - + qi.num_inprogress_metadata_uploads - + qi.num_inprogress_deletions, - qi.inprogress_tasks.len() - ); - // We don't need to do anything here for in-progress tasks. They will finish // on their own, decrement the unfinished-task counter themselves, and observe // that the queue is Stopped. @@ -2899,8 +2834,8 @@ mod tests { let mut guard = client.upload_queue.lock().unwrap(); let upload_queue = guard.initialized_mut().unwrap(); assert!(upload_queue.queued_operations.is_empty()); - assert!(upload_queue.inprogress_tasks.len() == 2); - assert!(upload_queue.num_inprogress_layer_uploads == 2); + assert_eq!(upload_queue.inprogress_tasks.len(), 2); + assert_eq!(upload_queue.num_inprogress_layer_uploads(), 2); // also check that `latest_file_changes` was updated assert!(upload_queue.latest_files_changes_since_metadata_upload_scheduled == 2); @@ -2970,8 +2905,8 @@ mod tests { // Deletion schedules upload of the index file, and the file deletion itself assert_eq!(upload_queue.queued_operations.len(), 2); assert_eq!(upload_queue.inprogress_tasks.len(), 1); - assert_eq!(upload_queue.num_inprogress_layer_uploads, 1); - assert_eq!(upload_queue.num_inprogress_deletions, 0); + assert_eq!(upload_queue.num_inprogress_layer_uploads(), 1); + assert_eq!(upload_queue.num_inprogress_deletions(), 0); assert_eq!( upload_queue.latest_files_changes_since_metadata_upload_scheduled, 0 diff --git a/pageserver/src/tenant/remote_timeline_client/index.rs b/pageserver/src/tenant/remote_timeline_client/index.rs index 51f093cb87..244be5bbb7 100644 --- a/pageserver/src/tenant/remote_timeline_client/index.rs +++ b/pageserver/src/tenant/remote_timeline_client/index.rs @@ -104,7 +104,7 @@ impl IndexPart { pub const FILE_NAME: &'static str = "index_part.json"; - pub(crate) fn empty(metadata: TimelineMetadata) -> Self { + pub fn empty(metadata: TimelineMetadata) -> Self { IndexPart { version: Self::LATEST_VERSION, layer_metadata: Default::default(), diff --git a/pageserver/src/tenant/storage_layer/layer.rs b/pageserver/src/tenant/storage_layer/layer.rs index 8933e8ceb1..2b06c88e8b 100644 --- a/pageserver/src/tenant/storage_layer/layer.rs +++ b/pageserver/src/tenant/storage_layer/layer.rs @@ -1812,7 +1812,7 @@ enum LayerKind { /// Guard for forcing a layer be resident while it exists. #[derive(Clone)] -pub(crate) struct ResidentLayer { +pub struct ResidentLayer { owner: Layer, downloaded: Arc, } diff --git a/pageserver/src/tenant/upload_queue.rs b/pageserver/src/tenant/upload_queue.rs index ef3aa759f3..bd524e8153 100644 --- a/pageserver/src/tenant/upload_queue.rs +++ b/pageserver/src/tenant/upload_queue.rs @@ -1,28 +1,33 @@ +use std::collections::{HashMap, HashSet, VecDeque}; +use std::fmt::Debug; +use std::sync::atomic::AtomicU32; +use std::sync::Arc; + +use super::remote_timeline_client::is_same_remote_layer_path; +use super::storage_layer::AsLayerDesc as _; use super::storage_layer::LayerName; use super::storage_layer::ResidentLayer; use crate::tenant::metadata::TimelineMetadata; use crate::tenant::remote_timeline_client::index::IndexPart; use crate::tenant::remote_timeline_client::index::LayerFileMetadata; -use std::collections::HashSet; -use std::collections::{HashMap, VecDeque}; -use std::fmt::Debug; +use utils::generation::Generation; +use utils::lsn::{AtomicLsn, Lsn}; use chrono::NaiveDateTime; -use std::sync::Arc; +use once_cell::sync::Lazy; use tracing::info; -use utils::lsn::AtomicLsn; -use std::sync::atomic::AtomicU32; -use utils::lsn::Lsn; - -use utils::generation::Generation; +/// Kill switch for upload queue reordering in case it causes problems. +/// TODO: remove this once we have confidence in it. +static DISABLE_UPLOAD_QUEUE_REORDERING: Lazy = + Lazy::new(|| std::env::var("DISABLE_UPLOAD_QUEUE_REORDERING").as_deref() == Ok("true")); // clippy warns that Uninitialized is much smaller than Initialized, which wastes // memory for Uninitialized variants. Doesn't matter in practice, there are not // that many upload queues in a running pageserver, and most of them are initialized // anyway. #[allow(clippy::large_enum_variant)] -pub(super) enum UploadQueue { +pub enum UploadQueue { Uninitialized, Initialized(UploadQueueInitialized), Stopped(UploadQueueStopped), @@ -39,13 +44,13 @@ impl UploadQueue { } #[derive(Copy, Clone, PartialEq, Eq, Hash, Debug)] -pub(crate) enum OpType { +pub enum OpType { MayReorder, FlushDeletion, } /// This keeps track of queued and in-progress tasks. -pub(crate) struct UploadQueueInitialized { +pub struct UploadQueueInitialized { /// Counter to assign task IDs pub(crate) task_counter: u64, @@ -70,21 +75,16 @@ pub(crate) struct UploadQueueInitialized { /// we skip validation) pub(crate) visible_remote_consistent_lsn: Arc, - // Breakdown of different kinds of tasks currently in-progress - pub(crate) num_inprogress_layer_uploads: usize, - pub(crate) num_inprogress_metadata_uploads: usize, - pub(crate) num_inprogress_deletions: usize, - /// Tasks that are currently in-progress. In-progress means that a tokio Task /// has been launched for it. An in-progress task can be busy uploading, but it can /// also be waiting on the `concurrency_limiter` Semaphore in S3Bucket, or it can /// be waiting for retry in `exponential_backoff`. - pub(crate) inprogress_tasks: HashMap>, + pub inprogress_tasks: HashMap>, /// Queued operations that have not been launched yet. They might depend on previous /// tasks to finish. For example, metadata upload cannot be performed before all /// preceding layer file uploads have completed. - pub(crate) queued_operations: VecDeque, + pub queued_operations: VecDeque, /// Files which have been unlinked but not yet had scheduled a deletion for. Only kept around /// for error logging. @@ -122,6 +122,129 @@ impl UploadQueueInitialized { let lsn = self.clean.0.metadata.disk_consistent_lsn(); self.clean.1.map(|_| lsn) } + + /// Returns and removes the next ready operation from the queue, if any. This isn't necessarily + /// the first operation in the queue, to avoid head-of-line blocking -- an operation can jump + /// the queue if it doesn't conflict with operations ahead of it. + /// + /// None may be returned even if the queue isn't empty, if no operations are ready yet. + pub fn next_ready(&mut self) -> Option { + // NB: this is quadratic, but queues are expected to be small. + for (i, candidate) in self.queued_operations.iter().enumerate() { + // If this candidate is ready, go for it. Otherwise, try the next one. + if self.is_ready(i) { + // Shutdown operations are left at the head of the queue, to prevent further + // operations from starting. Signal that we're ready to shut down. + if matches!(candidate, UploadOp::Shutdown) { + assert!(self.inprogress_tasks.is_empty(), "shutdown with tasks"); + assert_eq!(i, 0, "shutdown not at head of queue"); + self.shutdown_ready.close(); + return None; + } + + return self.queued_operations.remove(i); + } + + // Nothing can bypass a barrier or shutdown. If it wasn't scheduled above, give up. + if matches!(candidate, UploadOp::Barrier(_) | UploadOp::Shutdown) { + return None; + } + + // If upload queue reordering is disabled, bail out after the first operation. + if *DISABLE_UPLOAD_QUEUE_REORDERING { + return None; + } + } + None + } + + /// Returns true if the queued operation at the given position is ready to be uploaded, i.e. if + /// it doesn't conflict with any in-progress or queued operations ahead of it. Operations are + /// allowed to skip the queue when it's safe to do so, to increase parallelism. + /// + /// The position must be valid for the queue size. + fn is_ready(&self, pos: usize) -> bool { + let candidate = self.queued_operations.get(pos).expect("invalid position"); + self + // Look at in-progress operations, in random order. + .inprogress_tasks + .values() + .map(|task| &task.op) + // Then queued operations ahead of the candidate, front-to-back. + .chain(self.queued_operations.iter().take(pos)) + // Keep track of the active index ahead of each operation. This is used to ensure that + // an upload doesn't skip the queue too far, such that it modifies a layer that's + // referenced by an active index. + // + // It's okay that in-progress operations are emitted in random order above, since at + // most one of them can be an index upload (enforced by can_bypass). + .scan(&self.clean.0, |next_active_index, op| { + let active_index = *next_active_index; + if let UploadOp::UploadMetadata { ref uploaded } = op { + *next_active_index = uploaded; // stash index for next operation after this + } + Some((op, active_index)) + }) + // Check if the candidate can bypass all of them. + .all(|(op, active_index)| candidate.can_bypass(op, active_index)) + } + + /// Returns the number of in-progress deletion operations. + #[cfg(test)] + pub(crate) fn num_inprogress_deletions(&self) -> usize { + self.inprogress_tasks + .iter() + .filter(|(_, t)| matches!(t.op, UploadOp::Delete(_))) + .count() + } + + /// Returns the number of in-progress layer uploads. + #[cfg(test)] + pub(crate) fn num_inprogress_layer_uploads(&self) -> usize { + self.inprogress_tasks + .iter() + .filter(|(_, t)| matches!(t.op, UploadOp::UploadLayer(_, _, _))) + .count() + } + + /// Test helper that schedules all ready operations into inprogress_tasks, and returns + /// references to them. + /// + /// TODO: the corresponding production logic should be moved from RemoteTimelineClient into + /// UploadQueue, so we can use the same code path. + #[cfg(test)] + fn schedule_ready(&mut self) -> Vec> { + let mut tasks = Vec::new(); + // NB: schedule operations one by one, to handle conflicts with inprogress_tasks. + while let Some(op) = self.next_ready() { + self.task_counter += 1; + let task = Arc::new(UploadTask { + task_id: self.task_counter, + op, + retries: 0.into(), + }); + self.inprogress_tasks.insert(task.task_id, task.clone()); + tasks.push(task); + } + tasks + } + + /// Test helper that marks an operation as completed, removing it from inprogress_tasks. + /// + /// TODO: the corresponding production logic should be moved from RemoteTimelineClient into + /// UploadQueue, so we can use the same code path. + #[cfg(test)] + fn complete(&mut self, task_id: u64) { + let Some(task) = self.inprogress_tasks.remove(&task_id) else { + return; + }; + // Update the clean index on uploads. + if let UploadOp::UploadMetadata { ref uploaded } = task.op { + if task.task_id > self.clean.1.unwrap_or_default() { + self.clean = (*uploaded.clone(), Some(task.task_id)); + } + } + } } #[derive(Clone, Copy)] @@ -131,12 +254,12 @@ pub(super) enum SetDeletedFlagProgress { Successful(NaiveDateTime), } -pub(super) struct UploadQueueStoppedDeletable { +pub struct UploadQueueStoppedDeletable { pub(super) upload_queue_for_deletion: UploadQueueInitialized, pub(super) deleted_at: SetDeletedFlagProgress, } -pub(super) enum UploadQueueStopped { +pub enum UploadQueueStopped { Deletable(UploadQueueStoppedDeletable), Uninitialized, } @@ -163,7 +286,7 @@ impl NotInitialized { } impl UploadQueue { - pub(crate) fn initialize_empty_remote( + pub fn initialize_empty_remote( &mut self, metadata: &TimelineMetadata, ) -> anyhow::Result<&mut UploadQueueInitialized> { @@ -185,9 +308,6 @@ impl UploadQueue { visible_remote_consistent_lsn: Arc::new(AtomicLsn::new(0)), // what follows are boring default initializations task_counter: 0, - num_inprogress_layer_uploads: 0, - num_inprogress_metadata_uploads: 0, - num_inprogress_deletions: 0, inprogress_tasks: HashMap::new(), queued_operations: VecDeque::new(), #[cfg(feature = "testing")] @@ -202,7 +322,7 @@ impl UploadQueue { Ok(self.initialized_mut().expect("we just set it")) } - pub(crate) fn initialize_with_current_remote_index_part( + pub fn initialize_with_current_remote_index_part( &mut self, index_part: &IndexPart, ) -> anyhow::Result<&mut UploadQueueInitialized> { @@ -227,9 +347,6 @@ impl UploadQueue { ), // what follows are boring default initializations task_counter: 0, - num_inprogress_layer_uploads: 0, - num_inprogress_metadata_uploads: 0, - num_inprogress_deletions: 0, inprogress_tasks: HashMap::new(), queued_operations: VecDeque::new(), #[cfg(feature = "testing")] @@ -244,9 +361,7 @@ impl UploadQueue { Ok(self.initialized_mut().expect("we just set it")) } - pub(crate) fn initialized_mut( - &mut self, - ) -> Result<&mut UploadQueueInitialized, NotInitialized> { + pub fn initialized_mut(&mut self) -> Result<&mut UploadQueueInitialized, NotInitialized> { use UploadQueue::*; match self { Uninitialized => Err(NotInitialized::Uninitialized), @@ -276,23 +391,23 @@ impl UploadQueue { /// An in-progress upload or delete task. #[derive(Debug)] -pub(crate) struct UploadTask { +pub struct UploadTask { /// Unique ID of this task. Used as the key in `inprogress_tasks` above. - pub(crate) task_id: u64, - pub(crate) retries: AtomicU32, + pub task_id: u64, + pub retries: AtomicU32, - pub(crate) op: UploadOp, + pub op: UploadOp, } /// A deletion of some layers within the lifetime of a timeline. This is not used /// for timeline deletion, which skips this queue and goes directly to DeletionQueue. #[derive(Debug, Clone)] -pub(crate) struct Delete { - pub(crate) layers: Vec<(LayerName, LayerFileMetadata)>, +pub struct Delete { + pub layers: Vec<(LayerName, LayerFileMetadata)>, } -#[derive(Debug)] -pub(crate) enum UploadOp { +#[derive(Clone, Debug)] +pub enum UploadOp { /// Upload a layer file. The last field indicates the last operation for thie file. UploadLayer(ResidentLayer, LayerFileMetadata, Option), @@ -338,3 +453,765 @@ impl std::fmt::Display for UploadOp { } } } + +impl UploadOp { + /// Returns true if self can bypass other, i.e. if the operations don't conflict. index is the + /// active index when other would be uploaded -- if we allow self to bypass other, this would + /// be the active index when self is uploaded. + pub fn can_bypass(&self, other: &UploadOp, index: &IndexPart) -> bool { + match (self, other) { + // Nothing can bypass a barrier or shutdown, and it can't bypass anything. + (UploadOp::Barrier(_), _) | (_, UploadOp::Barrier(_)) => false, + (UploadOp::Shutdown, _) | (_, UploadOp::Shutdown) => false, + + // Uploads and deletes can bypass each other unless they're for the same file. + (UploadOp::UploadLayer(a, ameta, _), UploadOp::UploadLayer(b, bmeta, _)) => { + let aname = &a.layer_desc().layer_name(); + let bname = &b.layer_desc().layer_name(); + !is_same_remote_layer_path(aname, ameta, bname, bmeta) + } + (UploadOp::UploadLayer(u, umeta, _), UploadOp::Delete(d)) + | (UploadOp::Delete(d), UploadOp::UploadLayer(u, umeta, _)) => { + d.layers.iter().all(|(dname, dmeta)| { + !is_same_remote_layer_path(&u.layer_desc().layer_name(), umeta, dname, dmeta) + }) + } + + // Deletes are idempotent and can always bypass each other. + (UploadOp::Delete(_), UploadOp::Delete(_)) => true, + + // Uploads and deletes can bypass an index upload as long as neither the uploaded index + // nor the active index below it references the file. A layer can't be modified or + // deleted while referenced by an index. + // + // Similarly, index uploads can bypass uploads and deletes as long as neither the + // uploaded index nor the active index references the file (the latter would be + // incorrect use by the caller). + (UploadOp::UploadLayer(u, umeta, _), UploadOp::UploadMetadata { uploaded: i }) + | (UploadOp::UploadMetadata { uploaded: i }, UploadOp::UploadLayer(u, umeta, _)) => { + let uname = u.layer_desc().layer_name(); + !i.references(&uname, umeta) && !index.references(&uname, umeta) + } + (UploadOp::Delete(d), UploadOp::UploadMetadata { uploaded: i }) + | (UploadOp::UploadMetadata { uploaded: i }, UploadOp::Delete(d)) => { + d.layers.iter().all(|(dname, dmeta)| { + !i.references(dname, dmeta) && !index.references(dname, dmeta) + }) + } + + // Indexes can never bypass each other. + // TODO: we could coalesce them though, by only uploading the newest ready index. This + // is left for later, out of caution. + (UploadOp::UploadMetadata { .. }, UploadOp::UploadMetadata { .. }) => false, + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::tenant::harness::{TenantHarness, TIMELINE_ID}; + use crate::tenant::storage_layer::layer::local_layer_path; + use crate::tenant::storage_layer::Layer; + use crate::tenant::Timeline; + use crate::DEFAULT_PG_VERSION; + use itertools::Itertools as _; + use std::str::FromStr as _; + use utils::shard::{ShardCount, ShardIndex, ShardNumber}; + + /// Test helper which asserts that two operations are the same, in lieu of UploadOp PartialEq. + #[track_caller] + fn assert_same_op(a: &UploadOp, b: &UploadOp) { + use UploadOp::*; + match (a, b) { + (UploadLayer(a, ameta, atype), UploadLayer(b, bmeta, btype)) => { + assert_eq!(a.layer_desc().layer_name(), b.layer_desc().layer_name()); + assert_eq!(ameta, bmeta); + assert_eq!(atype, btype); + } + (Delete(a), Delete(b)) => assert_eq!(a.layers, b.layers), + (UploadMetadata { uploaded: a }, UploadMetadata { uploaded: b }) => assert_eq!(a, b), + (Barrier(_), Barrier(_)) => {} + (Shutdown, Shutdown) => {} + (a, b) => panic!("{a:?} != {b:?}"), + } + } + + /// Test helper which asserts that two sets of operations are the same. + #[track_caller] + fn assert_same_ops<'a>( + a: impl IntoIterator, + b: impl IntoIterator, + ) { + a.into_iter() + .zip_eq(b) + .for_each(|(a, b)| assert_same_op(a, b)) + } + + /// Test helper to construct a test timeline. + /// + /// TODO: it really shouldn't be necessary to construct an entire tenant and timeline just to + /// test the upload queue -- decouple ResidentLayer from Timeline. + /// + /// TODO: the upload queue uses TimelineMetadata::example() instead, because there's no way to + /// obtain a TimelineMetadata from a Timeline. + fn make_timeline() -> Arc { + // Grab the current test name from the current thread name. + // TODO: TenantHarness shouldn't take a &'static str, but just leak the test name for now. + let test_name = std::thread::current().name().unwrap().to_string(); + let test_name = Box::leak(test_name.into_boxed_str()); + + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .expect("failed to create runtime"); + + runtime + .block_on(async { + let harness = TenantHarness::create(test_name).await?; + let (tenant, ctx) = harness.load().await; + tenant + .create_test_timeline(TIMELINE_ID, Lsn(8), DEFAULT_PG_VERSION, &ctx) + .await + }) + .expect("failed to create timeline") + } + + /// Test helper to construct an (empty) resident layer. + fn make_layer(timeline: &Arc, name: &str) -> ResidentLayer { + make_layer_with_size(timeline, name, 0) + } + + /// Test helper to construct a resident layer with the given size. + fn make_layer_with_size(timeline: &Arc, name: &str, size: usize) -> ResidentLayer { + let metadata = LayerFileMetadata { + generation: timeline.generation, + shard: timeline.get_shard_index(), + file_size: size as u64, + }; + make_layer_with_metadata(timeline, name, metadata) + } + + /// Test helper to construct a layer with the given metadata. + fn make_layer_with_metadata( + timeline: &Arc, + name: &str, + metadata: LayerFileMetadata, + ) -> ResidentLayer { + let name = LayerName::from_str(name).expect("invalid name"); + let local_path = local_layer_path( + timeline.conf, + &timeline.tenant_shard_id, + &timeline.timeline_id, + &name, + &metadata.generation, + ); + std::fs::write(&local_path, vec![0; metadata.file_size as usize]) + .expect("failed to write file"); + Layer::for_resident(timeline.conf, timeline, local_path, name, metadata) + } + + /// Test helper to add a layer to an index and return a new index. + fn index_with(index: &IndexPart, layer: &ResidentLayer) -> Box { + let mut index = index.clone(); + index + .layer_metadata + .insert(layer.layer_desc().layer_name(), layer.metadata()); + Box::new(index) + } + + /// Test helper to remove a layer from an index and return a new index. + fn index_without(index: &IndexPart, layer: &ResidentLayer) -> Box { + let mut index = index.clone(); + index + .layer_metadata + .remove(&layer.layer_desc().layer_name()); + Box::new(index) + } + + /// Nothing can bypass a barrier, and it can't bypass inprogress tasks. + #[test] + fn schedule_barrier() -> anyhow::Result<()> { + let mut queue = UploadQueue::Uninitialized; + let queue = queue.initialize_empty_remote(&TimelineMetadata::example())?; + let tli = make_timeline(); + + let index = Box::new(queue.clean.0.clone()); // empty, doesn't matter + let layer0 = make_layer(&tli, "000000000000000000000000000000000000-100000000000000000000000000000000000__00000000016B59D8-00000000016B5A51"); + let layer1 = make_layer(&tli, "100000000000000000000000000000000000-200000000000000000000000000000000000__00000000016B59D8-00000000016B5A51"); + let layer2 = make_layer(&tli, "200000000000000000000000000000000000-300000000000000000000000000000000000__00000000016B59D8-00000000016B5A51"); + let layer3 = make_layer(&tli, "300000000000000000000000000000000000-400000000000000000000000000000000000__00000000016B59D8-00000000016B5A51"); + let (barrier, _) = tokio::sync::watch::channel(()); + + // Enqueue non-conflicting upload, delete, and index before and after a barrier. + let ops = [ + UploadOp::UploadLayer(layer0.clone(), layer0.metadata(), None), + UploadOp::Delete(Delete { + layers: vec![(layer1.layer_desc().layer_name(), layer1.metadata())], + }), + UploadOp::UploadMetadata { + uploaded: index.clone(), + }, + UploadOp::Barrier(barrier), + UploadOp::UploadLayer(layer2.clone(), layer2.metadata(), None), + UploadOp::Delete(Delete { + layers: vec![(layer3.layer_desc().layer_name(), layer3.metadata())], + }), + UploadOp::UploadMetadata { + uploaded: index.clone(), + }, + ]; + + queue.queued_operations.extend(ops.clone()); + + // Schedule the initial operations ahead of the barrier. + let tasks = queue.schedule_ready(); + + assert_same_ops(tasks.iter().map(|t| &t.op), &ops[0..3]); + assert!(matches!( + queue.queued_operations.front(), + Some(&UploadOp::Barrier(_)) + )); + + // Complete the initial operations. The barrier isn't scheduled while they're pending. + for task in tasks { + assert!(queue.schedule_ready().is_empty()); + queue.complete(task.task_id); + } + + // Schedule the barrier. The later tasks won't schedule until it completes. + let tasks = queue.schedule_ready(); + + assert_eq!(tasks.len(), 1); + assert!(matches!(tasks[0].op, UploadOp::Barrier(_))); + assert_eq!(queue.queued_operations.len(), 3); + + // Complete the barrier. The rest of the tasks schedule immediately. + queue.complete(tasks[0].task_id); + + let tasks = queue.schedule_ready(); + assert_same_ops(tasks.iter().map(|t| &t.op), &ops[4..]); + assert!(queue.queued_operations.is_empty()); + + Ok(()) + } + + /// Deletes can be scheduled in parallel, even if they're for the same file. + #[test] + fn schedule_delete_parallel() -> anyhow::Result<()> { + let mut queue = UploadQueue::Uninitialized; + let queue = queue.initialize_empty_remote(&TimelineMetadata::example())?; + let tli = make_timeline(); + + // Enqueue a bunch of deletes, some with conflicting names. + let layer0 = make_layer(&tli, "000000000000000000000000000000000000-100000000000000000000000000000000000__00000000016B59D8-00000000016B5A51"); + let layer1 = make_layer(&tli, "100000000000000000000000000000000000-200000000000000000000000000000000000__00000000016B59D8-00000000016B5A51"); + let layer2 = make_layer(&tli, "200000000000000000000000000000000000-300000000000000000000000000000000000__00000000016B59D8-00000000016B5A51"); + let layer3 = make_layer(&tli, "300000000000000000000000000000000000-400000000000000000000000000000000000__00000000016B59D8-00000000016B5A51"); + + let ops = [ + UploadOp::Delete(Delete { + layers: vec![(layer0.layer_desc().layer_name(), layer0.metadata())], + }), + UploadOp::Delete(Delete { + layers: vec![(layer1.layer_desc().layer_name(), layer1.metadata())], + }), + UploadOp::Delete(Delete { + layers: vec![ + (layer1.layer_desc().layer_name(), layer1.metadata()), + (layer2.layer_desc().layer_name(), layer2.metadata()), + ], + }), + UploadOp::Delete(Delete { + layers: vec![(layer2.layer_desc().layer_name(), layer2.metadata())], + }), + UploadOp::Delete(Delete { + layers: vec![(layer3.layer_desc().layer_name(), layer3.metadata())], + }), + ]; + + queue.queued_operations.extend(ops.clone()); + + // Schedule all ready operations. Since deletes don't conflict, they're all scheduled. + let tasks = queue.schedule_ready(); + + assert_same_ops(tasks.iter().map(|t| &t.op), &ops); + assert!(queue.queued_operations.is_empty()); + + Ok(()) + } + + /// Conflicting uploads are serialized. + #[test] + fn schedule_upload_conflicts() -> anyhow::Result<()> { + let mut queue = UploadQueue::Uninitialized; + let queue = queue.initialize_with_current_remote_index_part(&IndexPart::example())?; + let tli = make_timeline(); + + // Enqueue three versions of the same layer, with different file sizes. + let layer0a = make_layer_with_size(&tli, "000000000000000000000000000000000000-100000000000000000000000000000000000__00000000016B59D8-00000000016B5A51", 1); + let layer0b = make_layer_with_size(&tli, "000000000000000000000000000000000000-100000000000000000000000000000000000__00000000016B59D8-00000000016B5A51", 2); + let layer0c = make_layer_with_size(&tli, "000000000000000000000000000000000000-100000000000000000000000000000000000__00000000016B59D8-00000000016B5A51", 3); + + let ops = [ + UploadOp::UploadLayer(layer0a.clone(), layer0a.metadata(), None), + UploadOp::UploadLayer(layer0b.clone(), layer0b.metadata(), None), + UploadOp::UploadLayer(layer0c.clone(), layer0c.metadata(), None), + ]; + + queue.queued_operations.extend(ops.clone()); + + // Only one version should be scheduled and uploaded at a time. + for op in ops { + let tasks = queue.schedule_ready(); + assert_eq!(tasks.len(), 1); + assert_same_op(&tasks[0].op, &op); + queue.complete(tasks[0].task_id); + } + assert!(queue.schedule_ready().is_empty()); + assert!(queue.queued_operations.is_empty()); + + Ok(()) + } + + /// Conflicting uploads and deletes are serialized. + #[test] + fn schedule_upload_delete_conflicts() -> anyhow::Result<()> { + let mut queue = UploadQueue::Uninitialized; + let queue = queue.initialize_with_current_remote_index_part(&IndexPart::example())?; + let tli = make_timeline(); + + // Enqueue two layer uploads, with a delete of both layers in between them. These should be + // scheduled one at a time, since deletes can't bypass uploads and vice versa. + let layer0 = make_layer(&tli, "000000000000000000000000000000000000-100000000000000000000000000000000000__00000000016B59D8-00000000016B5A51"); + let layer1 = make_layer(&tli, "100000000000000000000000000000000000-200000000000000000000000000000000000__00000000016B59D8-00000000016B5A51"); + + let ops = [ + UploadOp::UploadLayer(layer0.clone(), layer0.metadata(), None), + UploadOp::Delete(Delete { + layers: vec![ + (layer0.layer_desc().layer_name(), layer0.metadata()), + (layer1.layer_desc().layer_name(), layer1.metadata()), + ], + }), + UploadOp::UploadLayer(layer1.clone(), layer1.metadata(), None), + ]; + + queue.queued_operations.extend(ops.clone()); + + // Only one version should be scheduled and uploaded at a time. + for op in ops { + let tasks = queue.schedule_ready(); + assert_eq!(tasks.len(), 1); + assert_same_op(&tasks[0].op, &op); + queue.complete(tasks[0].task_id); + } + assert!(queue.schedule_ready().is_empty()); + assert!(queue.queued_operations.is_empty()); + + Ok(()) + } + + /// Non-conflicting uploads and deletes can bypass the queue, avoiding the conflicting + /// delete/upload operations at the head of the queue. + #[test] + fn schedule_upload_delete_conflicts_bypass() -> anyhow::Result<()> { + let mut queue = UploadQueue::Uninitialized; + let queue = queue.initialize_with_current_remote_index_part(&IndexPart::example())?; + let tli = make_timeline(); + + // Enqueue two layer uploads, with a delete of both layers in between them. These should be + // scheduled one at a time, since deletes can't bypass uploads and vice versa. + // + // Also enqueue non-conflicting uploads and deletes at the end. These can bypass the queue + // and run immediately. + let layer0 = make_layer(&tli, "000000000000000000000000000000000000-100000000000000000000000000000000000__00000000016B59D8-00000000016B5A51"); + let layer1 = make_layer(&tli, "100000000000000000000000000000000000-200000000000000000000000000000000000__00000000016B59D8-00000000016B5A51"); + let layer2 = make_layer(&tli, "200000000000000000000000000000000000-300000000000000000000000000000000000__00000000016B59D8-00000000016B5A51"); + let layer3 = make_layer(&tli, "300000000000000000000000000000000000-400000000000000000000000000000000000__00000000016B59D8-00000000016B5A51"); + + let ops = [ + UploadOp::UploadLayer(layer0.clone(), layer0.metadata(), None), + UploadOp::Delete(Delete { + layers: vec![ + (layer0.layer_desc().layer_name(), layer0.metadata()), + (layer1.layer_desc().layer_name(), layer1.metadata()), + ], + }), + UploadOp::UploadLayer(layer1.clone(), layer1.metadata(), None), + UploadOp::UploadLayer(layer2.clone(), layer2.metadata(), None), + UploadOp::Delete(Delete { + layers: vec![(layer3.layer_desc().layer_name(), layer3.metadata())], + }), + ]; + + queue.queued_operations.extend(ops.clone()); + + // Operations 0, 3, and 4 are scheduled immediately. + let tasks = queue.schedule_ready(); + assert_same_ops(tasks.iter().map(|t| &t.op), [&ops[0], &ops[3], &ops[4]]); + assert_eq!(queue.queued_operations.len(), 2); + + Ok(()) + } + + /// Non-conflicting uploads are parallelized. + #[test] + fn schedule_upload_parallel() -> anyhow::Result<()> { + let mut queue = UploadQueue::Uninitialized; + let queue = queue.initialize_with_current_remote_index_part(&IndexPart::example())?; + let tli = make_timeline(); + + // Enqueue three different layer uploads. + let layer0 = make_layer(&tli, "000000000000000000000000000000000000-100000000000000000000000000000000000__00000000016B59D8-00000000016B5A51"); + let layer1 = make_layer(&tli, "100000000000000000000000000000000000-200000000000000000000000000000000000__00000000016B59D8-00000000016B5A51"); + let layer2 = make_layer(&tli, "200000000000000000000000000000000000-300000000000000000000000000000000000__00000000016B59D8-00000000016B5A51"); + + let ops = [ + UploadOp::UploadLayer(layer0.clone(), layer0.metadata(), None), + UploadOp::UploadLayer(layer1.clone(), layer1.metadata(), None), + UploadOp::UploadLayer(layer2.clone(), layer2.metadata(), None), + ]; + + queue.queued_operations.extend(ops.clone()); + + // All uploads should be scheduled concurrently. + let tasks = queue.schedule_ready(); + + assert_same_ops(tasks.iter().map(|t| &t.op), &ops); + assert!(queue.queued_operations.is_empty()); + + Ok(()) + } + + /// Index uploads are serialized. + #[test] + fn schedule_index_serial() -> anyhow::Result<()> { + let mut queue = UploadQueue::Uninitialized; + let queue = queue.initialize_with_current_remote_index_part(&IndexPart::example())?; + + // Enqueue three uploads of the current empty index. + let index = Box::new(queue.clean.0.clone()); + + let ops = [ + UploadOp::UploadMetadata { + uploaded: index.clone(), + }, + UploadOp::UploadMetadata { + uploaded: index.clone(), + }, + UploadOp::UploadMetadata { + uploaded: index.clone(), + }, + ]; + + queue.queued_operations.extend(ops.clone()); + + // The uploads should run serially. + for op in ops { + let tasks = queue.schedule_ready(); + assert_eq!(tasks.len(), 1); + assert_same_op(&tasks[0].op, &op); + queue.complete(tasks[0].task_id); + } + + assert!(queue.queued_operations.is_empty()); + + Ok(()) + } + + /// Chains of upload/index operations lead to parallel layer uploads and serial index uploads. + /// This is the common case with layer flushes. + #[test] + fn schedule_index_upload_chain() -> anyhow::Result<()> { + let mut queue = UploadQueue::Uninitialized; + let queue = queue.initialize_with_current_remote_index_part(&IndexPart::example())?; + let tli = make_timeline(); + + // Enqueue three uploads of the current empty index. + let index = Box::new(queue.clean.0.clone()); + let layer0 = make_layer(&tli, "000000000000000000000000000000000000-100000000000000000000000000000000000__00000000016B59D8-00000000016B5A51"); + let index0 = index_with(&index, &layer0); + let layer1 = make_layer(&tli, "100000000000000000000000000000000000-200000000000000000000000000000000000__00000000016B59D8-00000000016B5A51"); + let index1 = index_with(&index0, &layer1); + let layer2 = make_layer(&tli, "200000000000000000000000000000000000-300000000000000000000000000000000000__00000000016B59D8-00000000016B5A51"); + let index2 = index_with(&index1, &layer2); + + let ops = [ + UploadOp::UploadLayer(layer0.clone(), layer0.metadata(), None), + UploadOp::UploadMetadata { + uploaded: index0.clone(), + }, + UploadOp::UploadLayer(layer1.clone(), layer1.metadata(), None), + UploadOp::UploadMetadata { + uploaded: index1.clone(), + }, + UploadOp::UploadLayer(layer2.clone(), layer2.metadata(), None), + UploadOp::UploadMetadata { + uploaded: index2.clone(), + }, + ]; + + queue.queued_operations.extend(ops.clone()); + + // The layer uploads should be scheduled immediately. The indexes must wait. + let upload_tasks = queue.schedule_ready(); + assert_same_ops( + upload_tasks.iter().map(|t| &t.op), + [&ops[0], &ops[2], &ops[4]], + ); + + // layer2 completes first. None of the indexes can upload yet. + queue.complete(upload_tasks[2].task_id); + assert!(queue.schedule_ready().is_empty()); + + // layer0 completes. index0 can upload. It completes. + queue.complete(upload_tasks[0].task_id); + let index_tasks = queue.schedule_ready(); + assert_eq!(index_tasks.len(), 1); + assert_same_op(&index_tasks[0].op, &ops[1]); + queue.complete(index_tasks[0].task_id); + + // layer 1 completes. This unblocks index 1 then index 2. + queue.complete(upload_tasks[1].task_id); + + let index_tasks = queue.schedule_ready(); + assert_eq!(index_tasks.len(), 1); + assert_same_op(&index_tasks[0].op, &ops[3]); + queue.complete(index_tasks[0].task_id); + + let index_tasks = queue.schedule_ready(); + assert_eq!(index_tasks.len(), 1); + assert_same_op(&index_tasks[0].op, &ops[5]); + queue.complete(index_tasks[0].task_id); + + assert!(queue.queued_operations.is_empty()); + + Ok(()) + } + + /// A delete can't bypass an index upload if an index ahead of it still references it. + #[test] + fn schedule_index_delete_dereferenced() -> anyhow::Result<()> { + let mut queue = UploadQueue::Uninitialized; + let queue = queue.initialize_with_current_remote_index_part(&IndexPart::example())?; + let tli = make_timeline(); + + // Create a layer to upload. + let layer = make_layer(&tli, "000000000000000000000000000000000000-100000000000000000000000000000000000__00000000016B59D8-00000000016B5A51"); + let index_upload = index_with(&queue.clean.0, &layer); + + // Remove the layer reference in a new index, then delete the layer. + let index_deref = index_without(&index_upload, &layer); + + let ops = [ + // Initial upload. + UploadOp::UploadLayer(layer.clone(), layer.metadata(), None), + UploadOp::UploadMetadata { + uploaded: index_upload.clone(), + }, + // Dereference the layer and delete it. + UploadOp::UploadMetadata { + uploaded: index_deref.clone(), + }, + UploadOp::Delete(Delete { + layers: vec![(layer.layer_desc().layer_name(), layer.metadata())], + }), + ]; + + queue.queued_operations.extend(ops.clone()); + + // Operations are serialized. + for op in ops { + let tasks = queue.schedule_ready(); + assert_eq!(tasks.len(), 1); + assert_same_op(&tasks[0].op, &op); + queue.complete(tasks[0].task_id); + } + assert!(queue.queued_operations.is_empty()); + + Ok(()) + } + + /// An upload with a reused layer name doesn't clobber the previous layer. Specifically, a + /// dereference/upload/reference cycle can't allow the upload to bypass the reference. + #[test] + fn schedule_index_upload_dereferenced() -> anyhow::Result<()> { + let mut queue = UploadQueue::Uninitialized; + let queue = queue.initialize_with_current_remote_index_part(&IndexPart::example())?; + let tli = make_timeline(); + + // Create a layer to upload. + let layer = make_layer(&tli, "000000000000000000000000000000000000-100000000000000000000000000000000000__00000000016B59D8-00000000016B5A51"); + + // Upload the layer. Then dereference the layer, and upload/reference it again. + let index_upload = index_with(&queue.clean.0, &layer); + let index_deref = index_without(&index_upload, &layer); + let index_ref = index_with(&index_deref, &layer); + + let ops = [ + // Initial upload. + UploadOp::UploadLayer(layer.clone(), layer.metadata(), None), + UploadOp::UploadMetadata { + uploaded: index_upload.clone(), + }, + // Dereference the layer. + UploadOp::UploadMetadata { + uploaded: index_deref.clone(), + }, + // Replace and reference the layer. + UploadOp::UploadLayer(layer.clone(), layer.metadata(), None), + UploadOp::UploadMetadata { + uploaded: index_ref.clone(), + }, + ]; + + queue.queued_operations.extend(ops.clone()); + + // Operations are serialized. + for op in ops { + let tasks = queue.schedule_ready(); + assert_eq!(tasks.len(), 1); + assert_same_op(&tasks[0].op, &op); + queue.complete(tasks[0].task_id); + } + assert!(queue.queued_operations.is_empty()); + + Ok(()) + } + + /// Nothing can bypass a shutdown, and it waits for inprogress tasks. It's never returned from + /// next_ready(), but is left at the head of the queue. + #[test] + fn schedule_shutdown() -> anyhow::Result<()> { + let mut queue = UploadQueue::Uninitialized; + let queue = queue.initialize_empty_remote(&TimelineMetadata::example())?; + let tli = make_timeline(); + + let index = Box::new(queue.clean.0.clone()); // empty, doesn't matter + let layer0 = make_layer(&tli, "000000000000000000000000000000000000-100000000000000000000000000000000000__00000000016B59D8-00000000016B5A51"); + let layer1 = make_layer(&tli, "100000000000000000000000000000000000-200000000000000000000000000000000000__00000000016B59D8-00000000016B5A51"); + let layer2 = make_layer(&tli, "200000000000000000000000000000000000-300000000000000000000000000000000000__00000000016B59D8-00000000016B5A51"); + let layer3 = make_layer(&tli, "300000000000000000000000000000000000-400000000000000000000000000000000000__00000000016B59D8-00000000016B5A51"); + + // Enqueue non-conflicting upload, delete, and index before and after a shutdown. + let ops = [ + UploadOp::UploadLayer(layer0.clone(), layer0.metadata(), None), + UploadOp::Delete(Delete { + layers: vec![(layer1.layer_desc().layer_name(), layer1.metadata())], + }), + UploadOp::UploadMetadata { + uploaded: index.clone(), + }, + UploadOp::Shutdown, + UploadOp::UploadLayer(layer2.clone(), layer2.metadata(), None), + UploadOp::Delete(Delete { + layers: vec![(layer3.layer_desc().layer_name(), layer3.metadata())], + }), + UploadOp::UploadMetadata { + uploaded: index.clone(), + }, + ]; + + queue.queued_operations.extend(ops.clone()); + + // Schedule the initial operations ahead of the shutdown. + let tasks = queue.schedule_ready(); + + assert_same_ops(tasks.iter().map(|t| &t.op), &ops[0..3]); + assert!(matches!( + queue.queued_operations.front(), + Some(&UploadOp::Shutdown) + )); + + // Complete the initial operations. The shutdown isn't triggered while they're pending. + for task in tasks { + assert!(queue.schedule_ready().is_empty()); + queue.complete(task.task_id); + } + + // The shutdown is triggered the next time we try to pull an operation. It isn't returned, + // but is left in the queue. + assert!(!queue.shutdown_ready.is_closed()); + assert!(queue.next_ready().is_none()); + assert!(queue.shutdown_ready.is_closed()); + + Ok(()) + } + + /// Tests that can_bypass takes name, generation and shard index into account for all operations. + #[test] + fn can_bypass_path() -> anyhow::Result<()> { + let tli = make_timeline(); + + let name0 = &"000000000000000000000000000000000000-100000000000000000000000000000000000__00000000016B59D8-00000000016B5A51"; + let name1 = &"100000000000000000000000000000000000-200000000000000000000000000000000000__00000000016B59D8-00000000016B5A51"; + + // Asserts that layers a and b either can or can't bypass each other, for all combinations + // of operations (except Delete and UploadMetadata which are special-cased). + #[track_caller] + fn assert_can_bypass(a: ResidentLayer, b: ResidentLayer, can_bypass: bool) { + let index = IndexPart::empty(TimelineMetadata::example()); + for (a, b) in make_ops(a).into_iter().zip(make_ops(b)) { + match (&a, &b) { + // Deletes can always bypass each other. + (UploadOp::Delete(_), UploadOp::Delete(_)) => assert!(a.can_bypass(&b, &index)), + // Indexes can never bypass each other. + (UploadOp::UploadMetadata { .. }, UploadOp::UploadMetadata { .. }) => { + assert!(!a.can_bypass(&b, &index)) + } + // For other operations, assert as requested. + (a, b) => assert_eq!(a.can_bypass(b, &index), can_bypass), + } + } + } + + fn make_ops(layer: ResidentLayer) -> Vec { + let mut index = IndexPart::empty(TimelineMetadata::example()); + index + .layer_metadata + .insert(layer.layer_desc().layer_name(), layer.metadata()); + vec![ + UploadOp::UploadLayer(layer.clone(), layer.metadata(), None), + UploadOp::Delete(Delete { + layers: vec![(layer.layer_desc().layer_name(), layer.metadata())], + }), + UploadOp::UploadMetadata { + uploaded: Box::new(index), + }, + ] + } + + // Makes a ResidentLayer. + let layer = |name: &'static str, shard: Option, generation: u32| -> ResidentLayer { + let shard = shard + .map(|n| ShardIndex::new(ShardNumber(n), ShardCount(8))) + .unwrap_or(ShardIndex::unsharded()); + let metadata = LayerFileMetadata { + shard, + generation: Generation::Valid(generation), + file_size: 0, + }; + make_layer_with_metadata(&tli, name, metadata) + }; + + // Same name and metadata can't bypass. This goes both for unsharded and sharded, as well as + // 0 or >0 generation. + assert_can_bypass(layer(name0, None, 0), layer(name0, None, 0), false); + assert_can_bypass(layer(name0, Some(0), 0), layer(name0, Some(0), 0), false); + assert_can_bypass(layer(name0, None, 1), layer(name0, None, 1), false); + + // Different names can bypass. + assert_can_bypass(layer(name0, None, 0), layer(name1, None, 0), true); + + // Different shards can bypass. Shard 0 is different from unsharded. + assert_can_bypass(layer(name0, Some(0), 0), layer(name0, Some(1), 0), true); + assert_can_bypass(layer(name0, Some(0), 0), layer(name0, None, 0), true); + + // Different generations can bypass, both sharded and unsharded. + assert_can_bypass(layer(name0, None, 0), layer(name0, None, 1), true); + assert_can_bypass(layer(name0, Some(1), 0), layer(name0, Some(1), 1), true); + + Ok(()) + } +} From d36112d20fdc249c324308f82b2b81bb124065d9 Mon Sep 17 00:00:00 2001 From: Heikki Linnakangas Date: Tue, 14 Jan 2025 19:02:35 +0200 Subject: [PATCH 097/111] Simplify compute dockerfile by setting PATH just once (#10357) By setting PATH in the 'pg-build' layer, all the extension build layers will inherit. No need to pass PG_CONFIG to all the various make invocations either: once pg_config is in PATH, the Makefiles will pick it up from there. --- compute/compute-node.Dockerfile | 168 ++++++++++++-------------------- 1 file changed, 63 insertions(+), 105 deletions(-) diff --git a/compute/compute-node.Dockerfile b/compute/compute-node.Dockerfile index 89cee6761f..299f4444a3 100644 --- a/compute/compute-node.Dockerfile +++ b/compute/compute-node.Dockerfile @@ -104,16 +104,18 @@ RUN cd postgres && \ esac; \ done; +# Set PATH for all the subsequent build steps +ENV PATH="/usr/local/pgsql/bin:$PATH" + ######################################################################################### # # Layer "postgis-build" # Build PostGIS from the upstream PostGIS mirror. # ######################################################################################### -FROM build-deps AS postgis-build +FROM pg-build AS postgis-build ARG DEBIAN_VERSION ARG PG_VERSION -COPY --from=pg-build /usr/local/pgsql/ /usr/local/pgsql/ RUN apt update && \ apt install --no-install-recommends --no-install-suggests -y \ gdal-bin libboost-dev libboost-thread-dev libboost-filesystem-dev \ @@ -151,8 +153,6 @@ RUN case "${DEBIAN_VERSION}" in \ DESTDIR=/sfcgal ninja install -j $(getconf _NPROCESSORS_ONLN) && \ ninja clean && cp -R /sfcgal/* / -ENV PATH="/usr/local/pgsql/bin:$PATH" - # Postgis 3.5.0 supports v17 RUN case "${PG_VERSION}" in \ "v17") \ @@ -227,9 +227,8 @@ RUN case "${PG_VERSION}" in \ # Build plv8 # ######################################################################################### -FROM build-deps AS plv8-build +FROM pg-build AS plv8-build ARG PG_VERSION -COPY --from=pg-build /usr/local/pgsql/ /usr/local/pgsql/ COPY compute/patches/plv8-3.1.10.patch /plv8-3.1.10.patch @@ -264,7 +263,6 @@ RUN case "${PG_VERSION}" in \ # generate and copy upgrade scripts mkdir -p upgrade && ./generate_upgrade.sh ${PLV8_TAG#v} && \ cp upgrade/* /usr/local/pgsql/share/extension/ && \ - export PATH="/usr/local/pgsql/bin:$PATH" && \ make DOCKER=1 -j $(getconf _NPROCESSORS_ONLN) install && \ rm -rf /plv8-* && \ find /usr/local/pgsql/ -name "plv8-*.so" | xargs strip && \ @@ -291,9 +289,8 @@ RUN case "${PG_VERSION}" in \ # Build h3_pg # ######################################################################################### -FROM build-deps AS h3-pg-build +FROM pg-build AS h3-pg-build ARG PG_VERSION -COPY --from=pg-build /usr/local/pgsql/ /usr/local/pgsql/ # not version-specific # last release v4.1.0 - Jan 18, 2023 @@ -314,7 +311,6 @@ RUN mkdir -p /h3/usr/ && \ RUN wget https://github.com/zachasme/h3-pg/archive/refs/tags/v4.1.3.tar.gz -O h3-pg.tar.gz && \ echo "5c17f09a820859ffe949f847bebf1be98511fb8f1bd86f94932512c00479e324 h3-pg.tar.gz" | sha256sum --check && \ mkdir h3-pg-src && cd h3-pg-src && tar xzf ../h3-pg.tar.gz --strip-components=1 -C . && \ - export PATH="/usr/local/pgsql/bin:$PATH" && \ make -j $(getconf _NPROCESSORS_ONLN) && \ make -j $(getconf _NPROCESSORS_ONLN) install && \ echo 'trusted = true' >> /usr/local/pgsql/share/extension/h3.control && \ @@ -326,17 +322,16 @@ RUN wget https://github.com/zachasme/h3-pg/archive/refs/tags/v4.1.3.tar.gz -O h3 # compile unit extension # ######################################################################################### -FROM build-deps AS unit-pg-build +FROM pg-build AS unit-pg-build ARG PG_VERSION -COPY --from=pg-build /usr/local/pgsql/ /usr/local/pgsql/ # not version-specific # last release 7.9 - Sep 15, 2024 RUN wget https://github.com/df7cb/postgresql-unit/archive/refs/tags/7.9.tar.gz -O postgresql-unit.tar.gz && \ echo "e46de6245dcc8b2c2ecf29873dbd43b2b346773f31dd5ce4b8315895a052b456 postgresql-unit.tar.gz" | sha256sum --check && \ mkdir postgresql-unit-src && cd postgresql-unit-src && tar xzf ../postgresql-unit.tar.gz --strip-components=1 -C . && \ - make -j $(getconf _NPROCESSORS_ONLN) PG_CONFIG=/usr/local/pgsql/bin/pg_config && \ - make -j $(getconf _NPROCESSORS_ONLN) install PG_CONFIG=/usr/local/pgsql/bin/pg_config && \ + make -j $(getconf _NPROCESSORS_ONLN) && \ + make -j $(getconf _NPROCESSORS_ONLN) install && \ # unit extension's "create extension" script relies on absolute install path to fill some reference tables. # We move the extension from '/usr/local/pgsql/' to '/usr/local/' after it is build. So we need to adjust the path. # This one-liner removes pgsql/ part of the path. @@ -350,9 +345,8 @@ RUN wget https://github.com/df7cb/postgresql-unit/archive/refs/tags/7.9.tar.gz - # compile pgvector extension # ######################################################################################### -FROM build-deps AS vector-pg-build +FROM pg-build AS vector-pg-build ARG PG_VERSION -COPY --from=pg-build /usr/local/pgsql/ /usr/local/pgsql/ COPY compute/patches/pgvector.patch /pgvector.patch @@ -366,8 +360,8 @@ RUN wget https://github.com/pgvector/pgvector/archive/refs/tags/v0.8.0.tar.gz -O echo "867a2c328d4928a5a9d6f052cd3bc78c7d60228a9b914ad32aa3db88e9de27b0 pgvector.tar.gz" | sha256sum --check && \ mkdir pgvector-src && cd pgvector-src && tar xzf ../pgvector.tar.gz --strip-components=1 -C . && \ patch -p1 < /pgvector.patch && \ - make -j $(getconf _NPROCESSORS_ONLN) OPTFLAGS="" PG_CONFIG=/usr/local/pgsql/bin/pg_config && \ - make -j $(getconf _NPROCESSORS_ONLN) OPTFLAGS="" install PG_CONFIG=/usr/local/pgsql/bin/pg_config && \ + make -j $(getconf _NPROCESSORS_ONLN) OPTFLAGS="" && \ + make -j $(getconf _NPROCESSORS_ONLN) OPTFLAGS="" install && \ echo 'trusted = true' >> /usr/local/pgsql/share/extension/vector.control ######################################################################################### @@ -376,16 +370,15 @@ RUN wget https://github.com/pgvector/pgvector/archive/refs/tags/v0.8.0.tar.gz -O # compile pgjwt extension # ######################################################################################### -FROM build-deps AS pgjwt-pg-build +FROM pg-build AS pgjwt-pg-build ARG PG_VERSION -COPY --from=pg-build /usr/local/pgsql/ /usr/local/pgsql/ # not version-specific # doesn't use releases, last commit f3d82fd - Mar 2, 2023 RUN wget https://github.com/michelp/pgjwt/archive/f3d82fd30151e754e19ce5d6a06c71c20689ce3d.tar.gz -O pgjwt.tar.gz && \ echo "dae8ed99eebb7593b43013f6532d772b12dfecd55548d2673f2dfd0163f6d2b9 pgjwt.tar.gz" | sha256sum --check && \ mkdir pgjwt-src && cd pgjwt-src && tar xzf ../pgjwt.tar.gz --strip-components=1 -C . && \ - make -j $(getconf _NPROCESSORS_ONLN) install PG_CONFIG=/usr/local/pgsql/bin/pg_config && \ + make -j $(getconf _NPROCESSORS_ONLN) install && \ echo 'trusted = true' >> /usr/local/pgsql/share/extension/pgjwt.control ######################################################################################### @@ -394,17 +387,16 @@ RUN wget https://github.com/michelp/pgjwt/archive/f3d82fd30151e754e19ce5d6a06c71 # compile hypopg extension # ######################################################################################### -FROM build-deps AS hypopg-pg-build +FROM pg-build AS hypopg-pg-build ARG PG_VERSION -COPY --from=pg-build /usr/local/pgsql/ /usr/local/pgsql/ # HypoPG 1.4.1 supports v17 # last release 1.4.1 - Apr 28, 2024 RUN wget https://github.com/HypoPG/hypopg/archive/refs/tags/1.4.1.tar.gz -O hypopg.tar.gz && \ echo "9afe6357fd389d8d33fad81703038ce520b09275ec00153c6c89282bcdedd6bc hypopg.tar.gz" | sha256sum --check && \ mkdir hypopg-src && cd hypopg-src && tar xzf ../hypopg.tar.gz --strip-components=1 -C . && \ - make -j $(getconf _NPROCESSORS_ONLN) PG_CONFIG=/usr/local/pgsql/bin/pg_config && \ - make -j $(getconf _NPROCESSORS_ONLN) install PG_CONFIG=/usr/local/pgsql/bin/pg_config && \ + make -j $(getconf _NPROCESSORS_ONLN) && \ + make -j $(getconf _NPROCESSORS_ONLN) install && \ echo 'trusted = true' >> /usr/local/pgsql/share/extension/hypopg.control ######################################################################################### @@ -413,17 +405,16 @@ RUN wget https://github.com/HypoPG/hypopg/archive/refs/tags/1.4.1.tar.gz -O hypo # compile pg_hashids extension # ######################################################################################### -FROM build-deps AS pg-hashids-pg-build +FROM pg-build AS pg-hashids-pg-build ARG PG_VERSION -COPY --from=pg-build /usr/local/pgsql/ /usr/local/pgsql/ # not version-specific # last release v1.2.1 -Jan 12, 2018 RUN wget https://github.com/iCyberon/pg_hashids/archive/refs/tags/v1.2.1.tar.gz -O pg_hashids.tar.gz && \ echo "74576b992d9277c92196dd8d816baa2cc2d8046fe102f3dcd7f3c3febed6822a pg_hashids.tar.gz" | sha256sum --check && \ mkdir pg_hashids-src && cd pg_hashids-src && tar xzf ../pg_hashids.tar.gz --strip-components=1 -C . && \ - make -j $(getconf _NPROCESSORS_ONLN) PG_CONFIG=/usr/local/pgsql/bin/pg_config USE_PGXS=1 && \ - make -j $(getconf _NPROCESSORS_ONLN) install PG_CONFIG=/usr/local/pgsql/bin/pg_config USE_PGXS=1 && \ + make -j $(getconf _NPROCESSORS_ONLN) USE_PGXS=1 && \ + make -j $(getconf _NPROCESSORS_ONLN) install USE_PGXS=1 && \ echo 'trusted = true' >> /usr/local/pgsql/share/extension/pg_hashids.control ######################################################################################### @@ -432,9 +423,8 @@ RUN wget https://github.com/iCyberon/pg_hashids/archive/refs/tags/v1.2.1.tar.gz # compile rum extension # ######################################################################################### -FROM build-deps AS rum-pg-build +FROM pg-build AS rum-pg-build ARG PG_VERSION -COPY --from=pg-build /usr/local/pgsql/ /usr/local/pgsql/ COPY compute/patches/rum.patch /rum.patch @@ -445,8 +435,8 @@ RUN wget https://github.com/postgrespro/rum/archive/cb1edffc57736cd2a4455f8d0fea echo "65e0a752e99f4c3226400c9b899f997049e93503db8bf5c8072efa136d32fd83 rum.tar.gz" | sha256sum --check && \ mkdir rum-src && cd rum-src && tar xzf ../rum.tar.gz --strip-components=1 -C . && \ patch -p1 < /rum.patch && \ - make -j $(getconf _NPROCESSORS_ONLN) PG_CONFIG=/usr/local/pgsql/bin/pg_config USE_PGXS=1 && \ - make -j $(getconf _NPROCESSORS_ONLN) install PG_CONFIG=/usr/local/pgsql/bin/pg_config USE_PGXS=1 && \ + make -j $(getconf _NPROCESSORS_ONLN) USE_PGXS=1 && \ + make -j $(getconf _NPROCESSORS_ONLN) install USE_PGXS=1 && \ echo 'trusted = true' >> /usr/local/pgsql/share/extension/rum.control ######################################################################################### @@ -455,17 +445,16 @@ RUN wget https://github.com/postgrespro/rum/archive/cb1edffc57736cd2a4455f8d0fea # compile pgTAP extension # ######################################################################################### -FROM build-deps AS pgtap-pg-build +FROM pg-build AS pgtap-pg-build ARG PG_VERSION -COPY --from=pg-build /usr/local/pgsql/ /usr/local/pgsql/ # pgtap 1.3.3 supports v17 # last release v1.3.3 - Apr 8, 2024 RUN wget https://github.com/theory/pgtap/archive/refs/tags/v1.3.3.tar.gz -O pgtap.tar.gz && \ echo "325ea79d0d2515bce96bce43f6823dcd3effbd6c54cb2a4d6c2384fffa3a14c7 pgtap.tar.gz" | sha256sum --check && \ mkdir pgtap-src && cd pgtap-src && tar xzf ../pgtap.tar.gz --strip-components=1 -C . && \ - make -j $(getconf _NPROCESSORS_ONLN) PG_CONFIG=/usr/local/pgsql/bin/pg_config && \ - make -j $(getconf _NPROCESSORS_ONLN) install PG_CONFIG=/usr/local/pgsql/bin/pg_config && \ + make -j $(getconf _NPROCESSORS_ONLN) && \ + make -j $(getconf _NPROCESSORS_ONLN) install && \ echo 'trusted = true' >> /usr/local/pgsql/share/extension/pgtap.control ######################################################################################### @@ -474,17 +463,16 @@ RUN wget https://github.com/theory/pgtap/archive/refs/tags/v1.3.3.tar.gz -O pgta # compile ip4r extension # ######################################################################################### -FROM build-deps AS ip4r-pg-build +FROM pg-build AS ip4r-pg-build ARG PG_VERSION -COPY --from=pg-build /usr/local/pgsql/ /usr/local/pgsql/ # not version-specific # last release v2.4.2 - Jul 29, 2023 RUN wget https://github.com/RhodiumToad/ip4r/archive/refs/tags/2.4.2.tar.gz -O ip4r.tar.gz && \ echo "0f7b1f159974f49a47842a8ab6751aecca1ed1142b6d5e38d81b064b2ead1b4b ip4r.tar.gz" | sha256sum --check && \ mkdir ip4r-src && cd ip4r-src && tar xzf ../ip4r.tar.gz --strip-components=1 -C . && \ - make -j $(getconf _NPROCESSORS_ONLN) PG_CONFIG=/usr/local/pgsql/bin/pg_config && \ - make -j $(getconf _NPROCESSORS_ONLN) install PG_CONFIG=/usr/local/pgsql/bin/pg_config && \ + make -j $(getconf _NPROCESSORS_ONLN) && \ + make -j $(getconf _NPROCESSORS_ONLN) install && \ echo 'trusted = true' >> /usr/local/pgsql/share/extension/ip4r.control ######################################################################################### @@ -493,17 +481,16 @@ RUN wget https://github.com/RhodiumToad/ip4r/archive/refs/tags/2.4.2.tar.gz -O i # compile Prefix extension # ######################################################################################### -FROM build-deps AS prefix-pg-build +FROM pg-build AS prefix-pg-build ARG PG_VERSION -COPY --from=pg-build /usr/local/pgsql/ /usr/local/pgsql/ # not version-specific # last release v1.2.10 - Jul 5, 2023 RUN wget https://github.com/dimitri/prefix/archive/refs/tags/v1.2.10.tar.gz -O prefix.tar.gz && \ echo "4342f251432a5f6fb05b8597139d3ccde8dcf87e8ca1498e7ee931ca057a8575 prefix.tar.gz" | sha256sum --check && \ mkdir prefix-src && cd prefix-src && tar xzf ../prefix.tar.gz --strip-components=1 -C . && \ - make -j $(getconf _NPROCESSORS_ONLN) PG_CONFIG=/usr/local/pgsql/bin/pg_config && \ - make -j $(getconf _NPROCESSORS_ONLN) install PG_CONFIG=/usr/local/pgsql/bin/pg_config && \ + make -j $(getconf _NPROCESSORS_ONLN) && \ + make -j $(getconf _NPROCESSORS_ONLN) install && \ echo 'trusted = true' >> /usr/local/pgsql/share/extension/prefix.control ######################################################################################### @@ -512,17 +499,16 @@ RUN wget https://github.com/dimitri/prefix/archive/refs/tags/v1.2.10.tar.gz -O p # compile hll extension # ######################################################################################### -FROM build-deps AS hll-pg-build +FROM pg-build AS hll-pg-build ARG PG_VERSION -COPY --from=pg-build /usr/local/pgsql/ /usr/local/pgsql/ # not version-specific # last release v2.18 - Aug 29, 2023 RUN wget https://github.com/citusdata/postgresql-hll/archive/refs/tags/v2.18.tar.gz -O hll.tar.gz && \ echo "e2f55a6f4c4ab95ee4f1b4a2b73280258c5136b161fe9d059559556079694f0e hll.tar.gz" | sha256sum --check && \ mkdir hll-src && cd hll-src && tar xzf ../hll.tar.gz --strip-components=1 -C . && \ - make -j $(getconf _NPROCESSORS_ONLN) PG_CONFIG=/usr/local/pgsql/bin/pg_config && \ - make -j $(getconf _NPROCESSORS_ONLN) install PG_CONFIG=/usr/local/pgsql/bin/pg_config && \ + make -j $(getconf _NPROCESSORS_ONLN) && \ + make -j $(getconf _NPROCESSORS_ONLN) install && \ echo 'trusted = true' >> /usr/local/pgsql/share/extension/hll.control ######################################################################################### @@ -531,17 +517,16 @@ RUN wget https://github.com/citusdata/postgresql-hll/archive/refs/tags/v2.18.tar # compile plpgsql_check extension # ######################################################################################### -FROM build-deps AS plpgsql-check-pg-build +FROM pg-build AS plpgsql-check-pg-build ARG PG_VERSION -COPY --from=pg-build /usr/local/pgsql/ /usr/local/pgsql/ # plpgsql_check v2.7.11 supports v17 # last release v2.7.11 - Sep 16, 2024 RUN wget https://github.com/okbob/plpgsql_check/archive/refs/tags/v2.7.11.tar.gz -O plpgsql_check.tar.gz && \ echo "208933f8dbe8e0d2628eb3851e9f52e6892b8e280c63700c0f1ce7883625d172 plpgsql_check.tar.gz" | sha256sum --check && \ mkdir plpgsql_check-src && cd plpgsql_check-src && tar xzf ../plpgsql_check.tar.gz --strip-components=1 -C . && \ - make -j $(getconf _NPROCESSORS_ONLN) PG_CONFIG=/usr/local/pgsql/bin/pg_config USE_PGXS=1 && \ - make -j $(getconf _NPROCESSORS_ONLN) install PG_CONFIG=/usr/local/pgsql/bin/pg_config USE_PGXS=1 && \ + make -j $(getconf _NPROCESSORS_ONLN) USE_PGXS=1 && \ + make -j $(getconf _NPROCESSORS_ONLN) install USE_PGXS=1 && \ echo 'trusted = true' >> /usr/local/pgsql/share/extension/plpgsql_check.control ######################################################################################### @@ -550,11 +535,8 @@ RUN wget https://github.com/okbob/plpgsql_check/archive/refs/tags/v2.7.11.tar.gz # compile timescaledb extension # ######################################################################################### -FROM build-deps AS timescaledb-pg-build -COPY --from=pg-build /usr/local/pgsql/ /usr/local/pgsql/ - +FROM pg-build AS timescaledb-pg-build ARG PG_VERSION -ENV PATH="/usr/local/pgsql/bin:$PATH" RUN case "${PG_VERSION}" in \ "v14" | "v15") \ @@ -585,11 +567,8 @@ RUN case "${PG_VERSION}" in \ # compile pg_hint_plan extension # ######################################################################################### -FROM build-deps AS pg-hint-plan-pg-build -COPY --from=pg-build /usr/local/pgsql/ /usr/local/pgsql/ - +FROM pg-build AS pg-hint-plan-pg-build ARG PG_VERSION -ENV PATH="/usr/local/pgsql/bin:$PATH" # version-specific, has separate releases for each version RUN case "${PG_VERSION}" in \ @@ -627,14 +606,12 @@ RUN case "${PG_VERSION}" in \ # compile pg_cron extension # ######################################################################################### -FROM build-deps AS pg-cron-pg-build +FROM pg-build AS pg-cron-pg-build ARG PG_VERSION -COPY --from=pg-build /usr/local/pgsql/ /usr/local/pgsql/ # This is an experimental extension that we do not support on prod yet. # !Do not remove! # We set it in shared_preload_libraries and computes will fail to start if library is not found. -ENV PATH="/usr/local/pgsql/bin/:$PATH" RUN wget https://github.com/citusdata/pg_cron/archive/refs/tags/v1.6.4.tar.gz -O pg_cron.tar.gz && \ echo "52d1850ee7beb85a4cb7185731ef4e5a90d1de216709d8988324b0d02e76af61 pg_cron.tar.gz" | sha256sum --check && \ mkdir pg_cron-src && cd pg_cron-src && tar xzf ../pg_cron.tar.gz --strip-components=1 -C . && \ @@ -648,9 +625,8 @@ RUN wget https://github.com/citusdata/pg_cron/archive/refs/tags/v1.6.4.tar.gz -O # compile rdkit extension # ######################################################################################### -FROM build-deps AS rdkit-pg-build +FROM pg-build AS rdkit-pg-build ARG PG_VERSION -COPY --from=pg-build /usr/local/pgsql/ /usr/local/pgsql/ RUN apt update && \ apt install --no-install-recommends --no-install-suggests -y \ @@ -668,7 +644,13 @@ RUN apt update && \ # Use new version only for v17 # because Release_2024_09_1 has some backward incompatible changes # https://github.com/rdkit/rdkit/releases/tag/Release_2024_09_1 -ENV PATH="/usr/local/pgsql/bin/:/usr/local/pgsql/:$PATH" + +# XXX: /usr/local/pgsql/bin is already in PATH, and that should be enough to find +# pg_config. For some reason the rdkit cmake script doesn't work with just that, +# however. By also adding /usr/local/pgsql, it works, which is weird because there +# are no executables in that directory. +ENV PATH="/usr/local/pgsql:$PATH" + RUN case "${PG_VERSION}" in \ "v17") \ export RDKIT_VERSION=Release_2024_09_1 \ @@ -721,13 +703,11 @@ RUN case "${PG_VERSION}" in \ # compile pg_uuidv7 extension # ######################################################################################### -FROM build-deps AS pg-uuidv7-pg-build +FROM pg-build AS pg-uuidv7-pg-build ARG PG_VERSION -COPY --from=pg-build /usr/local/pgsql/ /usr/local/pgsql/ # not version-specific # last release v1.6.0 - Oct 9, 2024 -ENV PATH="/usr/local/pgsql/bin/:$PATH" RUN wget https://github.com/fboulnois/pg_uuidv7/archive/refs/tags/v1.6.0.tar.gz -O pg_uuidv7.tar.gz && \ echo "0fa6c710929d003f6ce276a7de7a864e9d1667b2d78be3dc2c07f2409eb55867 pg_uuidv7.tar.gz" | sha256sum --check && \ mkdir pg_uuidv7-src && cd pg_uuidv7-src && tar xzf ../pg_uuidv7.tar.gz --strip-components=1 -C . && \ @@ -741,13 +721,11 @@ RUN wget https://github.com/fboulnois/pg_uuidv7/archive/refs/tags/v1.6.0.tar.gz # compile pg_roaringbitmap extension # ######################################################################################### -FROM build-deps AS pg-roaringbitmap-pg-build +FROM pg-build AS pg-roaringbitmap-pg-build ARG PG_VERSION -COPY --from=pg-build /usr/local/pgsql/ /usr/local/pgsql/ # not version-specific # last release v0.5.4 - Jun 28, 2022 -ENV PATH="/usr/local/pgsql/bin/:$PATH" RUN wget https://github.com/ChenHuajun/pg_roaringbitmap/archive/refs/tags/v0.5.4.tar.gz -O pg_roaringbitmap.tar.gz && \ echo "b75201efcb1c2d1b014ec4ae6a22769cc7a224e6e406a587f5784a37b6b5a2aa pg_roaringbitmap.tar.gz" | sha256sum --check && \ mkdir pg_roaringbitmap-src && cd pg_roaringbitmap-src && tar xzf ../pg_roaringbitmap.tar.gz --strip-components=1 -C . && \ @@ -761,16 +739,14 @@ RUN wget https://github.com/ChenHuajun/pg_roaringbitmap/archive/refs/tags/v0.5.4 # compile pg_semver extension # ######################################################################################### -FROM build-deps AS pg-semver-pg-build +FROM pg-build AS pg-semver-pg-build ARG PG_VERSION -COPY --from=pg-build /usr/local/pgsql/ /usr/local/pgsql/ # Release 0.40.0 breaks backward compatibility with previous versions # see release note https://github.com/theory/pg-semver/releases/tag/v0.40.0 # Use new version only for v17 # # last release v0.40.0 - Jul 22, 2024 -ENV PATH="/usr/local/pgsql/bin/:$PATH" RUN case "${PG_VERSION}" in \ "v17") \ export SEMVER_VERSION=0.40.0 \ @@ -797,13 +773,11 @@ RUN case "${PG_VERSION}" in \ # compile pg_embedding extension # ######################################################################################### -FROM build-deps AS pg-embedding-pg-build -COPY --from=pg-build /usr/local/pgsql/ /usr/local/pgsql/ +FROM pg-build AS pg-embedding-pg-build # This is our extension, support stopped in favor of pgvector # TODO: deprecate it ARG PG_VERSION -ENV PATH="/usr/local/pgsql/bin/:$PATH" RUN case "${PG_VERSION}" in \ "v14" | "v15") \ export PG_EMBEDDING_VERSION=0.3.5 \ @@ -824,20 +798,18 @@ RUN case "${PG_VERSION}" in \ # compile anon extension # ######################################################################################### -FROM build-deps AS pg-anon-pg-build +FROM pg-build AS pg-anon-pg-build ARG PG_VERSION -COPY --from=pg-build /usr/local/pgsql/ /usr/local/pgsql/ # This is an experimental extension, never got to real production. # !Do not remove! It can be present in shared_preload_libraries and compute will fail to start if library is not found. -ENV PATH="/usr/local/pgsql/bin/:$PATH" RUN case "${PG_VERSION}" in "v17") \ echo "postgresql_anonymizer does not yet support PG17" && exit 0;; \ esac && \ wget https://github.com/neondatabase/postgresql_anonymizer/archive/refs/tags/neon_1.1.1.tar.gz -O pg_anon.tar.gz && \ echo "321ea8d5c1648880aafde850a2c576e4a9e7b9933a34ce272efc839328999fa9 pg_anon.tar.gz" | sha256sum --check && \ mkdir pg_anon-src && cd pg_anon-src && tar xzf ../pg_anon.tar.gz --strip-components=1 -C . && \ - make -j $(getconf _NPROCESSORS_ONLN) install PG_CONFIG=/usr/local/pgsql/bin/pg_config && \ + make -j $(getconf _NPROCESSORS_ONLN) install && \ echo 'trusted = true' >> /usr/local/pgsql/share/extension/anon.control ######################################################################################### @@ -846,9 +818,8 @@ RUN case "${PG_VERSION}" in "v17") \ # This layer is used to build `pgrx` deps # ######################################################################################### -FROM build-deps AS rust-extensions-build +FROM pg-build AS rust-extensions-build ARG PG_VERSION -COPY --from=pg-build /usr/local/pgsql/ /usr/local/pgsql/ RUN apt update && \ apt install --no-install-recommends --no-install-suggests -y curl libclang-dev && \ @@ -856,7 +827,7 @@ RUN apt update && \ useradd -ms /bin/bash nonroot -b /home ENV HOME=/home/nonroot -ENV PATH="/home/nonroot/.cargo/bin:/usr/local/pgsql/bin/:$PATH" +ENV PATH="/home/nonroot/.cargo/bin:$PATH" USER nonroot WORKDIR /home/nonroot @@ -883,9 +854,8 @@ USER root # and eventually get merged with `rust-extensions-build` # ######################################################################################### -FROM build-deps AS rust-extensions-build-pgrx12 +FROM pg-build AS rust-extensions-build-pgrx12 ARG PG_VERSION -COPY --from=pg-build /usr/local/pgsql/ /usr/local/pgsql/ RUN apt update && \ apt install --no-install-recommends --no-install-suggests -y curl libclang-dev && \ @@ -893,7 +863,7 @@ RUN apt update && \ useradd -ms /bin/bash nonroot -b /home ENV HOME=/home/nonroot -ENV PATH="/home/nonroot/.cargo/bin:/usr/local/pgsql/bin/:$PATH" +ENV PATH="/home/nonroot/.cargo/bin:$PATH" USER nonroot WORKDIR /home/nonroot @@ -1068,13 +1038,11 @@ RUN wget https://github.com/neondatabase/pg_session_jwt/archive/refs/tags/v0.2.0 # ######################################################################################### -FROM build-deps AS wal2json-pg-build +FROM pg-build AS wal2json-pg-build ARG PG_VERSION -COPY --from=pg-build /usr/local/pgsql/ /usr/local/pgsql/ # wal2json wal2json_2_6 supports v17 # last release wal2json_2_6 - Apr 25, 2024 -ENV PATH="/usr/local/pgsql/bin/:$PATH" RUN wget https://github.com/eulerto/wal2json/archive/refs/tags/wal2json_2_6.tar.gz -O wal2json.tar.gz && \ echo "18b4bdec28c74a8fc98a11c72de38378a760327ef8e5e42e975b0029eb96ba0d wal2json.tar.gz" | sha256sum --check && \ mkdir wal2json-src && cd wal2json-src && tar xzf ../wal2json.tar.gz --strip-components=1 -C . && \ @@ -1087,13 +1055,11 @@ RUN wget https://github.com/eulerto/wal2json/archive/refs/tags/wal2json_2_6.tar. # compile pg_ivm extension # ######################################################################################### -FROM build-deps AS pg-ivm-build +FROM pg-build AS pg-ivm-build ARG PG_VERSION -COPY --from=pg-build /usr/local/pgsql/ /usr/local/pgsql/ # pg_ivm v1.9 supports v17 # last release v1.9 - Jul 31 -ENV PATH="/usr/local/pgsql/bin/:$PATH" RUN wget https://github.com/sraoss/pg_ivm/archive/refs/tags/v1.9.tar.gz -O pg_ivm.tar.gz && \ echo "59e15722939f274650abf637f315dd723c87073496ca77236b044cb205270d8b pg_ivm.tar.gz" | sha256sum --check && \ mkdir pg_ivm-src && cd pg_ivm-src && tar xzf ../pg_ivm.tar.gz --strip-components=1 -C . && \ @@ -1107,13 +1073,11 @@ RUN wget https://github.com/sraoss/pg_ivm/archive/refs/tags/v1.9.tar.gz -O pg_iv # compile pg_partman extension # ######################################################################################### -FROM build-deps AS pg-partman-build +FROM pg-build AS pg-partman-build ARG PG_VERSION -COPY --from=pg-build /usr/local/pgsql/ /usr/local/pgsql/ # should support v17 https://github.com/pgpartman/pg_partman/discussions/693 # last release 5.1.0 Apr 2, 2024 -ENV PATH="/usr/local/pgsql/bin/:$PATH" RUN wget https://github.com/pgpartman/pg_partman/archive/refs/tags/v5.1.0.tar.gz -O pg_partman.tar.gz && \ echo "3e3a27d7ff827295d5c55ef72f07a49062d6204b3cb0b9a048645d6db9f3cb9f pg_partman.tar.gz" | sha256sum --check && \ mkdir pg_partman-src && cd pg_partman-src && tar xzf ../pg_partman.tar.gz --strip-components=1 -C . && \ @@ -1129,9 +1093,6 @@ RUN wget https://github.com/pgpartman/pg_partman/archive/refs/tags/v5.1.0.tar.gz ######################################################################################### FROM rust-extensions-build AS pg-mooncake-build ARG PG_VERSION -COPY --from=pg-build /usr/local/pgsql/ /usr/local/pgsql/ - -ENV PATH="/usr/local/pgsql/bin/:$PATH" RUN wget https://github.com/Mooncake-Labs/pg_mooncake/releases/download/v0.1.0/pg_mooncake-0.1.0.tar.gz -O pg_mooncake.tar.gz && \ echo "eafd059b77f541f11525eb8affcd66a176968cbd8fe7c0d436e733f2aa4da59f pg_mooncake.tar.gz" | sha256sum --check && \ @@ -1147,11 +1108,8 @@ RUN wget https://github.com/Mooncake-Labs/pg_mooncake/releases/download/v0.1.0/p # ######################################################################################### -FROM build-deps AS pg-repack-build +FROM pg-build AS pg-repack-build ARG PG_VERSION -COPY --from=pg-build /usr/local/pgsql/ /usr/local/pgsql/ - -ENV PATH="/usr/local/pgsql/bin/:$PATH" RUN wget https://github.com/reorg/pg_repack/archive/refs/tags/ver_1.5.2.tar.gz -O pg_repack.tar.gz && \ echo '4516cad42251ed3ad53ff619733004db47d5755acac83f75924cd94d1c4fb681 pg_repack.tar.gz' | sha256sum --check && \ From e58e29e63994373e5645a5123e0be191bf693813 Mon Sep 17 00:00:00 2001 From: Erik Grinaker Date: Tue, 14 Jan 2025 19:01:14 +0100 Subject: [PATCH 098/111] pageserver: limit number of upload queue tasks (#10384) ## Problem The upload queue can currently schedule an arbitrary number of tasks. This can both spawn an unbounded number of Tokio tasks, and also significantly slow down upload queue scheduling as it's quadratic in number of operations. Touches #10096. ## Summary of changes Limit the number of inprogress tasks to the remote storage upload concurrency. While this concurrency limit is shared across all tenants, there's certainly no point in scheduling more than this -- we could even consider setting the limit lower, but don't for now to avoid artificially constraining tenants. --- libs/remote_storage/src/config.rs | 11 +++ pageserver/benches/upload_queue.rs | 2 +- .../src/tenant/remote_timeline_client.rs | 29 +++++++- pageserver/src/tenant/upload_queue.rs | 73 ++++++++++++++++--- 4 files changed, 99 insertions(+), 16 deletions(-) diff --git a/libs/remote_storage/src/config.rs b/libs/remote_storage/src/config.rs index 49b1d9dc87..dae141bf77 100644 --- a/libs/remote_storage/src/config.rs +++ b/libs/remote_storage/src/config.rs @@ -43,6 +43,17 @@ impl RemoteStorageKind { } } +impl RemoteStorageConfig { + /// Helper to fetch the configured concurrency limit. + pub fn concurrency_limit(&self) -> Option { + match &self.storage { + RemoteStorageKind::LocalFs { .. } => None, + RemoteStorageKind::AwsS3(c) => Some(c.concurrency_limit.into()), + RemoteStorageKind::AzureContainer(c) => Some(c.concurrency_limit.into()), + } + } +} + fn default_timeout() -> Duration { RemoteStorageConfig::DEFAULT_TIMEOUT } diff --git a/pageserver/benches/upload_queue.rs b/pageserver/benches/upload_queue.rs index 528b3d5490..ed644b0e3c 100644 --- a/pageserver/benches/upload_queue.rs +++ b/pageserver/benches/upload_queue.rs @@ -53,7 +53,7 @@ fn bench_upload_queue_next_ready(c: &mut Criterion) { // Construct the queue. let mut queue = UploadQueue::Uninitialized; - let queue = queue.initialize_with_current_remote_index_part(&index)?; + let queue = queue.initialize_with_current_remote_index_part(&index, 0)?; // Populate inprogress_tasks with a bunch of layer1 deletions. let delete = UploadOp::Delete(Delete { diff --git a/pageserver/src/tenant/remote_timeline_client.rs b/pageserver/src/tenant/remote_timeline_client.rs index 75e8da496d..1602765585 100644 --- a/pageserver/src/tenant/remote_timeline_client.rs +++ b/pageserver/src/tenant/remote_timeline_client.rs @@ -425,8 +425,16 @@ impl RemoteTimelineClient { /// an index file upload, i.e., it's not empty. /// The given `index_part` must be the one on the remote. pub fn init_upload_queue(&self, index_part: &IndexPart) -> anyhow::Result<()> { + // Set the maximum number of inprogress tasks to the remote storage concurrency. There's + // certainly no point in starting more upload tasks than this. + let inprogress_limit = self + .conf + .remote_storage_config + .as_ref() + .and_then(|r| r.concurrency_limit()) + .unwrap_or(0); let mut upload_queue = self.upload_queue.lock().unwrap(); - upload_queue.initialize_with_current_remote_index_part(index_part)?; + upload_queue.initialize_with_current_remote_index_part(index_part, inprogress_limit)?; self.update_remote_physical_size_gauge(Some(index_part)); info!( "initialized upload queue from remote index with {} layer files", @@ -441,8 +449,16 @@ impl RemoteTimelineClient { &self, local_metadata: &TimelineMetadata, ) -> anyhow::Result<()> { + // Set the maximum number of inprogress tasks to the remote storage concurrency. There's + // certainly no point in starting more upload tasks than this. + let inprogress_limit = self + .conf + .remote_storage_config + .as_ref() + .and_then(|r| r.concurrency_limit()) + .unwrap_or(0); let mut upload_queue = self.upload_queue.lock().unwrap(); - upload_queue.initialize_empty_remote(local_metadata)?; + upload_queue.initialize_empty_remote(local_metadata, inprogress_limit)?; self.update_remote_physical_size_gauge(None); info!("initialized upload queue as empty"); Ok(()) @@ -458,9 +474,15 @@ impl RemoteTimelineClient { let deleted_at = index_part.deleted_at.ok_or(anyhow::anyhow!( "bug: it is responsibility of the caller to provide index part from MaybeDeletedIndexPart::Deleted" ))?; + let inprogress_limit = self + .conf + .remote_storage_config + .as_ref() + .and_then(|r| r.concurrency_limit()) + .unwrap_or(0); let mut upload_queue = self.upload_queue.lock().unwrap(); - upload_queue.initialize_with_current_remote_index_part(index_part)?; + upload_queue.initialize_with_current_remote_index_part(index_part, inprogress_limit)?; self.update_remote_physical_size_gauge(Some(index_part)); self.stop_impl(&mut upload_queue); @@ -2355,6 +2377,7 @@ impl RemoteTimelineClient { // but for this use case it doesnt really makes sense to bring unsafe code only for this usage point. // Deletion is not really perf sensitive so there shouldnt be any problems with cloning a fraction of it. let upload_queue_for_deletion = UploadQueueInitialized { + inprogress_limit: initialized.inprogress_limit, task_counter: 0, dirty: initialized.dirty.clone(), clean: initialized.clean.clone(), diff --git a/pageserver/src/tenant/upload_queue.rs b/pageserver/src/tenant/upload_queue.rs index bd524e8153..09c8f6ad8c 100644 --- a/pageserver/src/tenant/upload_queue.rs +++ b/pageserver/src/tenant/upload_queue.rs @@ -51,6 +51,9 @@ pub enum OpType { /// This keeps track of queued and in-progress tasks. pub struct UploadQueueInitialized { + /// Maximum number of inprogress tasks to schedule. 0 is no limit. + pub(crate) inprogress_limit: usize, + /// Counter to assign task IDs pub(crate) task_counter: u64, @@ -128,8 +131,14 @@ impl UploadQueueInitialized { /// the queue if it doesn't conflict with operations ahead of it. /// /// None may be returned even if the queue isn't empty, if no operations are ready yet. + /// + /// NB: this is quadratic, but queues are expected to be small, and bounded by inprogress_limit. pub fn next_ready(&mut self) -> Option { - // NB: this is quadratic, but queues are expected to be small. + // If inprogress_tasks is already at limit, don't schedule anything more. + if self.inprogress_limit > 0 && self.inprogress_tasks.len() >= self.inprogress_limit { + return None; + } + for (i, candidate) in self.queued_operations.iter().enumerate() { // If this candidate is ready, go for it. Otherwise, try the next one. if self.is_ready(i) { @@ -289,6 +298,7 @@ impl UploadQueue { pub fn initialize_empty_remote( &mut self, metadata: &TimelineMetadata, + inprogress_limit: usize, ) -> anyhow::Result<&mut UploadQueueInitialized> { match self { UploadQueue::Uninitialized => (), @@ -302,6 +312,7 @@ impl UploadQueue { let index_part = IndexPart::empty(metadata.clone()); let state = UploadQueueInitialized { + inprogress_limit, dirty: index_part.clone(), clean: (index_part, None), latest_files_changes_since_metadata_upload_scheduled: 0, @@ -325,6 +336,7 @@ impl UploadQueue { pub fn initialize_with_current_remote_index_part( &mut self, index_part: &IndexPart, + inprogress_limit: usize, ) -> anyhow::Result<&mut UploadQueueInitialized> { match self { UploadQueue::Uninitialized => (), @@ -339,6 +351,7 @@ impl UploadQueue { ); let state = UploadQueueInitialized { + inprogress_limit, dirty: index_part.clone(), clean: (index_part.clone(), None), latest_files_changes_since_metadata_upload_scheduled: 0, @@ -633,7 +646,7 @@ mod tests { #[test] fn schedule_barrier() -> anyhow::Result<()> { let mut queue = UploadQueue::Uninitialized; - let queue = queue.initialize_empty_remote(&TimelineMetadata::example())?; + let queue = queue.initialize_empty_remote(&TimelineMetadata::example(), 0)?; let tli = make_timeline(); let index = Box::new(queue.clean.0.clone()); // empty, doesn't matter @@ -700,7 +713,7 @@ mod tests { #[test] fn schedule_delete_parallel() -> anyhow::Result<()> { let mut queue = UploadQueue::Uninitialized; - let queue = queue.initialize_empty_remote(&TimelineMetadata::example())?; + let queue = queue.initialize_empty_remote(&TimelineMetadata::example(), 0)?; let tli = make_timeline(); // Enqueue a bunch of deletes, some with conflicting names. @@ -745,7 +758,7 @@ mod tests { #[test] fn schedule_upload_conflicts() -> anyhow::Result<()> { let mut queue = UploadQueue::Uninitialized; - let queue = queue.initialize_with_current_remote_index_part(&IndexPart::example())?; + let queue = queue.initialize_with_current_remote_index_part(&IndexPart::example(), 0)?; let tli = make_timeline(); // Enqueue three versions of the same layer, with different file sizes. @@ -778,7 +791,7 @@ mod tests { #[test] fn schedule_upload_delete_conflicts() -> anyhow::Result<()> { let mut queue = UploadQueue::Uninitialized; - let queue = queue.initialize_with_current_remote_index_part(&IndexPart::example())?; + let queue = queue.initialize_with_current_remote_index_part(&IndexPart::example(), 0)?; let tli = make_timeline(); // Enqueue two layer uploads, with a delete of both layers in between them. These should be @@ -817,7 +830,7 @@ mod tests { #[test] fn schedule_upload_delete_conflicts_bypass() -> anyhow::Result<()> { let mut queue = UploadQueue::Uninitialized; - let queue = queue.initialize_with_current_remote_index_part(&IndexPart::example())?; + let queue = queue.initialize_with_current_remote_index_part(&IndexPart::example(), 0)?; let tli = make_timeline(); // Enqueue two layer uploads, with a delete of both layers in between them. These should be @@ -859,7 +872,7 @@ mod tests { #[test] fn schedule_upload_parallel() -> anyhow::Result<()> { let mut queue = UploadQueue::Uninitialized; - let queue = queue.initialize_with_current_remote_index_part(&IndexPart::example())?; + let queue = queue.initialize_with_current_remote_index_part(&IndexPart::example(), 0)?; let tli = make_timeline(); // Enqueue three different layer uploads. @@ -888,7 +901,7 @@ mod tests { #[test] fn schedule_index_serial() -> anyhow::Result<()> { let mut queue = UploadQueue::Uninitialized; - let queue = queue.initialize_with_current_remote_index_part(&IndexPart::example())?; + let queue = queue.initialize_with_current_remote_index_part(&IndexPart::example(), 0)?; // Enqueue three uploads of the current empty index. let index = Box::new(queue.clean.0.clone()); @@ -925,7 +938,7 @@ mod tests { #[test] fn schedule_index_upload_chain() -> anyhow::Result<()> { let mut queue = UploadQueue::Uninitialized; - let queue = queue.initialize_with_current_remote_index_part(&IndexPart::example())?; + let queue = queue.initialize_with_current_remote_index_part(&IndexPart::example(), 0)?; let tli = make_timeline(); // Enqueue three uploads of the current empty index. @@ -994,7 +1007,7 @@ mod tests { #[test] fn schedule_index_delete_dereferenced() -> anyhow::Result<()> { let mut queue = UploadQueue::Uninitialized; - let queue = queue.initialize_with_current_remote_index_part(&IndexPart::example())?; + let queue = queue.initialize_with_current_remote_index_part(&IndexPart::example(), 0)?; let tli = make_timeline(); // Create a layer to upload. @@ -1038,7 +1051,7 @@ mod tests { #[test] fn schedule_index_upload_dereferenced() -> anyhow::Result<()> { let mut queue = UploadQueue::Uninitialized; - let queue = queue.initialize_with_current_remote_index_part(&IndexPart::example())?; + let queue = queue.initialize_with_current_remote_index_part(&IndexPart::example(), 0)?; let tli = make_timeline(); // Create a layer to upload. @@ -1085,7 +1098,7 @@ mod tests { #[test] fn schedule_shutdown() -> anyhow::Result<()> { let mut queue = UploadQueue::Uninitialized; - let queue = queue.initialize_empty_remote(&TimelineMetadata::example())?; + let queue = queue.initialize_empty_remote(&TimelineMetadata::example(), 0)?; let tli = make_timeline(); let index = Box::new(queue.clean.0.clone()); // empty, doesn't matter @@ -1139,6 +1152,42 @@ mod tests { Ok(()) } + /// Scheduling respects inprogress_limit. + #[test] + fn schedule_inprogress_limit() -> anyhow::Result<()> { + // Create a queue with inprogress_limit=2. + let mut queue = UploadQueue::Uninitialized; + let queue = queue.initialize_empty_remote(&TimelineMetadata::example(), 2)?; + let tli = make_timeline(); + + // Enqueue a bunch of uploads. + let layer0 = make_layer(&tli, "000000000000000000000000000000000000-100000000000000000000000000000000000__00000000016B59D8-00000000016B5A51"); + let layer1 = make_layer(&tli, "100000000000000000000000000000000000-200000000000000000000000000000000000__00000000016B59D8-00000000016B5A51"); + let layer2 = make_layer(&tli, "200000000000000000000000000000000000-300000000000000000000000000000000000__00000000016B59D8-00000000016B5A51"); + let layer3 = make_layer(&tli, "300000000000000000000000000000000000-400000000000000000000000000000000000__00000000016B59D8-00000000016B5A51"); + + let ops = [ + UploadOp::UploadLayer(layer0.clone(), layer0.metadata(), None), + UploadOp::UploadLayer(layer1.clone(), layer1.metadata(), None), + UploadOp::UploadLayer(layer2.clone(), layer2.metadata(), None), + UploadOp::UploadLayer(layer3.clone(), layer3.metadata(), None), + ]; + + queue.queued_operations.extend(ops.clone()); + + // Schedule all ready operations. Only 2 are scheduled. + let tasks = queue.schedule_ready(); + assert_same_ops(tasks.iter().map(|t| &t.op), &ops[0..2]); + assert!(queue.next_ready().is_none()); + + // When one completes, another is scheduled. + queue.complete(tasks[0].task_id); + let tasks = queue.schedule_ready(); + assert_same_ops(tasks.iter().map(|t| &t.op), &ops[2..3]); + + Ok(()) + } + /// Tests that can_bypass takes name, generation and shard index into account for all operations. #[test] fn can_bypass_path() -> anyhow::Result<()> { From 9a02bc0cfdc94685ed3469579c20dbd00945df8f Mon Sep 17 00:00:00 2001 From: Christian Schwarz Date: Tue, 14 Jan 2025 20:42:01 +0100 Subject: [PATCH 099/111] try to repro root cause hypothesis for https://github.com/neondatabase/neon/issues/10309 This approach here doesn't work because it slows down all the responses. the workload() thread gets stuck in auth, prob with zero pipeline depth 0x00007fa28fe48e63 in epoll_wait () from /lib/x86_64-linux-gnu/libc.so.6 (gdb) bt #0 0x00007fa28fe48e63 in epoll_wait () from /lib/x86_64-linux-gnu/libc.so.6 #1 0x0000561a285bf44e in WaitEventSetWaitBlock (set=0x561a292723e8, cur_timeout=9999, occurred_events=0x7ffd1f11c970, nevents=1) at /home/christian/src/neon//vendor/postgres-v16/src/backend/storage/ipc/latch.c:1535 #2 0x0000561a285bf338 in WaitEventSetWait (set=0x561a292723e8, timeout=9999, occurred_events=0x7ffd1f11c970, nevents=1, wait_event_info=117440512) at /home/christian/src/neon//vendor/postgres-v16/src/backend/storage/ipc/latch.c:1481 #3 0x00007fa2904a7345 in call_PQgetCopyData (shard_no=0, buffer=0x7ffd1f11cad0) at /home/christian/src/neon//pgxn/neon/libpagestore.c:703 #4 0x00007fa2904a7aec in pageserver_receive (shard_no=0) at /home/christian/src/neon//pgxn/neon/libpagestore.c:899 #5 0x00007fa2904af471 in prefetch_read (slot=0x561a292863b0) at /home/christian/src/neon//pgxn/neon/pagestore_smgr.c:644 #6 0x00007fa2904af26b in prefetch_wait_for (ring_index=0) at /home/christian/src/neon//pgxn/neon/pagestore_smgr.c:596 #7 0x00007fa2904b489d in neon_read_at_lsnv (rinfo=..., forkNum=MAIN_FORKNUM, base_blockno=0, request_lsns=0x7ffd1f11cd60, buffers=0x7ffd1f11cd30, nblocks=1, mask=0x0) at /home/christian/src/neon//pgxn/neon/pagestore_smgr.c:3024 #8 0x00007fa2904b4f34 in neon_read_at_lsn (rinfo=..., forkNum=MAIN_FORKNUM, blkno=0, request_lsns=..., buffer=0x7fa28b969000) at /home/christian/src/neon//pgxn/neon/pagestore_smgr.c:3104 #9 0x00007fa2904b515d in neon_read (reln=0x561a292ef448, forkNum=MAIN_FORKNUM, blkno=0, buffer=0x7fa28b969000) at /home/christian/src/neon//pgxn/neon/pagestore_smgr.c:3146 #10 0x0000561a285f1ed5 in smgrread (reln=0x561a292ef448, forknum=MAIN_FORKNUM, blocknum=0, buffer=0x7fa28b969000) at /home/christian/src/neon//vendor/postgres-v16/src/backend/storage/smgr/smgr.c:567 #11 0x0000561a285a528a in ReadBuffer_common (smgr=0x561a292ef448, relpersistence=112 'p', forkNum=MAIN_FORKNUM, blockNum=0, mode=RBM_NORMAL, strategy=0x0, hit=0x7ffd1f11cf1b) at /home/christian/src/neon//vendor/postgres-v16/src/backend/storage/buffer/bufmgr.c:1134 #12 0x0000561a285a47e3 in ReadBufferExtended (reln=0x7fa28ce1c888, forkNum=MAIN_FORKNUM, blockNum=0, mode=RBM_NORMAL, strategy=0x0) at /home/christian/src/neon//vendor/postgres-v16/src/backend/storage/buffer/bufmgr.c:781 #13 0x0000561a285a46ad in ReadBuffer (reln=0x7fa28ce1c888, blockNum=0) at /home/christian/src/neon//vendor/postgres-v16/src/backend/storage/buffer/bufmgr.c:715 #14 0x0000561a2811d511 in _bt_getbuf (rel=0x7fa28ce1c888, blkno=0, access=1) at /home/christian/src/neon//vendor/postgres-v16/src/backend/access/nbtree/nbtpage.c:852 #15 0x0000561a2811d1b2 in _bt_metaversion (rel=0x7fa28ce1c888, heapkeyspace=0x7ffd1f11d9f0, allequalimage=0x7ffd1f11d9f1) at /home/christian/src/neon//vendor/postgres-v16/src/backend/access/nbtree/nbtpage.c:747 #16 0x0000561a28126220 in _bt_first (scan=0x561a292d0348, dir=ForwardScanDirection) at /home/christian/src/neon//vendor/postgres-v16/src/backend/access/nbtree/nbtsearch.c:1465 #17 0x0000561a28121a07 in btgettuple (scan=0x561a292d0348, dir=ForwardScanDirection) at /home/christian/src/neon//vendor/postgres-v16/src/backend/access/nbtree/nbtree.c:246 #18 0x0000561a28111afa in index_getnext_tid (scan=0x561a292d0348, direction=ForwardScanDirection) at /home/christian/src/neon//vendor/postgres-v16/src/backend/access/index/indexam.c:583 #19 0x0000561a28111d14 in index_getnext_slot (scan=0x561a292d0348, direction=ForwardScanDirection, slot=0x561a292d01a8) at /home/christian/src/neon//vendor/postgres-v16/src/backend/access/index/indexam.c:675 #20 0x0000561a2810fbcc in systable_getnext (sysscan=0x561a292d0158) at /home/christian/src/neon//vendor/postgres-v16/src/backend/access/index/genam.c:512 #21 0x0000561a287a1ee1 in SearchCatCacheMiss (cache=0x561a292a0f80, nkeys=1, hashValue=3028463561, hashIndex=1, v1=94670359561576, v2=0, v3=0, v4=0) at /home/christian/src/neon//vendor/postgres-v16/src/backend/utils/cache/catcache.c:1440 #22 0x0000561a287a1d8a in SearchCatCacheInternal (cache=0x561a292a0f80, nkeys=1, v1=94670359561576, v2=0, v3=0, v4=0) at /home/christian/src/neon//vendor/postgres-v16/src/backend/utils/cache/catcache.c:1360 #23 0x0000561a287a1a4f in SearchCatCache (cache=0x561a292a0f80, v1=94670359561576, v2=0, v3=0, v4=0) at /home/christian/src/neon//vendor/postgres-v16/src/backend/utils/cache/catcache.c:1214 #24 0x0000561a287be060 in SearchSysCache (cacheId=10, key1=94670359561576, key2=0, key3=0, key4=0) at /home/christian/src/neon//vendor/postgres-v16/src/backend/utils/cache/syscache.c:817 #25 0x0000561a287be66f in GetSysCacheOid (cacheId=10, oidcol=1, key1=94670359561576, key2=0, key3=0, key4=0) at /home/christian/src/neon//vendor/postgres-v16/src/backend/utils/cache/syscache.c:1055 #26 0x0000561a286319a5 in get_role_oid (rolname=0x561a29270568 "cloud_admin", missing_ok=true) at /home/christian/src/neon//vendor/postgres-v16/src/backend/utils/adt/acl.c:5251 #27 0x0000561a283d42ca in check_hba (port=0x561a29268de0) at /home/christian/src/neon//vendor/postgres-v16/src/backend/libpq/hba.c:2493 #28 0x0000561a283d5537 in hba_getauthmethod (port=0x561a29268de0) at /home/christian/src/neon//vendor/postgres-v16/src/backend/libpq/hba.c:3067 #29 0x0000561a283c6fd7 in ClientAuthentication (port=0x561a29268de0) at /home/christian/src/neon//vendor/postgres-v16/src/backend/libpq/auth.c:395 #30 0x0000561a287dc943 in PerformAuthentication (port=0x561a29268de0) at /home/christian/src/neon//vendor/postgres-v16/src/backend/utils/init/postinit.c:247 #31 0x0000561a287dd9cd in InitPostgres (in_dbname=0x561a29270588 "postgres", dboid=0, username=0x561a29270568 "cloud_admin", useroid=0, load_session_libraries=true, override_allow_connections=false, out_dbname=0x0) at /home/christian/src/neon//vendor/postgres-v16/src/backend/utils/init/postinit.c:929 #32 0x0000561a285fa10b in PostgresMain (dbname=0x561a29270588 "postgres", username=0x561a29270568 "cloud_admin") at /home/christian/src/neon//vendor/postgres-v16/src/backend/tcop/postgres.c:4293 #33 0x0000561a28524ce4 in BackendRun (port=0x561a29268de0) at /home/christian/src/neon//vendor/postgres-v16/src/backend/postmaster/postmaster.c:4465 #34 0x0000561a285245da in BackendStartup (port=0x561a29268de0) at /home/christian/src/neon//vendor/postgres-v16/src/backend/postmaster/postmaster.c:4193 #35 0x0000561a285209c4 in ServerLoop () at /home/christian/src/neon//vendor/postgres-v16/src/backend/postmaster/postmaster.c:1782 #36 0x0000561a2852030f in PostmasterMain (argc=3, argv=0x561a291c5fc0) at /home/christian/src/neon//vendor/postgres-v16/src/backend/postmaster/postmaster.c:1466 #37 0x0000561a283dd987 in main (argc=3, argv=0x561a291c5fc0) at /home/christian/src/neon//vendor/postgres-v16/src/backend/main/main.c:238 --- pageserver/src/page_service.rs | 6 +- .../test_page_service_batching_regressions.py | 105 ++++++++++++++++++ 2 files changed, 110 insertions(+), 1 deletion(-) create mode 100644 test_runner/regress/test_page_service_batching_regressions.py diff --git a/pageserver/src/page_service.rs b/pageserver/src/page_service.rs index b3e18fed99..8b4bab3385 100644 --- a/pageserver/src/page_service.rs +++ b/pageserver/src/page_service.rs @@ -39,6 +39,7 @@ use tokio::io::{AsyncWriteExt, BufWriter}; use tokio::task::JoinHandle; use tokio_util::sync::CancellationToken; use tracing::*; +use utils::failpoint_support; use utils::sync::spsc_fold; use utils::{ auth::{Claims, Scope, SwappableJwtAuth}, @@ -1127,7 +1128,10 @@ impl PageServerHandler { }); // what we want to do - let flush_fut = pgb_writer.flush(); + let flush_fut = async { + failpoint_support::sleep_millis_async!("page_service:flush:pre"); + pgb_writer.flush().await + }; // metric for how long flushing takes let flush_fut = match flushing_timer { Some(flushing_timer) => { diff --git a/test_runner/regress/test_page_service_batching_regressions.py b/test_runner/regress/test_page_service_batching_regressions.py new file mode 100644 index 0000000000..4d456cad79 --- /dev/null +++ b/test_runner/regress/test_page_service_batching_regressions.py @@ -0,0 +1,105 @@ +# NB: there are benchmarks that double-serve as tests inside the `performance` directory. + +import threading +import time + +import requests.exceptions + +import fixtures +from fixtures.common_types import NodeId +from fixtures.log_helper import log +from fixtures.neon_fixtures import NeonEnvBuilder, StorageControllerApiException + + +def test_slow_flush(neon_env_builder: NeonEnvBuilder): + tablesize_mib = 500 + + def patch_pageserver_toml(config): + config["page_service_pipelining"] = { + "mode": "pipelined", + "max_batch_size": 32, + "execution": "concurrent-futures", + } + + neon_env_builder.pageserver_config_override = patch_pageserver_toml + neon_env_builder.num_pageservers = 2 + env = neon_env_builder.init_start() + + ep = env.endpoints.create_start( + "main", + config_lines=[ + "max_parallel_workers_per_gather=0", # disable parallel backends + "effective_io_concurrency=100", # give plenty of opportunity for pipelining + "neon.readahead_buffer_size=128", # this is the default value at time of writing + "shared_buffers=128MB", # keep lower than tablesize_mib + # debug + "log_statement=all", + ], + ) + + conn = ep.connect() + cur = conn.cursor() + + cur.execute("CREATE EXTENSION IF NOT EXISTS neon;") + cur.execute("CREATE EXTENSION IF NOT EXISTS neon_test_utils;") + + log.info("Filling the table") + cur.execute("CREATE TABLE t (data char(1000)) with (fillfactor=10)") + tablesize = tablesize_mib * 1024 * 1024 + npages = tablesize // (8 * 1024) + cur.execute("INSERT INTO t SELECT generate_series(1, %s)", (npages,)) + + cur.close() + conn.close() + + def workload(stop: threading.Event, max_iters=None): + iters = 0 + while stop.is_set() is False and (max_iters == None or iters < max_iters): + log.info("Seqscan %d", iters) + conn = ep.connect() + cur = conn.cursor() + cur.execute( + "select clear_buffer_cache()" + ) # TODO: what about LFC? doesn't matter right now because LFC isn't enabled by default in tests + cur.execute("select sum(data::bigint) from t") + assert cur.fetchall()[0][0] == npages * (npages + 1) // 2 + iters += 1 + log.info("workload done") + + stop = threading.Event() + + log.info("calibrating workload duration") + workload(stop, 1) + before = time.time() + workload(stop, 1) + after = time.time() + duration = after - before + log.info("duration: %f", duration) + assert(duration > 3) + + log.info("begin") + threading.Thread(target=workload, args=[stop]).start() + + # make flush appear slow + ps_http = [p.http_client() for p in env.pageservers] + ps_http[0].configure_failpoints(("page_service:flush:pre", "return(10000000)")) + ps_http[1].configure_failpoints(("page_service:flush:pre", "return(10000000)")) + + time.sleep(1) + + # try to shut down the tenant + for i in range(1, 10): + log.info(f"start migration {i}") + try: + env.storage_controller.tenant_shard_migrate(env.initial_tenant, (i % 2)+1) + except StorageControllerApiException as e: + log.info(f"shard migrate request failed: {e}") + while True: + node_id = NodeId(env.storage_controller.tenant_describe(env.initial_tenant)["node_id"]) + if node_id == NodeId(i % 2)+1: + break + log.info(f"waiting for migration to complete") + time.sleep(1) + log.info(f"migration done") + time.sleep(1) + From 6debb49b87dcf2dcca53bf17ee005b936eac446c Mon Sep 17 00:00:00 2001 From: Erik Grinaker Date: Tue, 14 Jan 2025 22:10:17 +0100 Subject: [PATCH 100/111] pageserver: coalesce index uploads when possible (#10248) ## Problem With upload queue reordering in #10218, we can easily get into a situation where multiple index uploads are queued back to back, which can't be parallelized. This will happen e.g. when multiple layer flushes enqueue layer/index/layer/index/... and the layers skip the queue and are uploaded in parallel. These index uploads will incur serial S3 roundtrip latencies, and may block later operations. Touches #10096. ## Summary of changes When multiple back-to-back index uploads are ready to upload, only upload the most recent index and drop the rest. --- pageserver/benches/upload_queue.rs | 1 + .../src/tenant/remote_timeline_client.rs | 6 +- pageserver/src/tenant/upload_queue.rs | 86 +++++++++++++------ 3 files changed, 67 insertions(+), 26 deletions(-) diff --git a/pageserver/benches/upload_queue.rs b/pageserver/benches/upload_queue.rs index ed644b0e3c..ed5daa8ae1 100644 --- a/pageserver/benches/upload_queue.rs +++ b/pageserver/benches/upload_queue.rs @@ -67,6 +67,7 @@ fn bench_upload_queue_next_ready(c: &mut Criterion) { task_id, retries: AtomicU32::new(0), op: delete.clone(), + coalesced_ops: Vec::new(), }), ); } diff --git a/pageserver/src/tenant/remote_timeline_client.rs b/pageserver/src/tenant/remote_timeline_client.rs index 1602765585..47c4a8637d 100644 --- a/pageserver/src/tenant/remote_timeline_client.rs +++ b/pageserver/src/tenant/remote_timeline_client.rs @@ -1880,7 +1880,7 @@ impl RemoteTimelineClient { /// This can launch an unbounded number of queued tasks. `UploadQueue::next_ready()` also has /// worst-case quadratic cost in the number of tasks, and may struggle beyond 10,000 tasks. fn launch_queued_tasks(self: &Arc, upload_queue: &mut UploadQueueInitialized) { - while let Some(mut next_op) = upload_queue.next_ready() { + while let Some((mut next_op, coalesced_ops)) = upload_queue.next_ready() { debug!("starting op: {next_op}"); // Prepare upload. @@ -1918,6 +1918,7 @@ impl RemoteTimelineClient { let task = Arc::new(UploadTask { task_id: upload_task_id, op: next_op, + coalesced_ops, retries: AtomicU32::new(0), }); upload_queue @@ -2285,6 +2286,9 @@ impl RemoteTimelineClient { } self.metric_end(&task.op); + for coalesced_op in &task.coalesced_ops { + self.metric_end(coalesced_op); + } } fn metric_impl( diff --git a/pageserver/src/tenant/upload_queue.rs b/pageserver/src/tenant/upload_queue.rs index 09c8f6ad8c..d302205ffe 100644 --- a/pageserver/src/tenant/upload_queue.rs +++ b/pageserver/src/tenant/upload_queue.rs @@ -22,6 +22,11 @@ use tracing::info; static DISABLE_UPLOAD_QUEUE_REORDERING: Lazy = Lazy::new(|| std::env::var("DISABLE_UPLOAD_QUEUE_REORDERING").as_deref() == Ok("true")); +/// Kill switch for index upload coalescing in case it causes problems. +/// TODO: remove this once we have confidence in it. +static DISABLE_UPLOAD_QUEUE_INDEX_COALESCING: Lazy = + Lazy::new(|| std::env::var("DISABLE_UPLOAD_QUEUE_INDEX_COALESCING").as_deref() == Ok("true")); + // clippy warns that Uninitialized is much smaller than Initialized, which wastes // memory for Uninitialized variants. Doesn't matter in practice, there are not // that many upload queues in a running pageserver, and most of them are initialized @@ -130,10 +135,12 @@ impl UploadQueueInitialized { /// the first operation in the queue, to avoid head-of-line blocking -- an operation can jump /// the queue if it doesn't conflict with operations ahead of it. /// + /// Also returns any operations that were coalesced into this one, e.g. multiple index uploads. + /// /// None may be returned even if the queue isn't empty, if no operations are ready yet. /// /// NB: this is quadratic, but queues are expected to be small, and bounded by inprogress_limit. - pub fn next_ready(&mut self) -> Option { + pub fn next_ready(&mut self) -> Option<(UploadOp, Vec)> { // If inprogress_tasks is already at limit, don't schedule anything more. if self.inprogress_limit > 0 && self.inprogress_tasks.len() >= self.inprogress_limit { return None; @@ -151,7 +158,36 @@ impl UploadQueueInitialized { return None; } - return self.queued_operations.remove(i); + let mut op = self.queued_operations.remove(i).expect("i can't disappear"); + + // Coalesce any back-to-back index uploads by only uploading the newest one that's + // ready. This typically happens with layer/index/layer/index/... sequences, where + // the layers bypass the indexes, leaving the indexes queued. + // + // If other operations are interleaved between index uploads we don't try to + // coalesce them, since we may as well update the index concurrently with them. + // This keeps the index fresh and avoids starvation. + // + // NB: we assume that all uploaded indexes have the same remote path. This + // is true at the time of writing: the path only depends on the tenant, + // timeline and generation, all of which are static for a timeline instance. + // Otherwise, we must be careful not to coalesce different paths. + let mut coalesced_ops = Vec::new(); + if matches!(op, UploadOp::UploadMetadata { .. }) { + while let Some(UploadOp::UploadMetadata { .. }) = self.queued_operations.get(i) + { + if *DISABLE_UPLOAD_QUEUE_INDEX_COALESCING { + break; + } + if !self.is_ready(i) { + break; + } + coalesced_ops.push(op); + op = self.queued_operations.remove(i).expect("i can't disappear"); + } + } + + return Some((op, coalesced_ops)); } // Nothing can bypass a barrier or shutdown. If it wasn't scheduled above, give up. @@ -225,11 +261,12 @@ impl UploadQueueInitialized { fn schedule_ready(&mut self) -> Vec> { let mut tasks = Vec::new(); // NB: schedule operations one by one, to handle conflicts with inprogress_tasks. - while let Some(op) = self.next_ready() { + while let Some((op, coalesced_ops)) = self.next_ready() { self.task_counter += 1; let task = Arc::new(UploadTask { task_id: self.task_counter, op, + coalesced_ops, retries: 0.into(), }); self.inprogress_tasks.insert(task.task_id, task.clone()); @@ -407,9 +444,13 @@ impl UploadQueue { pub struct UploadTask { /// Unique ID of this task. Used as the key in `inprogress_tasks` above. pub task_id: u64, + /// Number of task retries. pub retries: AtomicU32, - + /// The upload operation. pub op: UploadOp, + /// Any upload operations that were coalesced into this operation. This typically happens with + /// back-to-back index uploads, see `UploadQueueInitialized::next_ready()`. + pub coalesced_ops: Vec, } /// A deletion of some layers within the lifetime of a timeline. This is not used @@ -512,9 +553,8 @@ impl UploadOp { }) } - // Indexes can never bypass each other. - // TODO: we could coalesce them though, by only uploading the newest ready index. This - // is left for later, out of caution. + // Indexes can never bypass each other. They can coalesce though, and + // `UploadQueue::next_ready()` currently does this when possible. (UploadOp::UploadMetadata { .. }, UploadOp::UploadMetadata { .. }) => false, } } @@ -897,9 +937,9 @@ mod tests { Ok(()) } - /// Index uploads are serialized. + /// Index uploads are coalesced. #[test] - fn schedule_index_serial() -> anyhow::Result<()> { + fn schedule_index_coalesce() -> anyhow::Result<()> { let mut queue = UploadQueue::Uninitialized; let queue = queue.initialize_with_current_remote_index_part(&IndexPart::example(), 0)?; @@ -920,13 +960,11 @@ mod tests { queue.queued_operations.extend(ops.clone()); - // The uploads should run serially. - for op in ops { - let tasks = queue.schedule_ready(); - assert_eq!(tasks.len(), 1); - assert_same_op(&tasks[0].op, &op); - queue.complete(tasks[0].task_id); - } + // The index uploads are coalesced into a single operation. + let tasks = queue.schedule_ready(); + assert_eq!(tasks.len(), 1); + assert_same_op(&tasks[0].op, &ops[2]); + assert_same_ops(&tasks[0].coalesced_ops, &ops[0..2]); assert!(queue.queued_operations.is_empty()); @@ -985,18 +1023,14 @@ mod tests { assert_same_op(&index_tasks[0].op, &ops[1]); queue.complete(index_tasks[0].task_id); - // layer 1 completes. This unblocks index 1 then index 2. + // layer 1 completes. This unblocks index 1 and 2, which coalesce into + // a single upload for index 2. queue.complete(upload_tasks[1].task_id); - let index_tasks = queue.schedule_ready(); - assert_eq!(index_tasks.len(), 1); - assert_same_op(&index_tasks[0].op, &ops[3]); - queue.complete(index_tasks[0].task_id); - let index_tasks = queue.schedule_ready(); assert_eq!(index_tasks.len(), 1); assert_same_op(&index_tasks[0].op, &ops[5]); - queue.complete(index_tasks[0].task_id); + assert_same_ops(&index_tasks[0].coalesced_ops, &ops[3..4]); assert!(queue.queued_operations.is_empty()); @@ -1018,11 +1052,12 @@ mod tests { let index_deref = index_without(&index_upload, &layer); let ops = [ - // Initial upload. + // Initial upload, with a barrier to prevent index coalescing. UploadOp::UploadLayer(layer.clone(), layer.metadata(), None), UploadOp::UploadMetadata { uploaded: index_upload.clone(), }, + UploadOp::Barrier(tokio::sync::watch::channel(()).0), // Dereference the layer and delete it. UploadOp::UploadMetadata { uploaded: index_deref.clone(), @@ -1063,11 +1098,12 @@ mod tests { let index_ref = index_with(&index_deref, &layer); let ops = [ - // Initial upload. + // Initial upload, with a barrier to prevent index coalescing. UploadOp::UploadLayer(layer.clone(), layer.metadata(), None), UploadOp::UploadMetadata { uploaded: index_upload.clone(), }, + UploadOp::Barrier(tokio::sync::watch::channel(()).0), // Dereference the layer. UploadOp::UploadMetadata { uploaded: index_deref.clone(), From 45e08d0aa553ff2d471cb3e8bb78aff21921c8c9 Mon Sep 17 00:00:00 2001 From: Christian Schwarz Date: Tue, 14 Jan 2025 22:16:27 +0100 Subject: [PATCH 101/111] it repros --- libs/pageserver_api/src/models.rs | 89 +++++++++++ pageserver/client/src/page_service.rs | 132 +++++++++++++--- .../src/bin/test_helper_slow_client_reads.rs | 63 ++++++++ pageserver/src/metrics.rs | 2 + pageserver/src/page_service.rs | 141 ++++++++++++++++++ .../test_page_service_batching_regressions.py | 112 ++++---------- 6 files changed, 429 insertions(+), 110 deletions(-) create mode 100644 pageserver/src/bin/test_helper_slow_client_reads.rs diff --git a/libs/pageserver_api/src/models.rs b/libs/pageserver_api/src/models.rs index 39390d7647..227bc19d67 100644 --- a/libs/pageserver_api/src/models.rs +++ b/libs/pageserver_api/src/models.rs @@ -1398,6 +1398,8 @@ pub enum PagestreamFeMessage { GetPage(PagestreamGetPageRequest), DbSize(PagestreamDbSizeRequest), GetSlruSegment(PagestreamGetSlruSegmentRequest), + #[cfg(feature = "testing")] + Test(PagestreamTestRequest), } // Wrapped in libpq CopyData @@ -1409,6 +1411,8 @@ pub enum PagestreamBeMessage { Error(PagestreamErrorResponse), DbSize(PagestreamDbSizeResponse), GetSlruSegment(PagestreamGetSlruSegmentResponse), + #[cfg(feature = "testing")] + Test(PagestreamTestResponse), } // Keep in sync with `pagestore_client.h` @@ -1420,6 +1424,9 @@ enum PagestreamBeMessageTag { Error = 103, DbSize = 104, GetSlruSegment = 105, + /// Test message discrimimant is unstable + #[cfg(feature = "testing")] + Test = 106, } impl TryFrom for PagestreamBeMessageTag { type Error = u8; @@ -1431,6 +1438,8 @@ impl TryFrom for PagestreamBeMessageTag { 103 => Ok(PagestreamBeMessageTag::Error), 104 => Ok(PagestreamBeMessageTag::DbSize), 105 => Ok(PagestreamBeMessageTag::GetSlruSegment), + #[cfg(feature = "testing")] + 106 => Ok(PagestreamBeMessageTag::Test), _ => Err(value), } } @@ -1548,6 +1557,20 @@ pub struct PagestreamDbSizeResponse { pub db_size: i64, } +#[cfg(feature = "testing")] +#[derive(Debug, PartialEq, Eq, Clone)] +pub struct PagestreamTestRequest { + pub hdr: PagestreamRequest, + pub batch_key: u64, + pub message: String, +} + +#[cfg(feature = "testing")] +#[derive(Debug)] +pub struct PagestreamTestResponse { + pub req: PagestreamTestRequest, +} + // This is a cut-down version of TenantHistorySize from the pageserver crate, omitting fields // that require pageserver-internal types. It is sufficient to get the total size. #[derive(Serialize, Deserialize, Debug)] @@ -1616,6 +1639,16 @@ impl PagestreamFeMessage { bytes.put_u8(req.kind); bytes.put_u32(req.segno); } + + Self::Test(req) => { + bytes.put_u8(5); + bytes.put_u64(req.hdr.reqid); + bytes.put_u64(req.hdr.request_lsn.0); + bytes.put_u64(req.hdr.not_modified_since.0); + bytes.put_u64(req.batch_key); + bytes.put_u64(req.message.as_bytes().len() as u64); + bytes.put_slice(req.message.as_bytes()); + } } bytes.into() @@ -1703,6 +1736,21 @@ impl PagestreamFeMessage { segno: body.read_u32::()?, }, )), + #[cfg(feature = "testing")] + 5 => Ok(PagestreamFeMessage::Test(PagestreamTestRequest { + hdr: PagestreamRequest { + reqid, + request_lsn, + not_modified_since, + }, + batch_key: body.read_u64::()?, + message: { + let len = body.read_u64::()?; + let mut buf = vec![0; len as usize]; + body.read_exact(&mut buf)?; + String::from_utf8(buf)? + }, + })), _ => bail!("unknown smgr message tag: {:?}", msg_tag), } } @@ -1746,6 +1794,13 @@ impl PagestreamBeMessage { bytes.put_u32((resp.segment.len() / BLCKSZ as usize) as u32); bytes.put(&resp.segment[..]); } + + Self::Test(resp) => { + bytes.put_u8(Tag::Test as u8); + bytes.put_u64(resp.req.batch_key); + bytes.put_u64(resp.req.message.as_bytes().len() as u64); + bytes.put_slice(resp.req.message.as_bytes()); + } } } PagestreamProtocolVersion::V3 => { @@ -1814,6 +1869,16 @@ impl PagestreamBeMessage { bytes.put_u32((resp.segment.len() / BLCKSZ as usize) as u32); bytes.put(&resp.segment[..]); } + + Self::Test(resp) => { + bytes.put_u8(Tag::Test as u8); + bytes.put_u64(resp.req.hdr.reqid); + bytes.put_u64(resp.req.hdr.request_lsn.0); + bytes.put_u64(resp.req.hdr.not_modified_since.0); + bytes.put_u64(resp.req.batch_key); + bytes.put_u64(resp.req.message.as_bytes().len() as u64); + bytes.put_slice(resp.req.message.as_bytes()); + } } } } @@ -1956,6 +2021,28 @@ impl PagestreamBeMessage { segment: segment.into(), }) } + #[cfg(feature = "testing")] + Tag::Test => { + let reqid = buf.read_u64::()?; + let request_lsn = Lsn(buf.read_u64::()?); + let not_modified_since = Lsn(buf.read_u64::()?); + let batch_key = buf.read_u64::()?; + let len = buf.read_u64::()?; + let mut msg = vec![0; len as usize]; + buf.read_exact(&mut msg)?; + let message = String::from_utf8(msg)?; + Self::Test(PagestreamTestResponse { + req: PagestreamTestRequest { + hdr: PagestreamRequest { + reqid, + request_lsn, + not_modified_since, + }, + batch_key, + message, + }, + }) + } }; let remaining = buf.into_inner(); if !remaining.is_empty() { @@ -1975,6 +2062,8 @@ impl PagestreamBeMessage { Self::Error(_) => "Error", Self::DbSize(_) => "DbSize", Self::GetSlruSegment(_) => "GetSlruSegment", + #[cfg(feature = "testing")] + Self::Test(_) => "Test", } } } diff --git a/pageserver/client/src/page_service.rs b/pageserver/client/src/page_service.rs index 207ec4166c..3827e19aa4 100644 --- a/pageserver/client/src/page_service.rs +++ b/pageserver/client/src/page_service.rs @@ -1,6 +1,9 @@ -use std::pin::Pin; +use std::sync::{Arc, Mutex}; -use futures::SinkExt; +use futures::{ + stream::{SplitSink, SplitStream}, + SinkExt, StreamExt, +}; use pageserver_api::{ models::{ PagestreamBeMessage, PagestreamFeMessage, PagestreamGetPageRequest, @@ -10,7 +13,6 @@ use pageserver_api::{ }; use tokio::task::JoinHandle; use tokio_postgres::CopyOutStream; -use tokio_stream::StreamExt; use tokio_util::sync::CancellationToken; use utils::{ id::{TenantId, TimelineId}, @@ -62,15 +64,28 @@ impl Client { .client .copy_both_simple(&format!("pagestream_v3 {tenant_id} {timeline_id}")) .await?; + let (sink, stream) = copy_both.split(); // TODO: actually support splitting of the CopyBothDuplex so the lock inside this split adaptor goes away. let Client { cancel_on_client_drop, conn_task, client: _, } = self; + let shared = Arc::new(Mutex::new(PagestreamShared::ConnTaskRunning( + ConnTaskRunning { + cancel_on_client_drop, + conn_task, + }, + ))); Ok(PagestreamClient { - copy_both: Box::pin(copy_both), - conn_task, - cancel_on_client_drop, + sink: PagestreamSender { + shared: shared.clone(), + sink, + }, + stream: PagestreamReceiver { + shared: shared.clone(), + stream, + }, + shared, }) } @@ -97,7 +112,26 @@ impl Client { /// Create using [`Client::pagestream`]. pub struct PagestreamClient { - copy_both: Pin>>, + shared: Arc>, + sink: PagestreamSender, + stream: PagestreamReceiver, +} + +pub struct PagestreamSender { + shared: Arc>, + sink: SplitSink, bytes::Bytes>, +} + +pub struct PagestreamReceiver { + shared: Arc>, + stream: SplitStream>, +} + +enum PagestreamShared { + ConnTaskRunning(ConnTaskRunning), + ConnTaskCancelledJoinHandleReturnedOrDropped, +} +struct ConnTaskRunning { cancel_on_client_drop: Option, conn_task: JoinHandle<()>, } @@ -110,11 +144,11 @@ pub struct RelTagBlockNo { impl PagestreamClient { pub async fn shutdown(self) { let Self { - copy_both, - cancel_on_client_drop: cancel_conn_task, - conn_task, - } = self; - // The `copy_both` contains internal channel sender, the receiver of which is polled by `conn_task`. + shared, + sink, + stream, + } = { self }; + // The `copy_both` split into `sink` and `stream` contains internal channel sender, the receiver of which is polled by `conn_task`. // When `conn_task` observes the sender has been dropped, it sends a `FeMessage::CopyFail` into the connection. // (see https://github.com/neondatabase/rust-postgres/blob/2005bf79573b8add5cf205b52a2b208e356cc8b0/tokio-postgres/src/copy_both.rs#L56). // @@ -131,36 +165,86 @@ impl PagestreamClient { // // NB: page_service doesn't have a use case to exit the `pagestream` mode currently. // => https://github.com/neondatabase/neon/issues/6390 - let _ = cancel_conn_task.unwrap(); + let ConnTaskRunning { + cancel_on_client_drop, + conn_task, + } = { + let mut guard = shared.lock().unwrap(); + match std::mem::replace( + &mut *guard, + PagestreamShared::ConnTaskCancelledJoinHandleReturnedOrDropped, + ) { + PagestreamShared::ConnTaskRunning(conn_task_running) => conn_task_running, + PagestreamShared::ConnTaskCancelledJoinHandleReturnedOrDropped => unreachable!(), + } + }; + let _ = cancel_on_client_drop.unwrap(); conn_task.await.unwrap(); - drop(copy_both); + + // Now drop the split copy_both. + drop(sink); + drop(stream); + } + + pub fn split(self) -> (PagestreamSender, PagestreamReceiver) { + let Self { + shared, + sink, + stream, + } = self; + (sink, stream) } pub async fn getpage( &mut self, req: PagestreamGetPageRequest, ) -> anyhow::Result { - let req = PagestreamFeMessage::GetPage(req); - let req: bytes::Bytes = req.serialize(); - // let mut req = tokio_util::io::ReaderStream::new(&req); - let mut req = tokio_stream::once(Ok(req)); + self.getpage_send(req).await?; + self.getpage_recv().await + } - self.copy_both.send_all(&mut req).await?; + pub async fn getpage_send(&mut self, req: PagestreamGetPageRequest) -> anyhow::Result<()> { + self.sink.getpage_send(req).await + } - let next: Option> = self.copy_both.next().await; + pub async fn getpage_recv(&mut self) -> anyhow::Result { + self.stream.getpage_recv().await + } +} + +impl PagestreamSender { + // TODO: maybe make this impl Sink instead for better composability? + pub async fn send(&mut self, msg: PagestreamFeMessage) -> anyhow::Result<()> { + let msg = msg.serialize(); + self.sink.send_all(&mut tokio_stream::once(Ok(msg))).await?; + Ok(()) + } + + pub async fn getpage_send(&mut self, req: PagestreamGetPageRequest) -> anyhow::Result<()> { + self.send(PagestreamFeMessage::GetPage(req)).await + } +} + +impl PagestreamReceiver { + pub async fn recv(&mut self) -> anyhow::Result { + let next: Option> = self.stream.next().await; let next: bytes::Bytes = next.unwrap()?; + Ok(PagestreamBeMessage::deserialize(next)?) + } - let msg = PagestreamBeMessage::deserialize(next)?; - match msg { + pub async fn getpage_recv(&mut self) -> anyhow::Result { + let next: PagestreamBeMessage = self.recv().await?; + match next { PagestreamBeMessage::GetPage(p) => Ok(p), PagestreamBeMessage::Error(e) => anyhow::bail!("Error: {:?}", e), PagestreamBeMessage::Exists(_) | PagestreamBeMessage::Nblocks(_) | PagestreamBeMessage::DbSize(_) - | PagestreamBeMessage::GetSlruSegment(_) => { + | PagestreamBeMessage::GetSlruSegment(_) + | PagestreamBeMessage::Test(_) => { anyhow::bail!( "unexpected be message kind in response to getpage request: {}", - msg.kind() + next.kind() ) } } diff --git a/pageserver/src/bin/test_helper_slow_client_reads.rs b/pageserver/src/bin/test_helper_slow_client_reads.rs new file mode 100644 index 0000000000..6fad937bf5 --- /dev/null +++ b/pageserver/src/bin/test_helper_slow_client_reads.rs @@ -0,0 +1,63 @@ +use std::{ + io::{stdin, stdout, Read, Write}, + time::Duration, +}; + +use clap::Parser; +use pageserver_api::models::{PagestreamGetPageRequest, PagestreamRequest, PagestreamTestRequest}; +use utils::{ + id::{TenantId, TimelineId}, + lsn::Lsn, + shard::TenantShardId, +}; + +#[derive(clap::Parser)] +struct Args { + connstr: String, + tenant_id: TenantId, + timeline_id: TimelineId, +} + +#[tokio::main] +async fn main() -> anyhow::Result<()> { + let Args { + connstr, + tenant_id, + timeline_id, + } = Args::parse(); + let client = pageserver_client::page_service::Client::new(connstr).await?; + let client = client.pagestream(tenant_id, timeline_id).await?; + let (mut sender, mut receiver) = client.split(); + + eprintln!("filling the pipe"); + let mut msg = 0; + loop { + msg += 1; + let fut = sender.send(pageserver_api::models::PagestreamFeMessage::Test( + PagestreamTestRequest { + hdr: PagestreamRequest { + reqid: 0, + request_lsn: Lsn(23), + not_modified_since: Lsn(23), + }, + batch_key: 42, + message: format!("message {}", msg), + }, + )); + let Ok(res) = tokio::time::timeout(Duration::from_secs(1), fut).await else { + eprintln!("pipe seems full"); + break; + }; + let _: () = res?; + } + + stdout().write(b"R")?; + stdout().flush()?; + + let mut buf = [0u8; 1]; + stdin().read_exact(&mut buf)?; + + eprintln!("termination signal received, exiting"); + + anyhow::Ok(()) +} diff --git a/pageserver/src/metrics.rs b/pageserver/src/metrics.rs index d478aa87fa..9b877fc368 100644 --- a/pageserver/src/metrics.rs +++ b/pageserver/src/metrics.rs @@ -1472,6 +1472,8 @@ pub enum SmgrQueryType { GetPageAtLsn, GetDbSize, GetSlruSegment, + #[cfg(feature = "testing")] + Test, } pub(crate) struct SmgrQueryTimePerTimeline { diff --git a/pageserver/src/page_service.rs b/pageserver/src/page_service.rs index 8b4bab3385..301f248517 100644 --- a/pageserver/src/page_service.rs +++ b/pageserver/src/page_service.rs @@ -555,6 +555,12 @@ struct BatchedGetPageRequest { timer: SmgrOpTimer, } +#[cfg(feature = "testing")] +struct BatchedTestRequest { + req: models::PagestreamTestRequest, + timer: SmgrOpTimer, +} + enum BatchedFeMessage { Exists { span: Span, @@ -586,6 +592,12 @@ enum BatchedFeMessage { shard: timeline::handle::Handle, req: models::PagestreamGetSlruSegmentRequest, }, + #[cfg(feature = "testing")] + Test { + span: Span, + shard: timeline::handle::Handle, + requests: Vec, + }, RespondError { span: Span, error: BatchedPageStreamError, @@ -606,6 +618,11 @@ impl BatchedFeMessage { page.timer.observe_execution_start(at); } } + BatchedFeMessage::Test { requests, .. } => { + for req in requests { + req.timer.observe_execution_start(at); + } + } BatchedFeMessage::RespondError { .. } => {} } } @@ -866,6 +883,22 @@ impl PageServerHandler { pages: smallvec::smallvec![BatchedGetPageRequest { req, timer }], } } + #[cfg(feature = "testing")] + PagestreamFeMessage::Test(req) => { + let span = tracing::info_span!(parent: parent_span, "handle_test_request"); + let shard = timeline_handles + .get(tenant_id, timeline_id, ShardSelector::Zero) + .instrument(span.clone()) // sets `shard_id` field + .await?; + let timer = + record_op_start_and_throttle(&shard, metrics::SmgrQueryType::Test, received_at) + .await?; + BatchedFeMessage::Test { + span, + shard, + requests: vec![BatchedTestRequest { req, timer }], + } + } }; Ok(Some(batched_msg)) } @@ -928,6 +961,45 @@ impl PageServerHandler { accum_pages.extend(this_pages); Ok(()) } + ( + Ok(BatchedFeMessage::Test { + shard: accum_shard, + requests: accum_requests, + .. + }), + BatchedFeMessage::Test { + shard: this_shard, + requests: this_requests, + .. + }, + ) if (|| { + assert!(this_requests.len() == 1); + if accum_requests.len() >= max_batch_size.get() { + trace!(%max_batch_size, "stopping batching because of batch size"); + assert_eq!(accum_requests.len(), max_batch_size.get()); + return false; + } + if (accum_shard.tenant_shard_id, accum_shard.timeline_id) + != (this_shard.tenant_shard_id, this_shard.timeline_id) + { + trace!("stopping batching because timeline object mismatch"); + // TODO: we _could_ batch & execute each shard seperately (and in parallel). + // But the current logic for keeping responses in order does not support that. + return false; + } + let this_batch_key = this_requests[0].req.batch_key; + let accum_batch_key = accum_requests[0].req.batch_key; + if this_requests[0].req.batch_key != accum_requests[0].req.batch_key { + trace!(%accum_batch_key, %this_batch_key, "stopping batching because batch key changed"); + return false; + } + true + })() => + { + // ok to batch + accum_requests.extend(this_requests); + Ok(()) + } // something batched already but this message is unbatchable (_, this_msg) => { // by default, don't continue batching @@ -1054,6 +1126,27 @@ impl PageServerHandler { span, ) } + #[cfg(feature = "testing")] + BatchedFeMessage::Test { + span, + shard, + requests, + } => { + fail::fail_point!("ps::handle-pagerequest-message::test"); + ( + { + let npages = requests.len(); + trace!(npages, "handling getpage request"); + let res = self + .handle_test_request_batch(&shard, requests, ctx) + .instrument(span.clone()) + .await; + assert_eq!(res.len(), npages); + res + }, + span, + ) + } BatchedFeMessage::RespondError { span, error } => { // We've already decided to respond with an error, so we don't need to // call the handler. @@ -1780,6 +1873,54 @@ impl PageServerHandler { )) } + #[cfg(feature = "testing")] + #[instrument(skip_all, fields(shard_id))] + async fn handle_test_request_batch( + &mut self, + timeline: &Timeline, + requests: Vec, + ctx: &RequestContext, + ) -> Vec> { + use pageserver_api::models::PagestreamTestResponse; + + let latest_gc_cutoff_lsn = timeline.get_latest_gc_cutoff_lsn(); + + // real requests would do something with the timeline + let mut results = Vec::with_capacity(requests.len()); + for req in requests.iter() { + tokio::task::yield_now().await; + + results.push({ + if timeline.cancel.is_cancelled() { + Err(PageStreamError::Shutdown) + } else { + Ok(PagestreamTestResponse { req: req.req.clone() }) + } + }); + } + + // TODO: avoid creating the new Vec here + Vec::from_iter( + requests + .into_iter() + .zip(results.into_iter()) + .map(|(req, res)| { + res.map(|page| { + ( + PagestreamBeMessage::Test(models::PagestreamTestResponse { + req: req.req.clone(), + }), + req.timer, + ) + }) + .map_err(|e| BatchedPageStreamError { + err: PageStreamError::from(e), + req: req.req.hdr, + }) + }), + ) + } + /// Note on "fullbackup": /// Full basebackups should only be used for debugging purposes. /// Originally, it was introduced to enable breaking storage format changes, diff --git a/test_runner/regress/test_page_service_batching_regressions.py b/test_runner/regress/test_page_service_batching_regressions.py index 4d456cad79..e4312644b4 100644 --- a/test_runner/regress/test_page_service_batching_regressions.py +++ b/test_runner/regress/test_page_service_batching_regressions.py @@ -1,19 +1,13 @@ # NB: there are benchmarks that double-serve as tests inside the `performance` directory. -import threading -import time +import subprocess +from pathlib import Path -import requests.exceptions - -import fixtures -from fixtures.common_types import NodeId from fixtures.log_helper import log -from fixtures.neon_fixtures import NeonEnvBuilder, StorageControllerApiException +from fixtures.neon_fixtures import NeonEnvBuilder -def test_slow_flush(neon_env_builder: NeonEnvBuilder): - tablesize_mib = 500 - +def test_slow_flush(neon_env_builder: NeonEnvBuilder, neon_binpath: Path): def patch_pageserver_toml(config): config["page_service_pipelining"] = { "mode": "pipelined", @@ -22,84 +16,30 @@ def test_slow_flush(neon_env_builder: NeonEnvBuilder): } neon_env_builder.pageserver_config_override = patch_pageserver_toml - neon_env_builder.num_pageservers = 2 env = neon_env_builder.init_start() - ep = env.endpoints.create_start( - "main", - config_lines=[ - "max_parallel_workers_per_gather=0", # disable parallel backends - "effective_io_concurrency=100", # give plenty of opportunity for pipelining - "neon.readahead_buffer_size=128", # this is the default value at time of writing - "shared_buffers=128MB", # keep lower than tablesize_mib - # debug - "log_statement=all", + log.info("make flush appear slow") + ps_http = env.pageserver.http_client() + ps_http.configure_failpoints(("page_service:flush:pre", "return(10000000)")) + + log.info("filling pipe") + child = subprocess.Popen( + [ + neon_binpath / "test_helper_slow_client_reads", + env.pageserver.connstr(), + str(env.initial_tenant), + str(env.initial_timeline), ], + bufsize=0, # unbuffered + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, ) + buf = child.stdout.read(1) + if len(buf) != 1: + raise Exception("unexpected EOF") + if buf != b"R": + raise Exception(f"unexpected data: {buf!r}") + log.info("helper reports pipe filled") - conn = ep.connect() - cur = conn.cursor() - - cur.execute("CREATE EXTENSION IF NOT EXISTS neon;") - cur.execute("CREATE EXTENSION IF NOT EXISTS neon_test_utils;") - - log.info("Filling the table") - cur.execute("CREATE TABLE t (data char(1000)) with (fillfactor=10)") - tablesize = tablesize_mib * 1024 * 1024 - npages = tablesize // (8 * 1024) - cur.execute("INSERT INTO t SELECT generate_series(1, %s)", (npages,)) - - cur.close() - conn.close() - - def workload(stop: threading.Event, max_iters=None): - iters = 0 - while stop.is_set() is False and (max_iters == None or iters < max_iters): - log.info("Seqscan %d", iters) - conn = ep.connect() - cur = conn.cursor() - cur.execute( - "select clear_buffer_cache()" - ) # TODO: what about LFC? doesn't matter right now because LFC isn't enabled by default in tests - cur.execute("select sum(data::bigint) from t") - assert cur.fetchall()[0][0] == npages * (npages + 1) // 2 - iters += 1 - log.info("workload done") - - stop = threading.Event() - - log.info("calibrating workload duration") - workload(stop, 1) - before = time.time() - workload(stop, 1) - after = time.time() - duration = after - before - log.info("duration: %f", duration) - assert(duration > 3) - - log.info("begin") - threading.Thread(target=workload, args=[stop]).start() - - # make flush appear slow - ps_http = [p.http_client() for p in env.pageservers] - ps_http[0].configure_failpoints(("page_service:flush:pre", "return(10000000)")) - ps_http[1].configure_failpoints(("page_service:flush:pre", "return(10000000)")) - - time.sleep(1) - - # try to shut down the tenant - for i in range(1, 10): - log.info(f"start migration {i}") - try: - env.storage_controller.tenant_shard_migrate(env.initial_tenant, (i % 2)+1) - except StorageControllerApiException as e: - log.info(f"shard migrate request failed: {e}") - while True: - node_id = NodeId(env.storage_controller.tenant_describe(env.initial_tenant)["node_id"]) - if node_id == NodeId(i % 2)+1: - break - log.info(f"waiting for migration to complete") - time.sleep(1) - log.info(f"migration done") - time.sleep(1) - + log.info("try to shut down the tenant") + env.pageserver.tenant_detach(env.initial_tenant) From 8454e19a0ff4077154540d9dd6619a81a7526dc5 Mon Sep 17 00:00:00 2001 From: Christian Schwarz Date: Tue, 14 Jan 2025 22:28:08 +0100 Subject: [PATCH 102/111] address warnings and such --- pageserver/client/src/page_service.rs | 4 +++- .../src/bin/test_helper_slow_client_reads.rs | 9 +++++---- pageserver/src/page_service.rs | 19 ++++++------------- .../test_page_service_batching_regressions.py | 2 -- 4 files changed, 14 insertions(+), 20 deletions(-) diff --git a/pageserver/client/src/page_service.rs b/pageserver/client/src/page_service.rs index 3827e19aa4..f312b888b6 100644 --- a/pageserver/client/src/page_service.rs +++ b/pageserver/client/src/page_service.rs @@ -118,11 +118,13 @@ pub struct PagestreamClient { } pub struct PagestreamSender { + #[allow(dead_code)] shared: Arc>, sink: SplitSink, bytes::Bytes>, } pub struct PagestreamReceiver { + #[allow(dead_code)] shared: Arc>, stream: SplitStream>, } @@ -188,7 +190,7 @@ impl PagestreamClient { pub fn split(self) -> (PagestreamSender, PagestreamReceiver) { let Self { - shared, + shared: _, sink, stream, } = self; diff --git a/pageserver/src/bin/test_helper_slow_client_reads.rs b/pageserver/src/bin/test_helper_slow_client_reads.rs index 6fad937bf5..9a812e436d 100644 --- a/pageserver/src/bin/test_helper_slow_client_reads.rs +++ b/pageserver/src/bin/test_helper_slow_client_reads.rs @@ -4,11 +4,10 @@ use std::{ }; use clap::Parser; -use pageserver_api::models::{PagestreamGetPageRequest, PagestreamRequest, PagestreamTestRequest}; +use pageserver_api::models::{PagestreamRequest, PagestreamTestRequest}; use utils::{ id::{TenantId, TimelineId}, lsn::Lsn, - shard::TenantShardId, }; #[derive(clap::Parser)] @@ -27,7 +26,7 @@ async fn main() -> anyhow::Result<()> { } = Args::parse(); let client = pageserver_client::page_service::Client::new(connstr).await?; let client = client.pagestream(tenant_id, timeline_id).await?; - let (mut sender, mut receiver) = client.split(); + let (mut sender, _receiver) = client.split(); eprintln!("filling the pipe"); let mut msg = 0; @@ -44,7 +43,7 @@ async fn main() -> anyhow::Result<()> { message: format!("message {}", msg), }, )); - let Ok(res) = tokio::time::timeout(Duration::from_secs(1), fut).await else { + let Ok(res) = tokio::time::timeout(Duration::from_secs(10), fut).await else { eprintln!("pipe seems full"); break; }; @@ -54,6 +53,8 @@ async fn main() -> anyhow::Result<()> { stdout().write(b"R")?; stdout().flush()?; + eprintln!("waiting for signal to tell us to exit"); + let mut buf = [0u8; 1]; stdin().read_exact(&mut buf)?; diff --git a/pageserver/src/page_service.rs b/pageserver/src/page_service.rs index 301f248517..96d1a826a9 100644 --- a/pageserver/src/page_service.rs +++ b/pageserver/src/page_service.rs @@ -39,7 +39,6 @@ use tokio::io::{AsyncWriteExt, BufWriter}; use tokio::task::JoinHandle; use tokio_util::sync::CancellationToken; use tracing::*; -use utils::failpoint_support; use utils::sync::spsc_fold; use utils::{ auth::{Claims, Scope, SwappableJwtAuth}, @@ -1221,10 +1220,7 @@ impl PageServerHandler { }); // what we want to do - let flush_fut = async { - failpoint_support::sleep_millis_async!("page_service:flush:pre"); - pgb_writer.flush().await - }; + let flush_fut = pgb_writer.flush(); // metric for how long flushing takes let flush_fut = match flushing_timer { Some(flushing_timer) => { @@ -1873,28 +1869,25 @@ impl PageServerHandler { )) } + // NB: this impl mimics what we do for batched getpage requests. #[cfg(feature = "testing")] #[instrument(skip_all, fields(shard_id))] async fn handle_test_request_batch( &mut self, timeline: &Timeline, requests: Vec, - ctx: &RequestContext, + _ctx: &RequestContext, ) -> Vec> { - use pageserver_api::models::PagestreamTestResponse; - - let latest_gc_cutoff_lsn = timeline.get_latest_gc_cutoff_lsn(); - // real requests would do something with the timeline let mut results = Vec::with_capacity(requests.len()); - for req in requests.iter() { + for _req in requests.iter() { tokio::task::yield_now().await; results.push({ if timeline.cancel.is_cancelled() { Err(PageStreamError::Shutdown) } else { - Ok(PagestreamTestResponse { req: req.req.clone() }) + Ok(()) } }); } @@ -1905,7 +1898,7 @@ impl PageServerHandler { .into_iter() .zip(results.into_iter()) .map(|(req, res)| { - res.map(|page| { + res.map(|()| { ( PagestreamBeMessage::Test(models::PagestreamTestResponse { req: req.req.clone(), diff --git a/test_runner/regress/test_page_service_batching_regressions.py b/test_runner/regress/test_page_service_batching_regressions.py index e4312644b4..f781dd4850 100644 --- a/test_runner/regress/test_page_service_batching_regressions.py +++ b/test_runner/regress/test_page_service_batching_regressions.py @@ -19,8 +19,6 @@ def test_slow_flush(neon_env_builder: NeonEnvBuilder, neon_binpath: Path): env = neon_env_builder.init_start() log.info("make flush appear slow") - ps_http = env.pageserver.http_client() - ps_http.configure_failpoints(("page_service:flush:pre", "return(10000000)")) log.info("filling pipe") child = subprocess.Popen( From 53b05c4ba03633dd7e96e5c420f7148f203fc57a Mon Sep 17 00:00:00 2001 From: Christian Schwarz Date: Tue, 14 Jan 2025 22:45:09 +0100 Subject: [PATCH 103/111] cleanups to make CI pass (well, fail because the bug isn't fixed yet) --- libs/pageserver_api/src/models.rs | 19 ++++++++++++------- pageserver/Cargo.toml | 4 ++++ pageserver/client/Cargo.toml | 3 +++ pageserver/client/src/page_service.rs | 13 ++++++++++--- .../src/bin/test_helper_slow_client_reads.rs | 3 ++- pageserver/src/page_service.rs | 4 +++- .../test_page_service_batching_regressions.py | 1 + 7 files changed, 35 insertions(+), 12 deletions(-) diff --git a/libs/pageserver_api/src/models.rs b/libs/pageserver_api/src/models.rs index 2e6949e6ce..7e08b2825a 100644 --- a/libs/pageserver_api/src/models.rs +++ b/libs/pageserver_api/src/models.rs @@ -1641,15 +1641,16 @@ impl PagestreamFeMessage { bytes.put_u8(req.kind); bytes.put_u32(req.segno); } - + #[cfg(feature = "testing")] Self::Test(req) => { bytes.put_u8(5); bytes.put_u64(req.hdr.reqid); bytes.put_u64(req.hdr.request_lsn.0); bytes.put_u64(req.hdr.not_modified_since.0); bytes.put_u64(req.batch_key); - bytes.put_u64(req.message.as_bytes().len() as u64); - bytes.put_slice(req.message.as_bytes()); + let message = req.message.as_bytes(); + bytes.put_u64(message.len() as u64); + bytes.put_slice(message); } } @@ -1797,11 +1798,13 @@ impl PagestreamBeMessage { bytes.put(&resp.segment[..]); } + #[cfg(feature = "testing")] Self::Test(resp) => { bytes.put_u8(Tag::Test as u8); bytes.put_u64(resp.req.batch_key); - bytes.put_u64(resp.req.message.as_bytes().len() as u64); - bytes.put_slice(resp.req.message.as_bytes()); + let message = resp.req.message.as_bytes(); + bytes.put_u64(message.len() as u64); + bytes.put_slice(message); } } } @@ -1872,14 +1875,16 @@ impl PagestreamBeMessage { bytes.put(&resp.segment[..]); } + #[cfg(feature = "testing")] Self::Test(resp) => { bytes.put_u8(Tag::Test as u8); bytes.put_u64(resp.req.hdr.reqid); bytes.put_u64(resp.req.hdr.request_lsn.0); bytes.put_u64(resp.req.hdr.not_modified_since.0); bytes.put_u64(resp.req.batch_key); - bytes.put_u64(resp.req.message.as_bytes().len() as u64); - bytes.put_slice(resp.req.message.as_bytes()); + let message = resp.req.message.as_bytes(); + bytes.put_u64(message.len() as u64); + bytes.put_slice(message); } } } diff --git a/pageserver/Cargo.toml b/pageserver/Cargo.toml index 8547746d94..0fe3b8337b 100644 --- a/pageserver/Cargo.toml +++ b/pageserver/Cargo.toml @@ -113,3 +113,7 @@ harness = false [[bench]] name = "upload_queue" harness = false + +[[bin]] +name = "test_helper_slow_client_reads" +required-features = [ "testing" ] diff --git a/pageserver/client/Cargo.toml b/pageserver/client/Cargo.toml index d9b36bf3d4..f582d307a7 100644 --- a/pageserver/client/Cargo.toml +++ b/pageserver/client/Cargo.toml @@ -4,6 +4,9 @@ version = "0.1.0" edition.workspace = true license.workspace = true +[features] +testing = [ "pageserver_api/testing" ] + [dependencies] pageserver_api.workspace = true thiserror.workspace = true diff --git a/pageserver/client/src/page_service.rs b/pageserver/client/src/page_service.rs index f312b888b6..27280912b4 100644 --- a/pageserver/client/src/page_service.rs +++ b/pageserver/client/src/page_service.rs @@ -228,10 +228,11 @@ impl PagestreamSender { } impl PagestreamReceiver { + // TODO: maybe make this impl Stream instead for better composability? pub async fn recv(&mut self) -> anyhow::Result { let next: Option> = self.stream.next().await; let next: bytes::Bytes = next.unwrap()?; - Ok(PagestreamBeMessage::deserialize(next)?) + PagestreamBeMessage::deserialize(next) } pub async fn getpage_recv(&mut self) -> anyhow::Result { @@ -242,8 +243,14 @@ impl PagestreamReceiver { PagestreamBeMessage::Exists(_) | PagestreamBeMessage::Nblocks(_) | PagestreamBeMessage::DbSize(_) - | PagestreamBeMessage::GetSlruSegment(_) - | PagestreamBeMessage::Test(_) => { + | PagestreamBeMessage::GetSlruSegment(_) => { + anyhow::bail!( + "unexpected be message kind in response to getpage request: {}", + next.kind() + ) + } + #[cfg(feature = "testing")] + PagestreamBeMessage::Test(_) => { anyhow::bail!( "unexpected be message kind in response to getpage request: {}", next.kind() diff --git a/pageserver/src/bin/test_helper_slow_client_reads.rs b/pageserver/src/bin/test_helper_slow_client_reads.rs index 9a812e436d..c1ce332b6c 100644 --- a/pageserver/src/bin/test_helper_slow_client_reads.rs +++ b/pageserver/src/bin/test_helper_slow_client_reads.rs @@ -50,7 +50,8 @@ async fn main() -> anyhow::Result<()> { let _: () = res?; } - stdout().write(b"R")?; + let n = stdout().write(b"R")?; + assert_eq!(n, 1); stdout().flush()?; eprintln!("waiting for signal to tell us to exit"); diff --git a/pageserver/src/page_service.rs b/pageserver/src/page_service.rs index 96d1a826a9..f4a7517bbd 100644 --- a/pageserver/src/page_service.rs +++ b/pageserver/src/page_service.rs @@ -617,6 +617,7 @@ impl BatchedFeMessage { page.timer.observe_execution_start(at); } } + #[cfg(feature = "testing")] BatchedFeMessage::Test { requests, .. } => { for req in requests { req.timer.observe_execution_start(at); @@ -960,6 +961,7 @@ impl PageServerHandler { accum_pages.extend(this_pages); Ok(()) } + #[cfg(feature = "testing")] ( Ok(BatchedFeMessage::Test { shard: accum_shard, @@ -1885,7 +1887,7 @@ impl PageServerHandler { results.push({ if timeline.cancel.is_cancelled() { - Err(PageStreamError::Shutdown) + Err(PageReconstructError::Cancelled) } else { Ok(()) } diff --git a/test_runner/regress/test_page_service_batching_regressions.py b/test_runner/regress/test_page_service_batching_regressions.py index f781dd4850..46afc4cc3d 100644 --- a/test_runner/regress/test_page_service_batching_regressions.py +++ b/test_runner/regress/test_page_service_batching_regressions.py @@ -32,6 +32,7 @@ def test_slow_flush(neon_env_builder: NeonEnvBuilder, neon_binpath: Path): stdin=subprocess.PIPE, stdout=subprocess.PIPE, ) + assert child.stdout is not None buf = child.stdout.read(1) if len(buf) != 1: raise Exception("unexpected EOF") From 5450e54dab407ce53491dafcbe0fb7c81b4b4f20 Mon Sep 17 00:00:00 2001 From: Christian Schwarz Date: Tue, 14 Jan 2025 22:47:16 +0100 Subject: [PATCH 104/111] bump ci From a8f9b564beadb9b3bca304d492a11365e06f3efb Mon Sep 17 00:00:00 2001 From: Christian Schwarz Date: Tue, 14 Jan 2025 23:50:22 +0100 Subject: [PATCH 105/111] fix `cd pageserver && cargo clippy --features testing` build --- pageserver/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pageserver/Cargo.toml b/pageserver/Cargo.toml index 0fe3b8337b..0665c73a40 100644 --- a/pageserver/Cargo.toml +++ b/pageserver/Cargo.toml @@ -8,7 +8,7 @@ license.workspace = true default = [] # Enables test-only APIs, incuding failpoints. In particular, enables the `fail_point!` macro, # which adds some runtime cost to run tests on outage conditions -testing = ["fail/failpoints", "pageserver_api/testing", "wal_decoder/testing"] +testing = ["fail/failpoints", "pageserver_api/testing", "wal_decoder/testing", "pageserver_client/testing"] [dependencies] anyhow.workspace = true From 0340f00228962ee264bf902b2683476741ebae60 Mon Sep 17 00:00:00 2001 From: Christian Schwarz Date: Tue, 14 Jan 2025 23:56:35 +0100 Subject: [PATCH 106/111] post-merge fix the handling of the new pagestream Test message, so that the regression test now passes non-package-mode-py3.10christian@neon-hetzner-dev-christian:[~/src/neon]: BUILD_TYPE=debug DEFAULT_PG_VERSION=16 poetry run pytest ./test_runner/regress/test_page_service_batching_regressions.py --timeout=0 --pdb --- pageserver/src/page_service.rs | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/pageserver/src/page_service.rs b/pageserver/src/page_service.rs index 6052ad5d2e..685eea8541 100644 --- a/pageserver/src/page_service.rs +++ b/pageserver/src/page_service.rs @@ -594,7 +594,7 @@ enum BatchedFeMessage { #[cfg(feature = "testing")] Test { span: Span, - shard: timeline::handle::Handle, + shard: timeline::handle::WeakHandle, requests: Vec, }, RespondError { @@ -896,7 +896,7 @@ impl PageServerHandler { .await?; BatchedFeMessage::Test { span, - shard, + shard: shard.downgrade(), requests: vec![BatchedTestRequest { req, timer }], } } @@ -979,9 +979,7 @@ impl PageServerHandler { assert_eq!(accum_requests.len(), max_batch_size.get()); return false; } - if (accum_shard.tenant_shard_id, accum_shard.timeline_id) - != (this_shard.tenant_shard_id, this_shard.timeline_id) - { + if !Arc::ptr_eq(accum_shard.timeline(), this_shard.timeline()) { trace!("stopping batching because timeline object mismatch"); // TODO: we _could_ batch & execute each shard seperately (and in parallel). // But the current logic for keeping responses in order does not support that. @@ -1138,7 +1136,7 @@ impl PageServerHandler { let npages = requests.len(); trace!(npages, "handling getpage request"); let res = self - .handle_test_request_batch(&shard, requests, ctx) + .handle_test_request_batch(&*shard.upgrade()?, requests, ctx) .instrument(span.clone()) .await; assert_eq!(res.len(), npages); From dedd524d7eded937ecbd7df9d168b8cd588d7465 Mon Sep 17 00:00:00 2001 From: Christian Schwarz Date: Wed, 15 Jan 2025 00:21:28 +0100 Subject: [PATCH 107/111] refinements --- .../test_page_service_batching_regressions.py | 26 +++++++++++++++---- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/test_runner/regress/test_page_service_batching_regressions.py b/test_runner/regress/test_page_service_batching_regressions.py index 46afc4cc3d..302cc1ba22 100644 --- a/test_runner/regress/test_page_service_batching_regressions.py +++ b/test_runner/regress/test_page_service_batching_regressions.py @@ -3,11 +3,14 @@ import subprocess from pathlib import Path +import pytest from fixtures.log_helper import log from fixtures.neon_fixtures import NeonEnvBuilder -def test_slow_flush(neon_env_builder: NeonEnvBuilder, neon_binpath: Path): +@pytest.mark.timeout(30) # test takes <20s if pageserver impl is correct +@pytest.mark.parametrize("kind", ["pageserver-stop", "tenant-detach"]) +def test_slow_flush(neon_env_builder: NeonEnvBuilder, neon_binpath: Path, kind: str): def patch_pageserver_toml(config): config["page_service_pipelining"] = { "mode": "pipelined", @@ -20,7 +23,9 @@ def test_slow_flush(neon_env_builder: NeonEnvBuilder, neon_binpath: Path): log.info("make flush appear slow") - log.info("filling pipe") + log.info("sending requests until pageserver accepts no more") + # TODO: extract this into a helper, like subprocess_capture, + # so that we capture the stderr from the helper somewhere. child = subprocess.Popen( [ neon_binpath / "test_helper_slow_client_reads", @@ -38,7 +43,18 @@ def test_slow_flush(neon_env_builder: NeonEnvBuilder, neon_binpath: Path): raise Exception("unexpected EOF") if buf != b"R": raise Exception(f"unexpected data: {buf!r}") - log.info("helper reports pipe filled") + log.info("helper reports pageserver accepts no more requests") + log.info( + "assuming pageserver connection handle is in a state where TCP has backpressured pageserver=>client response flush() into userspace" + ) - log.info("try to shut down the tenant") - env.pageserver.tenant_detach(env.initial_tenant) + if kind == "pageserver": + log.info("try to shut down the pageserver cleanly") + env.pageserver.stop() + elif kind == "tenant-detach": + log.info("try to shut down the tenant") + env.pageserver.tenant_detach(env.initial_tenant) + else: + raise ValueError(f"unexpected kind: {kind}") + + log.info("shutdown did not time out, test passed") From 173f18832c75142a31a0d6cf013d5fb7200ef923 Mon Sep 17 00:00:00 2001 From: Christian Schwarz Date: Wed, 15 Jan 2025 00:24:59 +0100 Subject: [PATCH 108/111] fixup --- test_runner/regress/test_page_service_batching_regressions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_runner/regress/test_page_service_batching_regressions.py b/test_runner/regress/test_page_service_batching_regressions.py index 302cc1ba22..fa85e1210b 100644 --- a/test_runner/regress/test_page_service_batching_regressions.py +++ b/test_runner/regress/test_page_service_batching_regressions.py @@ -48,7 +48,7 @@ def test_slow_flush(neon_env_builder: NeonEnvBuilder, neon_binpath: Path, kind: "assuming pageserver connection handle is in a state where TCP has backpressured pageserver=>client response flush() into userspace" ) - if kind == "pageserver": + if kind == "pageserver-stop": log.info("try to shut down the pageserver cleanly") env.pageserver.stop() elif kind == "tenant-detach": From 22057362626cfff0f46710338bdb9aa78f86d623 Mon Sep 17 00:00:00 2001 From: Christian Schwarz Date: Wed, 15 Jan 2025 14:24:29 +0100 Subject: [PATCH 109/111] doc comment & one fixup --- pageserver/src/tenant/timeline/handle.rs | 190 +++++++++++++++++------ 1 file changed, 144 insertions(+), 46 deletions(-) diff --git a/pageserver/src/tenant/timeline/handle.rs b/pageserver/src/tenant/timeline/handle.rs index 80a4ec06e5..59f399d01a 100644 --- a/pageserver/src/tenant/timeline/handle.rs +++ b/pageserver/src/tenant/timeline/handle.rs @@ -32,54 +32,141 @@ //! //! # Design //! +//! ## Data Structures +//! //! There are three user-facing data structures: //! - `PerTimelineState`: a struct embedded into each Timeline struct. Lifetime == Timeline lifetime. //! - `Cache`: a struct private to each connection handler; Lifetime == connection lifetime. //! - `Handle`: a smart pointer that holds the Timeline gate open and derefs to `&Timeline`. -//! Lifetime: for a single request dispatch on the Timeline (i.e., one getpage request) +//! - `WeakHandle`: downgrade of a `Handle` that does not keep the gate open, but allows +//! trying to ugprade back to a `Handle`, guaranteeing it's the same `Timeline` *object*. //! -//! The `Handle` is just a wrapper around an `Arc`. +//! Internally, there is 0 or 1 `HandleInner` per `(Cache,Timeline)`. +//! Since Cache:Connection is 1:1, there is 0 or 1 `HandleInner` per `(Connection,Timeline)`. //! -//! There is one long-lived `Arc`, which is stored in the `PerTimelineState`. -//! The `Cache` stores a `Weak` for each cached Timeline. +//! The `HandleInner` is allocated as a `Arc>` and +//! referenced weakly and strongly from various places which we are now illustrating. +//! For bevity, we will omit the `Arc>` part in the following. +//! +//! - The `Handle` is a strong ref. +//! - The `WeakHandle` is a weak ref. +//! - The only long-lived strong reference is stored in the `PerTimelineState`. +//! - The `Cache` stores a weak ref. +//! +//! Lifetimes: +//! - `WeakHandle` and `Handle`: single pagestream request. +//! - `Cache`: single page service connection. +//! - `PerTimelineState`: lifetime of the Timeline object (i.e., i.e., till `Timeline::shutdown`). +//! +//! ## Request Handling Flow //! //! To dispatch a request, the page service connection calls `Cache::get`. //! //! A cache miss means we consult the tenant manager for shard routing, -//! resulting in an `Arc`. We enter its gate _once_ and construct an -//! `Arc`. We store a `Weak` in the cache -//! and the `Arc` in the `PerTimelineState`. +//! resulting in an `Arc`. We enter its gate _once_ and store it in the the +//! `Arc>>`. A weak ref is stored in the `Cache` +//! and a strong ref in the `PerTimelineState`. +//! A strong ref is returned wrapped in a `Handle`. //! //! For subsequent requests, `Cache::get` will perform a "fast path" shard routing -//! and find the `Weak` in the cache. -//! We upgrade the `Weak` to an `Arc` and wrap it in the user-facing `Handle` type. +//! and find the weak ref in the cache. +//! We upgrade the weak ref to a strong ref and return it wrapped in a `Handle`. //! -//! The request handler dispatches the request to the right `>::$request_method`. +//! The pagestream processing is pipelined and involves a batching step. +//! While a request is batching, the `Handle` is downgraded to a `WeakHandle`. +//! When the batch is ready to be executed, the `WeakHandle` is upgraded back to a `Handle` +//! and the request handler dispatches the request to the right `>::$request_method`. //! It then drops the `Handle`, which drops the `Arc`. //! -//! # Memory Management / How The Reference Cycle Is Broken +//! # Performance //! -//! The attentive reader may have noticed the strong reference cycle -//! from `Arc` to `PerTimelineState` to `Arc`. +//! Remember from the introductory section: //! -//! This cycle is intentional: while it exists, the `Cache` can upgrade its -//! `Weak` to an `Arc` in a single atomic operation. +//! > However, we want to avoid the overhead of entering the gate for every +//! > method invocation. +//! +//! Why do we want to avoid that? +//! Because the gate is a shared location in memory and entering it invovles +//! bumping refcounts, which leads to cache contention if done frequently +//! from multiple cores in parallel. +//! +//! So, we only acquire the `GateGuard` once on `Cache` miss, and wrap it in an `Arc`. +//! That `Arc` is private to the `HandleInner` and hence to the connection. +//! (Review the "Data Structures" section if that is unclear to you.) +//! +//! A `WeakHandle` is a weak ref to the HandleInner. +//! When upgrading a `WeakHandle`, we acquire a strong ref to the `Arc`. +//! Again, this manipulation of ref counts is is cheap because `Arc` is private to the connection. +//! +//! When downgrading a `Handle` to a `WeakHandle`, we drop the `Arc`. +//! Again, this is cheap because the `Arc` is private to the connection. +//! +//! In addition to the GateGuard, we need to provide `impl Deref for Handle` and `impl Deref for WeakHandle`. +//! For this, both `Handle` and `WeakHandle` need access to the `Arc`. +//! Upgrading a `WeakHandle` does not consume the `Handle`, so we can't move the `Arc` out of +//! `Handle` into `WeakHandle`. +//! We could clone the `Arc` when upgrading a `WeakHandle`, but that would cause contention +//! on the shared memory location that trakcs the refcount of the `Arc`. +//! Instead, we wrap the `Arc` into another `Arc`. +//! so that we can clone it cheaply when upgrading a `WeakHandle`. +//! +//! # Shutdown +//! +//! The attentive reader may have noticed the following reference cycle around the `Arc`: +//! +//! ``` +//! Timeline --owns--> PerTimelineState --strong--> HandleInner --strong--> WeakHandle --strong--> Timeline +//! ``` +//! +//! Further, there is this cycle: +//! +//! ``` +//! Timeline --owns--> PerTimelineState --strong--> HandleInner --strong--> GateGuard --keepalive--> Timeline +//! ``` +//! +//! The former cycle is a memory leak if not broken. +//! The latter cycle further prevent the Timeline from shutting down. +//! Preventing shutdown is the whole point of this handle/cache system, +//! but when the Timeline needs to shut down, we need to break the cycle. //! //! The cycle is broken by either -//! - `PerTimelineState::shutdown` or -//! - dropping the `Cache`. +//! - Timeline shutdown (=> `PerTimelineState::shutdown`) +//! - Connection shutdown (=> dropping the `Cache`). //! -//! Concurrently existing `Handle`s will extend the existence of the cycle. +//! Both transition the `HandleInner` from [`HandleInner::KeepingTimelineGateOpen`] to +//! [`HandleInner::ShutDown`], which drops the only long-lived strong ref to the +//! `Arc`. +//! +//! +//! +//! `PerTimelineState::shutdown` drops all the `HandleInners` it contains, +//! thereby breaking the cycle. +//! It also initiates draining of already existing `Handle`s, by +//! poisoning things so that no new `HandleInner`'s can be tracked +//! in the shut down `PerTimelineState`. +//! +//! Concurrently existing / already upgraded `Handle`s will extend the +//! existence of the cycle. //! However, since `Handle`s are short-lived and new `Handle`s are not //! handed out after either `PerTimelineState::shutdown` or `Cache` drop, //! that extension of the cycle is bounded. //! +//! Concurrently existing `WeakHandle`s will fail to `upgrade()`: +//! while they will succeed in upgradingtheir weak arc ref to a strong ref, +//! they will find the inner in state `HandleInner::ShutDown` state where the +//! `Arc` has already been dropped. +//! +//! Dropping the `Cache` undoes the registration of this `Cache`'s +//! `HandleInner`s from all the `PerTimelineState`s, i.e., it +//! removes the strong ref to each of its `HandleInner`s +//! from all the `PerTimelineState`. +//! //! # Fast Path for Shard Routing //! //! The `Cache` has a fast path for shard routing to avoid calling into //! the tenant manager for every request. //! -//! The `Cache` maintains a hash map of `ShardTimelineId` to `Weak`. +//! The `Cache` maintains a hash map of `ShardTimelineId` to `WeakHandle`s. //! //! The current implementation uses the first entry in the hash map //! to determine the `ShardParameters` and derive the correct @@ -87,18 +174,18 @@ //! //! It then looks up the hash map for that `ShardTimelineId := {ShardIndex,TimelineId}`. //! -//! If the lookup is successful and the `Weak` can be upgraded, +//! If the lookup is successful and the `WeakHandle` can be upgraded, //! it's a hit. //! //! ## Cache invalidation //! -//! The insight is that cache invalidation is sufficient and most efficiently done lazily. +//! The insight is that cache invalidation is sufficient and most efficiently if done lazily. //! The only reasons why an entry in the cache can become stale are: //! 1. The `PerTimelineState` / Timeline is shutting down e.g. because the shard is //! being detached, timeline or shard deleted, or pageserver is shutting down. //! 2. We're doing a shard split and new traffic should be routed to the child shards. //! -//! Regarding (1), we will eventually fail to upgrade the `Weak` once the +//! Regarding (1), we will eventually fail to upgrade the `WeakHandle` once the //! timeline has shut down, and when that happens, we remove the entry from the cache. //! //! Regarding (2), the insight is that it is toally fine to keep dispatching requests @@ -439,6 +526,22 @@ impl WeakHandle { } } +impl std::ops::Deref for Handle { + type Target = T::Timeline; + fn deref(&self) -> &Self::Target { + &self.timeline + } +} + +impl Handle { + pub(crate) fn downgrade(&self) -> WeakHandle { + WeakHandle { + timeline: Arc::clone(&self.timeline), + inner: Arc::downgrade(&self.inner), + } + } +} + impl PerTimelineState { /// After this method returns, [`Cache::get`] will never again return a [`Handle`] /// to the [`Types::Timeline`] that embeds this per-timeline state. @@ -463,33 +566,12 @@ impl PerTimelineState { for handle_inner_arc in handles.values() { // Make hits fail. let mut lock_guard = handle_inner_arc.lock().expect("poisoned"); - match &mut *lock_guard { - HandleInner::KeepingTimelineGateOpen { .. } => *lock_guard = HandleInner::ShutDown, - HandleInner::ShutDown => { - unreachable!("handles are only shut down once in their lifetime"); - } - } + lock_guard.shutdown(); } drop(handles); } } -impl std::ops::Deref for Handle { - type Target = T::Timeline; - fn deref(&self) -> &Self::Target { - &self.timeline - } -} - -impl Handle { - pub(crate) fn downgrade(&self) -> WeakHandle { - WeakHandle { - timeline: Arc::clone(&self.timeline), - inner: Arc::downgrade(&self.inner), - } - } -} - // When dropping a [`Cache`], prune its handles in the [`PerTimelineState`] to break the reference cycle. impl Drop for Cache { fn drop(&mut self) { @@ -507,11 +589,27 @@ impl Drop for Cache { let Some(handles) = &mut *handles else { continue; }; - let Some(removed) = handles.remove(&self.id) else { + let Some(removed_handle_inner_arc) = handles.remove(&self.id) else { // There could have been a shutdown inbetween us upgrading the weak and locking the mutex. continue; }; - assert!(Weak::ptr_eq(&Arc::downgrade(&removed), &handle_inner_weak)); + assert!(Weak::ptr_eq( + &Arc::downgrade(&removed_handle_inner_arc), + &handle_inner_weak + )); + let mut lock_guard = removed_handle_inner_arc.lock().expect("poisoned"); + lock_guard.shutdown(); + } + } +} + +impl HandleInner { + fn shutdown(&mut self) { + match self { + HandleInner::KeepingTimelineGateOpen { .. } => *self = HandleInner::ShutDown, + HandleInner::ShutDown => { + unreachable!("handles are only shut down once in their lifetime"); + } } } } From a4fc6a92c992183f18e3eea8e1c02df6ef337764 Mon Sep 17 00:00:00 2001 From: Christian Schwarz Date: Wed, 15 Jan 2025 16:10:04 +0100 Subject: [PATCH 110/111] fix cargo doc --- pageserver/src/tenant/timeline/handle.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pageserver/src/tenant/timeline/handle.rs b/pageserver/src/tenant/timeline/handle.rs index 59f399d01a..b8bc2ba2aa 100644 --- a/pageserver/src/tenant/timeline/handle.rs +++ b/pageserver/src/tenant/timeline/handle.rs @@ -114,13 +114,13 @@ //! //! The attentive reader may have noticed the following reference cycle around the `Arc`: //! -//! ``` +//! ```text //! Timeline --owns--> PerTimelineState --strong--> HandleInner --strong--> WeakHandle --strong--> Timeline //! ``` //! //! Further, there is this cycle: //! -//! ``` +//! ```text //! Timeline --owns--> PerTimelineState --strong--> HandleInner --strong--> GateGuard --keepalive--> Timeline //! ``` //! From c545d227b94edd834f6b523c9b6619385f9ffa70 Mon Sep 17 00:00:00 2001 From: Christian Schwarz Date: Wed, 15 Jan 2025 16:24:39 +0100 Subject: [PATCH 111/111] review doc comment --- pageserver/src/tenant/timeline/handle.rs | 41 +++++++++++++----------- 1 file changed, 22 insertions(+), 19 deletions(-) diff --git a/pageserver/src/tenant/timeline/handle.rs b/pageserver/src/tenant/timeline/handle.rs index b8bc2ba2aa..d57776a2fe 100644 --- a/pageserver/src/tenant/timeline/handle.rs +++ b/pageserver/src/tenant/timeline/handle.rs @@ -46,19 +46,21 @@ //! //! The `HandleInner` is allocated as a `Arc>` and //! referenced weakly and strongly from various places which we are now illustrating. -//! For bevity, we will omit the `Arc>` part in the following. +//! For brevity, we will omit the `Arc>` part in the following and instead +//! use `strong ref` and `weak ref` when referring to the `Arc>` +//! or `Weak>`, respectively. //! //! - The `Handle` is a strong ref. //! - The `WeakHandle` is a weak ref. -//! - The only long-lived strong reference is stored in the `PerTimelineState`. -//! - The `Cache` stores a weak ref. +//! - The `PerTimelineState` contains a `HashMap`. +//! - The `Cache` is a `HashMap`. //! //! Lifetimes: //! - `WeakHandle` and `Handle`: single pagestream request. //! - `Cache`: single page service connection. //! - `PerTimelineState`: lifetime of the Timeline object (i.e., i.e., till `Timeline::shutdown`). //! -//! ## Request Handling Flow +//! ## Request Handling Flow (= filling and using the `Cache``) //! //! To dispatch a request, the page service connection calls `Cache::get`. //! @@ -86,7 +88,7 @@ //! > method invocation. //! //! Why do we want to avoid that? -//! Because the gate is a shared location in memory and entering it invovles +//! Because the gate is a shared location in memory and entering it involves //! bumping refcounts, which leads to cache contention if done frequently //! from multiple cores in parallel. //! @@ -94,17 +96,19 @@ //! That `Arc` is private to the `HandleInner` and hence to the connection. //! (Review the "Data Structures" section if that is unclear to you.) //! -//! A `WeakHandle` is a weak ref to the HandleInner. -//! When upgrading a `WeakHandle`, we acquire a strong ref to the `Arc`. +//! A `WeakHandle` is a weak ref to the `HandleInner`. +//! When upgrading a `WeakHandle`, we upgrade to a strong ref to the `HandleInner` and +//! further acquire an additional strong ref to the `Arc` inside it. //! Again, this manipulation of ref counts is is cheap because `Arc` is private to the connection. //! //! When downgrading a `Handle` to a `WeakHandle`, we drop the `Arc`. //! Again, this is cheap because the `Arc` is private to the connection. //! -//! In addition to the GateGuard, we need to provide `impl Deref for Handle` and `impl Deref for WeakHandle`. +//! In addition to the GateGuard, we need to provide `Deref` impls +//! for `Handle` or `WeakHandle`. //! For this, both `Handle` and `WeakHandle` need access to the `Arc`. -//! Upgrading a `WeakHandle` does not consume the `Handle`, so we can't move the `Arc` out of -//! `Handle` into `WeakHandle`. +//! Upgrading a `WeakHandle` does not consume the `Handle`, so we can't move the +//! `Arc` out of `Handle` into `WeakHandle`. //! We could clone the `Arc` when upgrading a `WeakHandle`, but that would cause contention //! on the shared memory location that trakcs the refcount of the `Arc`. //! Instead, we wrap the `Arc` into another `Arc`. @@ -125,7 +129,8 @@ //! ``` //! //! The former cycle is a memory leak if not broken. -//! The latter cycle further prevent the Timeline from shutting down. +//! The latter cycle further prevents the Timeline from shutting down +//! because we certainly won't drop the Timeline while the GateGuard is alive. //! Preventing shutdown is the whole point of this handle/cache system, //! but when the Timeline needs to shut down, we need to break the cycle. //! @@ -137,19 +142,17 @@ //! [`HandleInner::ShutDown`], which drops the only long-lived strong ref to the //! `Arc`. //! -//! -//! //! `PerTimelineState::shutdown` drops all the `HandleInners` it contains, //! thereby breaking the cycle. -//! It also initiates draining of already existing `Handle`s, by -//! poisoning things so that no new `HandleInner`'s can be tracked -//! in the shut down `PerTimelineState`. +//! It also initiates draining of already existing `Handle`s by +//! poisoning things so that no new `HandleInner`'s can be added +//! to the `PerTimelineState`, which will make subsequent `Cache::get` fail. //! //! Concurrently existing / already upgraded `Handle`s will extend the -//! existence of the cycle. +//! lifetime of the `Arc>` and hence cycles. //! However, since `Handle`s are short-lived and new `Handle`s are not -//! handed out after either `PerTimelineState::shutdown` or `Cache` drop, -//! that extension of the cycle is bounded. +//! handed out from `Cache::get` or `WeakHandle::upgrade` after +//! `PerTimelineState::shutdown`, that extension of the cycle is bounded. //! //! Concurrently existing `WeakHandle`s will fail to `upgrade()`: //! while they will succeed in upgradingtheir weak arc ref to a strong ref,