diff --git a/.github/actions/run-python-test-set/action.yml b/.github/actions/run-python-test-set/action.yml index 60ccc56738..013b446307 100644 --- a/.github/actions/run-python-test-set/action.yml +++ b/.github/actions/run-python-test-set/action.yml @@ -145,7 +145,11 @@ runs: if [ "${RERUN_FLAKY}" == "true" ]; then mkdir -p $TEST_OUTPUT - poetry run ./scripts/flaky_tests.py "${TEST_RESULT_CONNSTR}" --days 10 --output "$TEST_OUTPUT/flaky.json" + poetry run ./scripts/flaky_tests.py "${TEST_RESULT_CONNSTR}" \ + --days 7 \ + --output "$TEST_OUTPUT/flaky.json" \ + --pg-version "${DEFAULT_PG_VERSION}" \ + --build-type "${BUILD_TYPE}" EXTRA_PARAMS="--flaky-tests-json $TEST_OUTPUT/flaky.json $EXTRA_PARAMS" fi diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index f44e0e6c05..1ec2a65a89 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -737,34 +737,6 @@ jobs: --destination neondatabase/compute-node-${{ matrix.version }}:${{needs.tag.outputs.build-tag}} --cleanup - # Due to a kaniko bug, we can't use cache for extensions image, thus it takes about the same amount of time as compute-node image to build (~10 min) - # During the transition period we need to have extensions in both places (in S3 and in compute-node image), - # so we won't build extension twice, but extract them from compute-node. - # - # For now we use extensions image only for new custom extensitons - - name: Kaniko build extensions only - run: | - # Kaniko is suposed to clean up after itself if --cleanup flag is set, but it doesn't. - # Despite some fixes were made in https://github.com/GoogleContainerTools/kaniko/pull/2504 (in kaniko v1.11.0), - # it still fails with error: - # error building image: could not save file: copying file: symlink postgres /kaniko/1/usr/local/pgsql/bin/postmaster: file exists - # - # Ref https://github.com/GoogleContainerTools/kaniko/issues/1406 - find /kaniko -maxdepth 1 -mindepth 1 -type d -regex "/kaniko/[0-9]*" -exec rm -rv {} \; - - /kaniko/executor --reproducible --snapshot-mode=redo --skip-unused-stages --cache=true \ - --cache-repo 369495373322.dkr.ecr.eu-central-1.amazonaws.com/cache \ - --context . \ - --build-arg GIT_VERSION=${{ github.event.pull_request.head.sha || github.sha }} \ - --build-arg PG_VERSION=${{ matrix.version }} \ - --build-arg BUILD_TAG=${{needs.tag.outputs.build-tag}} \ - --build-arg REPOSITORY=369495373322.dkr.ecr.eu-central-1.amazonaws.com \ - --dockerfile Dockerfile.compute-node \ - --destination 369495373322.dkr.ecr.eu-central-1.amazonaws.com/extensions-${{ matrix.version }}:${{needs.tag.outputs.build-tag}} \ - --destination neondatabase/extensions-${{ matrix.version }}:${{needs.tag.outputs.build-tag}} \ - --cleanup \ - --target postgres-extensions - # Cleanup script fails otherwise - rm: cannot remove '/nvme/actions-runner/_work/_temp/_github_home/.ecr': Permission denied - name: Cleanup ECR folder run: rm -rf ~/.ecr @@ -780,7 +752,7 @@ jobs: run: shell: sh -eu {0} env: - VM_BUILDER_VERSION: v0.16.3 + VM_BUILDER_VERSION: v0.17.5 steps: - name: Checkout @@ -803,7 +775,7 @@ jobs: run: | ./vm-builder \ -enable-file-cache \ - -enable-monitor \ + -cgroup-uid=postgres \ -src=369495373322.dkr.ecr.eu-central-1.amazonaws.com/compute-node-${{ matrix.version }}:${{needs.tag.outputs.build-tag}} \ -dst=369495373322.dkr.ecr.eu-central-1.amazonaws.com/vm-compute-node-${{ matrix.version }}:${{needs.tag.outputs.build-tag}} @@ -886,10 +858,8 @@ jobs: crane tag 369495373322.dkr.ecr.eu-central-1.amazonaws.com/compute-tools:${{needs.tag.outputs.build-tag}} latest crane tag 369495373322.dkr.ecr.eu-central-1.amazonaws.com/compute-node-v14:${{needs.tag.outputs.build-tag}} latest crane tag 369495373322.dkr.ecr.eu-central-1.amazonaws.com/vm-compute-node-v14:${{needs.tag.outputs.build-tag}} latest - crane tag 369495373322.dkr.ecr.eu-central-1.amazonaws.com/extensions-v14:${{needs.tag.outputs.build-tag}} latest crane tag 369495373322.dkr.ecr.eu-central-1.amazonaws.com/compute-node-v15:${{needs.tag.outputs.build-tag}} latest crane tag 369495373322.dkr.ecr.eu-central-1.amazonaws.com/vm-compute-node-v15:${{needs.tag.outputs.build-tag}} latest - crane tag 369495373322.dkr.ecr.eu-central-1.amazonaws.com/extensions-v15:${{needs.tag.outputs.build-tag}} latest - name: Push images to production ECR if: | @@ -900,10 +870,8 @@ jobs: crane copy 369495373322.dkr.ecr.eu-central-1.amazonaws.com/compute-tools:${{needs.tag.outputs.build-tag}} 093970136003.dkr.ecr.eu-central-1.amazonaws.com/compute-tools:latest crane copy 369495373322.dkr.ecr.eu-central-1.amazonaws.com/compute-node-v14:${{needs.tag.outputs.build-tag}} 093970136003.dkr.ecr.eu-central-1.amazonaws.com/compute-node-v14:latest crane copy 369495373322.dkr.ecr.eu-central-1.amazonaws.com/vm-compute-node-v14:${{needs.tag.outputs.build-tag}} 093970136003.dkr.ecr.eu-central-1.amazonaws.com/vm-compute-node-v14:latest - crane copy 369495373322.dkr.ecr.eu-central-1.amazonaws.com/extensions-v14:${{needs.tag.outputs.build-tag}} 093970136003.dkr.ecr.eu-central-1.amazonaws.com/extensions-v14:latest crane copy 369495373322.dkr.ecr.eu-central-1.amazonaws.com/compute-node-v15:${{needs.tag.outputs.build-tag}} 093970136003.dkr.ecr.eu-central-1.amazonaws.com/compute-node-v15:latest crane copy 369495373322.dkr.ecr.eu-central-1.amazonaws.com/vm-compute-node-v15:${{needs.tag.outputs.build-tag}} 093970136003.dkr.ecr.eu-central-1.amazonaws.com/vm-compute-node-v15:latest - crane copy 369495373322.dkr.ecr.eu-central-1.amazonaws.com/extensions-v15:${{needs.tag.outputs.build-tag}} 093970136003.dkr.ecr.eu-central-1.amazonaws.com/extensions-v15:latest - name: Configure Docker Hub login run: | @@ -925,65 +893,56 @@ jobs: crane tag neondatabase/compute-tools:${{needs.tag.outputs.build-tag}} latest crane tag neondatabase/compute-node-v14:${{needs.tag.outputs.build-tag}} latest crane tag neondatabase/vm-compute-node-v14:${{needs.tag.outputs.build-tag}} latest - crane tag neondatabase/extensions-v14:${{needs.tag.outputs.build-tag}} latest crane tag neondatabase/compute-node-v15:${{needs.tag.outputs.build-tag}} latest crane tag neondatabase/vm-compute-node-v15:${{needs.tag.outputs.build-tag}} latest - crane tag neondatabase/extensions-v15:${{needs.tag.outputs.build-tag}} latest - name: Cleanup ECR folder run: rm -rf ~/.ecr - upload-postgres-extensions-to-s3: - if: | - (github.ref_name == 'main' || github.ref_name == 'release') && - github.event_name != 'workflow_dispatch' - runs-on: ${{ github.ref_name == 'release' && fromJSON('["self-hosted", "prod", "x64"]') || fromJSON('["self-hosted", "gen3", "small"]') }} - needs: [ tag, promote-images ] - strategy: - fail-fast: false - matrix: - version: [ v14, v15 ] - - env: - EXTENSIONS_IMAGE: ${{ github.ref_name == 'release' && '093970136003' || '369495373322'}}.dkr.ecr.eu-central-1.amazonaws.com/extensions-${{ matrix.version }}:${{ needs.tag.outputs.build-tag }} - AWS_ACCESS_KEY_ID: ${{ github.ref_name == 'release' && secrets.AWS_ACCESS_KEY_PROD || secrets.AWS_ACCESS_KEY_DEV }} - AWS_SECRET_ACCESS_KEY: ${{ github.ref_name == 'release' && secrets.AWS_SECRET_KEY_PROD || secrets.AWS_SECRET_KEY_DEV }} - S3_BUCKETS: ${{ github.ref_name == 'release' && vars.S3_EXTENSIONS_BUCKETS_PROD || vars.S3_EXTENSIONS_BUCKETS_DEV }} - + build-private-extensions: + runs-on: [ self-hosted, gen3, small ] + container: + image: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/base:pinned + options: --init + needs: [ tag ] steps: - - name: Pull postgres-extensions image + - name: Set PR's status to pending and request a remote CI test run: | - docker pull ${EXTENSIONS_IMAGE} + COMMIT_SHA=${{ github.event.pull_request.head.sha }} + COMMIT_SHA=${COMMIT_SHA:-${{ github.sha }}} + REMOTE_REPO="${{ github.repository_owner }}/build-custom-extensions" - - name: Create postgres-extensions container - id: create-container - run: | - EID=$(docker create ${EXTENSIONS_IMAGE} true) - echo "EID=${EID}" >> $GITHUB_OUTPUT + curl -f -X POST \ + https://api.github.com/repos/${{ github.repository }}/statuses/$COMMIT_SHA \ + -H "Accept: application/vnd.github.v3+json" \ + --user "${{ secrets.CI_ACCESS_TOKEN }}" \ + --data \ + "{ + \"state\": \"pending\", + \"context\": \"build-and-upload-extensions\", + \"description\": \"[$REMOTE_REPO] Remote CI job is about to start\" + }" - - name: Extract postgres-extensions from container - run: | - rm -rf ./extensions-to-upload # Just in case - mkdir -p extensions-to-upload - - docker cp ${{ steps.create-container.outputs.EID }}:/extensions/ ./extensions-to-upload/ - docker cp ${{ steps.create-container.outputs.EID }}:/ext_index.json ./extensions-to-upload/ - - - name: Upload postgres-extensions to S3 - run: | - for BUCKET in $(echo ${S3_BUCKETS:-[]} | jq --raw-output '.[]'); do - aws s3 cp --recursive --only-show-errors ./extensions-to-upload s3://${BUCKET}/${{ needs.tag.outputs.build-tag }}/${{ matrix.version }} - done - - - name: Cleanup - if: ${{ always() && steps.create-container.outputs.EID }} - run: | - docker rm ${{ steps.create-container.outputs.EID }} || true + curl -f -X POST \ + https://api.github.com/repos/$REMOTE_REPO/actions/workflows/build_and_upload_extensions.yml/dispatches \ + -H "Accept: application/vnd.github.v3+json" \ + --user "${{ secrets.CI_ACCESS_TOKEN }}" \ + --data \ + "{ + \"ref\": \"main\", + \"inputs\": { + \"ci_job_name\": \"build-and-upload-extensions\", + \"commit_hash\": \"$COMMIT_SHA\", + \"remote_repo\": \"${{ github.repository }}\", + \"compute_image_tag\": \"${{ needs.tag.outputs.build-tag }}\", + \"remote_branch_name\": \"${{ github.ref_name }}\" + } + }" deploy: runs-on: [ self-hosted, gen3, small ] container: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/ansible:latest - needs: [ upload-postgres-extensions-to-s3, promote-images, tag, regress-tests ] + needs: [ promote-images, tag, regress-tests ] if: ( github.ref_name == 'main' || github.ref_name == 'release' ) && github.event_name != 'workflow_dispatch' steps: - name: Fix git ownership diff --git a/CODEOWNERS b/CODEOWNERS index 6b1273520d..b8ca54bc7e 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -1,11 +1,12 @@ -/compute_tools/ @neondatabase/control-plane +/compute_tools/ @neondatabase/control-plane @neondatabase/compute /control_plane/ @neondatabase/compute @neondatabase/storage /libs/pageserver_api/ @neondatabase/compute @neondatabase/storage -/libs/postgres_ffi/ @neondatabase/compute -/libs/remote_storage/ @neondatabase/storage -/libs/safekeeper_api/ @neondatabase/safekeepers -/pageserver/ @neondatabase/compute @neondatabase/storage +/libs/postgres_ffi/ @neondatabase/compute +/libs/remote_storage/ @neondatabase/storage +/libs/safekeeper_api/ @neondatabase/safekeepers +/libs/vm_monitor/ @neondatabase/autoscaling @neondatabase/compute +/pageserver/ @neondatabase/compute @neondatabase/storage /pgxn/ @neondatabase/compute -/proxy/ @neondatabase/control-plane +/proxy/ @neondatabase/proxy /safekeeper/ @neondatabase/safekeepers /vendor/ @neondatabase/compute diff --git a/Cargo.lock b/Cargo.lock index 23741e2a3d..c213cd4d0a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -190,7 +190,7 @@ checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193" dependencies = [ "proc-macro2", "quote", - "syn 2.0.16", + "syn 2.0.28", ] [[package]] @@ -201,7 +201,7 @@ checksum = "b9ccdd8f2a161be9bd5c023df56f1b2a0bd1d83872ae53b71a84a12c9bf6e842" dependencies = [ "proc-macro2", "quote", - "syn 2.0.16", + "syn 2.0.28", ] [[package]] @@ -553,12 +553,13 @@ dependencies = [ [[package]] name = "axum" -version = "0.6.18" +version = "0.6.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8175979259124331c1d7bf6586ee7e0da434155e4b2d48ec2c8386281d8df39" +checksum = "3b829e4e32b91e643de6eafe82b1d90675f5874230191a4ffbc1b336dec4d6bf" dependencies = [ "async-trait", "axum-core", + "base64 0.21.1", "bitflags", "bytes", "futures-util", @@ -573,7 +574,13 @@ dependencies = [ "pin-project-lite", "rustversion", "serde", + "serde_json", + "serde_path_to_error", + "serde_urlencoded", + "sha1", "sync_wrapper", + "tokio", + "tokio-tungstenite 0.20.0", "tower", "tower-layer", "tower-service", @@ -673,7 +680,7 @@ dependencies = [ "regex", "rustc-hash", "shlex", - "syn 2.0.16", + "syn 2.0.28", "which", ] @@ -765,6 +772,19 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" +[[package]] +name = "cgroups-rs" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fb3af90c8d48ad5f432d8afb521b5b40c2a2fce46dd60e05912de51c47fba64" +dependencies = [ + "libc", + "log", + "nix 0.25.1", + "regex", + "thiserror", +] + [[package]] name = "chrono" version = "0.4.24" @@ -849,7 +869,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.16", + "syn 2.0.28", ] [[package]] @@ -907,6 +927,7 @@ version = "0.1.0" dependencies = [ "anyhow", "async-compression", + "cfg-if", "chrono", "clap", "compute_api", @@ -925,6 +946,7 @@ dependencies = [ "tar", "tokio", "tokio-postgres", + "tokio-util", "toml_edit", "tracing", "tracing-opentelemetry", @@ -932,6 +954,7 @@ dependencies = [ "tracing-utils", "url", "utils", + "vm_monitor", "workspace_hack", "zstd", ] @@ -978,7 +1001,7 @@ dependencies = [ "comfy-table", "compute_api", "git-version", - "nix", + "nix 0.26.2", "once_cell", "pageserver_api", "postgres", @@ -1184,7 +1207,7 @@ dependencies = [ "proc-macro2", "quote", "strsim", - "syn 2.0.16", + "syn 2.0.28", ] [[package]] @@ -1195,7 +1218,7 @@ checksum = "29a358ff9f12ec09c3e61fef9b5a9902623a695a46a917b07f269bff1445611a" dependencies = [ "darling_core", "quote", - "syn 2.0.16", + "syn 2.0.28", ] [[package]] @@ -1260,7 +1283,7 @@ checksum = "487585f4d0c6655fe74905e2504d8ad6908e4db67f744eb140876906c2f3175d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.16", + "syn 2.0.28", ] [[package]] @@ -1316,7 +1339,7 @@ dependencies = [ "darling", "proc-macro2", "quote", - "syn 2.0.16", + "syn 2.0.28", ] [[package]] @@ -1512,7 +1535,7 @@ checksum = "89ca545a94061b6365f2c7355b4b32bd20df3ff95f02da9329b34ccc3bd6ee72" dependencies = [ "proc-macro2", "quote", - "syn 2.0.16", + "syn 2.0.28", ] [[package]] @@ -1863,8 +1886,8 @@ dependencies = [ "hyper", "pin-project", "tokio", - "tokio-tungstenite", - "tungstenite", + "tokio-tungstenite 0.18.0", + "tungstenite 0.18.0", ] [[package]] @@ -1928,6 +1951,19 @@ dependencies = [ "libc", ] +[[package]] +name = "inotify" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fdd168d97690d0b8c412d6b6c10360277f4d7ee495c5d0d5d5fe0854923255cc" +dependencies = [ + "bitflags", + "futures-core", + "inotify-sys", + "libc", + "tokio", +] + [[package]] name = "inotify-sys" version = "0.1.5" @@ -2251,6 +2287,18 @@ dependencies = [ "tempfile", ] +[[package]] +name = "nix" +version = "0.25.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f346ff70e7dbfd675fe90590b92d59ef2de15a8779ae305ebcbfd3f0caf59be4" +dependencies = [ + "autocfg", + "bitflags", + "cfg-if", + "libc", +] + [[package]] name = "nix" version = "0.26.2" @@ -2285,7 +2333,7 @@ dependencies = [ "crossbeam-channel", "filetime", "fsevent-sys", - "inotify", + "inotify 0.9.6", "kqueue", "libc", "mio", @@ -2293,6 +2341,15 @@ dependencies = [ "windows-sys 0.45.0", ] +[[package]] +name = "ntapi" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8a3895c6391c39d7fe7ebc444a87eb2991b2a0bc718fdabd071eec617fc68e4" +dependencies = [ + "winapi", +] + [[package]] name = "num-bigint" version = "0.4.3" @@ -2386,7 +2443,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.16", + "syn 2.0.28", ] [[package]] @@ -2573,7 +2630,7 @@ dependencies = [ "hyper", "itertools", "metrics", - "nix", + "nix 0.26.2", "num-traits", "num_cpus", "once_cell", @@ -2596,6 +2653,7 @@ dependencies = [ "serde_json", "serde_with", "signal-hook", + "smallvec", "storage_broker", "strum", "strum_macros", @@ -2773,7 +2831,7 @@ checksum = "39407670928234ebc5e6e580247dd567ad73a3578460c5990f9503df207e8f07" dependencies = [ "proc-macro2", "quote", - "syn 2.0.16", + "syn 2.0.28", ] [[package]] @@ -2970,7 +3028,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3b69d39aab54d069e7f2fe8cb970493e7834601ca2d8c65fd7bbd183578080d1" dependencies = [ "proc-macro2", - "syn 2.0.16", + "syn 2.0.28", ] [[package]] @@ -2981,9 +3039,9 @@ checksum = "dc375e1527247fe1a97d8b7156678dfe7c1af2fc075c9a4db3690ecd2a148068" [[package]] name = "proc-macro2" -version = "1.0.64" +version = "1.0.66" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78803b62cbf1f46fde80d7c0e803111524b9877184cfe7c3033659490ac7a7da" +checksum = "18fb31db3f9bddb2ea821cde30a9f70117e3f119938b5ee630b7403aa6e2ead9" dependencies = [ "unicode-ident", ] @@ -3145,9 +3203,9 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.27" +version = "1.0.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f4f29d145265ec1c483c7c654450edde0bfe043d3938d6972630663356d9500" +checksum = "50f3b39ccfb720540debaa0164757101c08ecb8d326b15358ce76a62c7e85965" dependencies = [ "proc-macro2", ] @@ -3569,9 +3627,9 @@ dependencies = [ [[package]] name = "rustls-webpki" -version = "0.100.1" +version = "0.100.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6207cd5ed3d8dca7816f8f3725513a34609c0c765bf652b8c3cb4cfd87db46b" +checksum = "e98ff011474fa39949b7e5c0428f9b4937eda7da7848bbb947786b7be0b27dab" dependencies = [ "ring", "untrusted", @@ -3798,22 +3856,22 @@ dependencies = [ [[package]] name = "serde" -version = "1.0.163" +version = "1.0.183" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2113ab51b87a539ae008b5c6c02dc020ffa39afd2d83cffcb3f4eb2722cebec2" +checksum = "32ac8da02677876d532745a130fc9d8e6edfa81a269b107c5b00829b91d8eb3c" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.163" +version = "1.0.183" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c805777e3930c8883389c602315a24224bcc738b63905ef87cd1420353ea93e" +checksum = "aafe972d60b0b9bee71a91b92fee2d4fb3c9d7e8f6b179aa99f27203d99a4816" dependencies = [ "proc-macro2", "quote", - "syn 2.0.16", + "syn 2.0.28", ] [[package]] @@ -3827,6 +3885,16 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_path_to_error" +version = "0.1.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4beec8bce849d58d06238cb50db2e1c417cfeafa4c63f692b15c82b7c80f8335" +dependencies = [ + "itoa", + "serde", +] + [[package]] name = "serde_spanned" version = "0.6.2" @@ -3873,7 +3941,7 @@ dependencies = [ "darling", "proc-macro2", "quote", - "syn 2.0.16", + "syn 2.0.28", ] [[package]] @@ -3972,9 +4040,9 @@ dependencies = [ [[package]] name = "smallvec" -version = "1.10.0" +version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a507befe795404456341dfab10cef66ead4c041f62b8b11bbb92bffe5d0953e0" +checksum = "62bb4feee49fdd9f707ef802e22365a35de4b7b299de4763d44bfea899442ff9" [[package]] name = "socket2" @@ -4111,9 +4179,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.16" +version = "2.0.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6f671d4b5ffdb8eadec19c0ae67fe2639df8684bd7bc4b83d986b8db549cf01" +checksum = "04361975b3f5e348b2189d8dc55bc942f278b2d482a6a0365de5bdd62d351567" dependencies = [ "proc-macro2", "quote", @@ -4139,14 +4207,29 @@ dependencies = [ ] [[package]] -name = "tar" -version = "0.4.38" +name = "sysinfo" +version = "0.29.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b55807c0344e1e6c04d7c965f5289c39a8d94ae23ed5c0b57aabac549f871c6" +checksum = "165d6d8539689e3d3bc8b98ac59541e1f21c7de7c85d60dc80e43ae0ed2113db" +dependencies = [ + "cfg-if", + "core-foundation-sys", + "libc", + "ntapi", + "once_cell", + "rayon", + "winapi", +] + +[[package]] +name = "tar" +version = "0.4.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b16afcea1f22891c49a00c751c7b63b2233284064f11a200fc624137c51e2ddb" dependencies = [ "filetime", "libc", - "xattr 0.2.3", + "xattr", ] [[package]] @@ -4228,7 +4311,7 @@ checksum = "f9456a42c5b0d803c8cd86e73dd7cc9edd429499f37a3550d286d5e86720569f" dependencies = [ "proc-macro2", "quote", - "syn 2.0.16", + "syn 2.0.28", ] [[package]] @@ -4343,7 +4426,7 @@ checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.16", + "syn 2.0.28", ] [[package]] @@ -4437,7 +4520,7 @@ dependencies = [ "redox_syscall 0.3.5", "tokio", "tokio-stream", - "xattr 1.0.0", + "xattr", ] [[package]] @@ -4449,7 +4532,19 @@ dependencies = [ "futures-util", "log", "tokio", - "tungstenite", + "tungstenite 0.18.0", +] + +[[package]] +name = "tokio-tungstenite" +version = "0.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b2dbec703c26b00d74844519606ef15d09a7d6857860f84ad223dec002ddea2" +dependencies = [ + "futures-util", + "log", + "tokio", + "tungstenite 0.20.0", ] [[package]] @@ -4641,7 +4736,7 @@ checksum = "0f57e3ca2a01450b1a921183a9c9cbfda207fd822cef4ccb00a65402cbba7a74" dependencies = [ "proc-macro2", "quote", - "syn 2.0.16", + "syn 2.0.28", ] [[package]] @@ -4770,6 +4865,25 @@ dependencies = [ "utf-8", ] +[[package]] +name = "tungstenite" +version = "0.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e862a1c4128df0112ab625f55cd5c934bcb4312ba80b39ae4b4835a3fd58e649" +dependencies = [ + "byteorder", + "bytes", + "data-encoding", + "http", + "httparse", + "log", + "rand", + "sha1", + "thiserror", + "url", + "utf-8", +] + [[package]] name = "typenum" version = "1.16.0" @@ -4897,9 +5011,10 @@ dependencies = [ "hyper", "jsonwebtoken", "metrics", - "nix", + "nix 0.26.2", "once_cell", "pin-project-lite", + "postgres_connection", "pq_proto", "rand", "regex", @@ -4915,6 +5030,7 @@ dependencies = [ "thiserror", "tokio", "tokio-stream", + "tokio-util", "tracing", "tracing-error", "tracing-subscriber", @@ -4951,6 +5067,28 @@ version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" +[[package]] +name = "vm_monitor" +version = "0.1.0" +dependencies = [ + "anyhow", + "axum", + "cgroups-rs", + "clap", + "futures", + "inotify 0.10.2", + "serde", + "serde_json", + "sysinfo", + "tokio", + "tokio-postgres", + "tokio-stream", + "tokio-util", + "tracing", + "tracing-subscriber", + "workspace_hack", +] + [[package]] name = "vsimd" version = "0.8.0" @@ -5021,7 +5159,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.16", + "syn 2.0.28", "wasm-bindgen-shared", ] @@ -5055,7 +5193,7 @@ checksum = "e128beba882dd1eb6200e1dc92ae6c5dbaa4311aa7bb211ca035779e5efc39f8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.16", + "syn 2.0.28", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -5340,12 +5478,14 @@ name = "workspace_hack" version = "0.1.0" dependencies = [ "anyhow", + "axum", "bytes", "cc", "chrono", "clap", "clap_builder", "crossbeam-utils", + "digest", "either", "fail", "futures", @@ -5354,6 +5494,7 @@ dependencies = [ "futures-executor", "futures-sink", "futures-util", + "hyper", "itertools", "libc", "log", @@ -5372,9 +5513,10 @@ dependencies = [ "scopeguard", "serde", "serde_json", + "smallvec", "socket2 0.4.9", "syn 1.0.109", - "syn 2.0.16", + "syn 2.0.28", "tokio", "tokio-rustls 0.23.4", "tokio-util", @@ -5383,7 +5525,6 @@ dependencies = [ "tower", "tracing", "tracing-core", - "tracing-subscriber", "url", ] @@ -5404,15 +5545,6 @@ dependencies = [ "time", ] -[[package]] -name = "xattr" -version = "0.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d1526bbe5aaeb5eb06885f4d987bcdfa5e23187055de9b83fe00156a821fabc" -dependencies = [ - "libc", -] - [[package]] name = "xattr" version = "1.0.0" diff --git a/Cargo.toml b/Cargo.toml index 5eab28e2e5..f38e67f487 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -23,6 +23,7 @@ members = [ "libs/remote_storage", "libs/tracing-utils", "libs/postgres_ffi/wal_craft", + "libs/vm_monitor", ] [workspace.package] @@ -41,12 +42,14 @@ aws-sdk-s3 = "0.27" aws-smithy-http = "0.55" aws-credential-types = "0.55" aws-types = "0.55" +axum = { version = "0.6.20", features = ["ws"] } base64 = "0.13.0" bincode = "1.3" bindgen = "0.65" bstr = "1.0" byteorder = "1.4" bytes = "1.0" +cfg-if = "1.0.0" chrono = { version = "0.4", default-features = false, features = ["clock"] } clap = { version = "4.0", features = ["derive"] } close_fds = "0.3.2" @@ -74,6 +77,7 @@ humantime = "2.1" humantime-serde = "1.1.1" hyper = "0.14" hyper-tungstenite = "0.9" +inotify = "0.10.2" itertools = "0.10" jsonwebtoken = "8" libc = "0.2" @@ -105,12 +109,14 @@ rustls = "0.20" rustls-pemfile = "1" rustls-split = "0.3" scopeguard = "1.1" +sysinfo = "0.29.2" sentry = { version = "0.30", default-features = false, features = ["backtrace", "contexts", "panic", "rustls", "reqwest" ] } serde = { version = "1.0", features = ["derive"] } serde_json = "1" serde_with = "2.0" sha2 = "0.10.2" signal-hook = "0.3" +smallvec = "1.11" socket2 = "0.5" strum = "0.24" strum_macros = "0.24" @@ -133,7 +139,7 @@ tonic = {version = "0.9", features = ["tls", "tls-roots"]} tracing = "0.1" tracing-error = "0.2.0" tracing-opentelemetry = "0.19.0" -tracing-subscriber = { version = "0.3", default_features = false, features = ["smallvec", "fmt", "tracing-log", "std", "env-filter"] } +tracing-subscriber = { version = "0.3", default_features = false, features = ["smallvec", "fmt", "tracing-log", "std", "env-filter", "json"] } url = "2.2" uuid = { version = "1.2", features = ["v4", "serde"] } walkdir = "2.3.2" @@ -169,6 +175,7 @@ storage_broker = { version = "0.1", path = "./storage_broker/" } # Note: main br tenant_size_model = { version = "0.1", path = "./libs/tenant_size_model/" } tracing-utils = { version = "0.1", path = "./libs/tracing-utils/" } utils = { version = "0.1", path = "./libs/utils/" } +vm_monitor = { version = "0.1", path = "./libs/vm_monitor/" } ## Common library dependency workspace_hack = { version = "0.1", path = "./workspace_hack/" } diff --git a/Dockerfile.compute-node b/Dockerfile.compute-node index 6dc6612e93..0416a98450 100644 --- a/Dockerfile.compute-node +++ b/Dockerfile.compute-node @@ -211,8 +211,8 @@ RUN wget https://github.com/df7cb/postgresql-unit/archive/refs/tags/7.7.tar.gz - FROM build-deps AS vector-pg-build COPY --from=pg-build /usr/local/pgsql/ /usr/local/pgsql/ -RUN wget https://github.com/pgvector/pgvector/archive/refs/tags/v0.4.4.tar.gz -O pgvector.tar.gz && \ - echo "1cb70a63f8928e396474796c22a20be9f7285a8a013009deb8152445b61b72e6 pgvector.tar.gz" | sha256sum --check && \ +RUN wget https://github.com/pgvector/pgvector/archive/refs/tags/v0.5.0.tar.gz -O pgvector.tar.gz && \ + echo "d8aa3504b215467ca528525a6de12c3f85f9891b091ce0e5864dd8a9b757f77b pgvector.tar.gz" | sha256sum --check && \ mkdir pgvector-src && cd pgvector-src && tar xvzf ../pgvector.tar.gz --strip-components=1 -C . && \ make -j $(getconf _NPROCESSORS_ONLN) PG_CONFIG=/usr/local/pgsql/bin/pg_config && \ make -j $(getconf _NPROCESSORS_ONLN) install PG_CONFIG=/usr/local/pgsql/bin/pg_config && \ @@ -764,29 +764,6 @@ RUN rm -r /usr/local/pgsql/include # if they were to be used by other libraries. RUN rm /usr/local/pgsql/lib/lib*.a -######################################################################################### -# -# Extenstion only -# -######################################################################################### -FROM python:3.9-slim-bullseye AS generate-ext-index -ARG PG_VERSION -ARG BUILD_TAG -RUN apt update && apt install -y zstd - -# copy the control files here -COPY --from=kq-imcx-pg-build /extensions/ /extensions/ -COPY --from=pg-anon-pg-build /extensions/ /extensions/ -COPY --from=postgis-build /extensions/ /extensions/ -COPY scripts/combine_control_files.py ./combine_control_files.py -RUN python3 ./combine_control_files.py ${PG_VERSION} ${BUILD_TAG} --public_extensions="anon,postgis" - -FROM scratch AS postgres-extensions -# After the transition this layer will include all extensitons. -# As for now, it's only a couple for testing purposses -COPY --from=generate-ext-index /extensions/*.tar.zst /extensions/ -COPY --from=generate-ext-index /ext_index.json /ext_index.json - ######################################################################################### # # Final layer diff --git a/compute_tools/Cargo.toml b/compute_tools/Cargo.toml index 08dcc21c7a..6c93befaa3 100644 --- a/compute_tools/Cargo.toml +++ b/compute_tools/Cargo.toml @@ -8,6 +8,7 @@ license.workspace = true anyhow.workspace = true async-compression.workspace = true chrono.workspace = true +cfg-if.workspace = true clap.workspace = true flate2.workspace = true futures.workspace = true @@ -23,6 +24,7 @@ tar.workspace = true reqwest = { workspace = true, features = ["json"] } tokio = { workspace = true, features = ["rt", "rt-multi-thread"] } tokio-postgres.workspace = true +tokio-util.workspace = true tracing.workspace = true tracing-opentelemetry.workspace = true tracing-subscriber.workspace = true @@ -34,4 +36,5 @@ utils.workspace = true workspace_hack.workspace = true toml_edit.workspace = true remote_storage = { version = "0.1", path = "../libs/remote_storage/" } +vm_monitor = { version = "0.1", path = "../libs/vm_monitor/" } zstd = "0.12.4" diff --git a/compute_tools/README.md b/compute_tools/README.md index 305ccae5dd..22a7de7cb7 100644 --- a/compute_tools/README.md +++ b/compute_tools/README.md @@ -19,9 +19,10 @@ Also `compute_ctl` spawns two separate service threads: - `http-endpoint` runs a Hyper HTTP API server, which serves readiness and the last activity requests. -If the `vm-informant` binary is present at `/bin/vm-informant`, it will also be started. For VM -compute nodes, `vm-informant` communicates with the VM autoscaling system. It coordinates -downscaling and (eventually) will request immediate upscaling under resource pressure. +If `AUTOSCALING` environment variable is set, `compute_ctl` will start the +`vm-monitor` located in [`neon/libs/vm_monitor`]. For VM compute nodes, +`vm-monitor` communicates with the VM autoscaling system. It coordinates +downscaling and requests immediate upscaling under resource pressure. Usage example: ```sh diff --git a/compute_tools/src/bin/compute_ctl.rs b/compute_tools/src/bin/compute_ctl.rs index 343bb41d3b..2fe07c4287 100644 --- a/compute_tools/src/bin/compute_ctl.rs +++ b/compute_tools/src/bin/compute_ctl.rs @@ -20,9 +20,10 @@ //! - `http-endpoint` runs a Hyper HTTP API server, which serves readiness and the //! last activity requests. //! -//! If the `vm-informant` binary is present at `/bin/vm-informant`, it will also be started. For VM -//! compute nodes, `vm-informant` communicates with the VM autoscaling system. It coordinates -//! downscaling and (eventually) will request immediate upscaling under resource pressure. +//! If `AUTOSCALING` environment variable is set, `compute_ctl` will start the +//! `vm-monitor` located in [`neon/libs/vm_monitor`]. For VM compute nodes, +//! `vm-monitor` communicates with the VM autoscaling system. It coordinates +//! downscaling and requests immediate upscaling under resource pressure. //! //! Usage example: //! ```sh @@ -35,7 +36,6 @@ //! use std::collections::HashMap; use std::fs::File; -use std::panic; use std::path::Path; use std::process::exit; use std::sync::{mpsc, Arc, Condvar, Mutex, RwLock}; @@ -271,6 +271,57 @@ fn main() -> Result<()> { } }; + // Start the vm-monitor if directed to. The vm-monitor only runs on linux + // because it requires cgroups. + cfg_if::cfg_if! { + if #[cfg(target_os = "linux")] { + use std::env; + use tokio_util::sync::CancellationToken; + use tracing::warn; + let vm_monitor_addr = matches.get_one::("vm-monitor-addr"); + let file_cache_connstr = matches.get_one::("filecache-connstr"); + let cgroup = matches.get_one::("cgroup"); + let file_cache_on_disk = matches.get_flag("file-cache-on-disk"); + + // Only make a runtime if we need to. + // Note: it seems like you can make a runtime in an inner scope and + // if you start a task in it it won't be dropped. However, make it + // in the outermost scope just to be safe. + let rt = match (env::var_os("AUTOSCALING"), vm_monitor_addr) { + (None, None) => None, + (None, Some(_)) => { + warn!("--vm-monitor-addr option set but AUTOSCALING env var not present"); + None + } + (Some(_), None) => { + panic!("AUTOSCALING env var present but --vm-monitor-addr option not set") + } + (Some(_), Some(_)) => Some( + tokio::runtime::Builder::new_multi_thread() + .worker_threads(4) + .enable_all() + .build() + .expect("failed to create tokio runtime for monitor"), + ), + }; + + // This token is used internally by the monitor to clean up all threads + let token = CancellationToken::new(); + + let vm_monitor = &rt.as_ref().map(|rt| { + rt.spawn(vm_monitor::start( + Box::leak(Box::new(vm_monitor::Args { + cgroup: cgroup.cloned(), + pgconnstr: file_cache_connstr.cloned(), + addr: vm_monitor_addr.cloned().unwrap(), + file_cache_on_disk, + })), + token.clone(), + )) + }); + } + } + // Wait for the child Postgres process forever. In this state Ctrl+C will // propagate to Postgres and it will be shut down as well. if let Some(mut pg) = pg { @@ -284,6 +335,24 @@ fn main() -> Result<()> { exit_code = ecode.code() } + // Terminate the vm_monitor so it releases the file watcher on + // /sys/fs/cgroup/neon-postgres. + // Note: the vm-monitor only runs on linux because it requires cgroups. + cfg_if::cfg_if! { + if #[cfg(target_os = "linux")] { + if let Some(handle) = vm_monitor { + // Kills all threads spawned by the monitor + token.cancel(); + // Kills the actual task running the monitor + handle.abort(); + + // If handle is some, rt must have been used to produce it, and + // hence is also some + rt.unwrap().shutdown_timeout(Duration::from_secs(2)); + } + } + } + // Maybe sync safekeepers again, to speed up next startup let compute_state = compute.state.lock().unwrap().clone(); let pspec = compute_state.pspec.as_ref().expect("spec must be set"); @@ -393,6 +462,34 @@ fn cli() -> clap::Command { .long("remote-ext-config") .value_name("REMOTE_EXT_CONFIG"), ) + // TODO(fprasx): we currently have default arguments because the cloud PR + // to pass them in hasn't been merged yet. We should get rid of them once + // the PR is merged. + .arg( + Arg::new("vm-monitor-addr") + .long("vm-monitor-addr") + .default_value("0.0.0.0:10301") + .value_name("VM_MONITOR_ADDR"), + ) + .arg( + Arg::new("cgroup") + .long("cgroup") + .default_value("neon-postgres") + .value_name("CGROUP"), + ) + .arg( + Arg::new("filecache-connstr") + .long("filecache-connstr") + .default_value( + "host=localhost port=5432 dbname=postgres user=cloud_admin sslmode=disable", + ) + .value_name("FILECACHE_CONNSTR"), + ) + .arg( + Arg::new("file-cache-on-disk") + .long("file-cache-on-disk") + .action(clap::ArgAction::SetTrue), + ) } #[test] diff --git a/compute_tools/src/compute.rs b/compute_tools/src/compute.rs index d9c71621c2..595ecb453e 100644 --- a/compute_tools/src/compute.rs +++ b/compute_tools/src/compute.rs @@ -1,4 +1,5 @@ use std::collections::HashMap; +use std::env; use std::fs; use std::io::BufRead; use std::os::unix::fs::PermissionsExt; @@ -175,6 +176,27 @@ impl TryFrom for ParsedSpec { } } +/// If we are a VM, returns a [`Command`] that will run in the `neon-postgres` +/// cgroup. Otherwise returns the default `Command::new(cmd)` +/// +/// This function should be used to start postgres, as it will start it in the +/// neon-postgres cgroup if we are a VM. This allows autoscaling to control +/// postgres' resource usage. The cgroup will exist in VMs because vm-builder +/// creates it during the sysinit phase of its inittab. +fn maybe_cgexec(cmd: &str) -> Command { + // The cplane sets this env var for autoscaling computes. + // use `var_os` so we don't have to worry about the variable being valid + // unicode. Should never be an concern . . . but just in case + if env::var_os("AUTOSCALING").is_some() { + let mut command = Command::new("cgexec"); + command.args(["-g", "memory:neon-postgres"]); + command.arg(cmd); + command + } else { + Command::new(cmd) + } +} + /// Create special neon_superuser role, that's a slightly nerfed version of a real superuser /// that we give to customers fn create_neon_superuser(spec: &ComputeSpec, client: &mut Client) -> Result<()> { @@ -451,7 +473,7 @@ impl ComputeNode { pub fn sync_safekeepers(&self, storage_auth_token: Option) -> Result { let start_time = Utc::now(); - let sync_handle = Command::new(&self.pgbin) + let sync_handle = maybe_cgexec(&self.pgbin) .args(["--sync-safekeepers"]) .env("PGDATA", &self.pgdata) // we cannot use -D in this mode .envs(if let Some(storage_auth_token) = &storage_auth_token { @@ -586,7 +608,7 @@ impl ComputeNode { // Start postgres info!("starting postgres"); - let mut pg = Command::new(&self.pgbin) + let mut pg = maybe_cgexec(&self.pgbin) .args(["-D", pgdata]) .spawn() .expect("cannot start postgres process"); @@ -614,7 +636,7 @@ impl ComputeNode { let pgdata_path = Path::new(&self.pgdata); // Run postgres as a child process. - let mut pg = Command::new(&self.pgbin) + let mut pg = maybe_cgexec(&self.pgbin) .args(["-D", &self.pgdata]) .envs(if let Some(storage_auth_token) = &storage_auth_token { vec![("NEON_AUTH_TOKEN", storage_auth_token)] diff --git a/deny.toml b/deny.toml index cfb699bb21..55c581ce3a 100644 --- a/deny.toml +++ b/deny.toml @@ -4,7 +4,12 @@ # to your expectations and requirements. # Root options -targets = [] +targets = [ + { triple = "x86_64-unknown-linux-gnu" }, + { triple = "aarch64-unknown-linux-gnu" }, + { triple = "aarch64-apple-darwin" }, + { triple = "x86_64-apple-darwin" }, +] all-features = false no-default-features = false feature-depth = 1 @@ -18,7 +23,7 @@ vulnerability = "deny" unmaintained = "warn" yanked = "warn" notice = "warn" -ignore = [] +ignore = ["RUSTSEC-2023-0052"] # This section is considered when running `cargo deny check licenses` # More documentation for the licenses section can be found here: diff --git a/docs/rfcs/025-generation-numbers.md b/docs/rfcs/025-generation-numbers.md new file mode 100644 index 0000000000..6a0131c66a --- /dev/null +++ b/docs/rfcs/025-generation-numbers.md @@ -0,0 +1,957 @@ +# Pageserver: split-brain safety for remote storage through generation numbers + +## Summary + +A scheme of logical "generation numbers" for tenant attachment to pageservers is proposed, along with +changes to the remote storage format to include these generation numbers in S3 keys. + +Using the control plane as the issuer of these generation numbers enables strong anti-split-brain +properties in the pageserver cluster without implementing a consensus mechanism directly +in the pageservers. + +## Motivation + +Currently, the pageserver's remote storage format does not provide a mechanism for addressing +split brain conditions that may happen when replacing a node or when migrating +a tenant from one pageserver to another. + +From a remote storage perspective, a split brain condition occurs whenever two nodes both think +they have the same tenant attached, and both can write to S3. This can happen in the case of a +network partition, pathologically long delays (e.g. suspended VM), or software bugs. + +In the current deployment model, control plane guarantees that a tenant is attached to one +pageserver at a time, thereby ruling out split-brain conditions resulting from dual +attachment (however, there is always the risk of a control plane bug). This control +plane guarantee prevents robust response to failures, as if a pageserver is unresponsive +we may not detach from it. The mechanism in this RFC fixes this, by making it safe to +attach to a new, different pageserver even if an unresponsive pageserver may be running. + +Futher, lack of safety during split-brain conditions blocks two important features where occasional +split-brain conditions are part of the design assumptions: + +- seamless tenant migration ([RFC PR](https://github.com/neondatabase/neon/pull/5029)) +- automatic pageserver instance failure handling (aka "failover") (RFC TBD) + +### Prior art + +- 020-pageserver-s3-coordination.md +- 023-the-state-of-pageserver-tenant-relocation.md +- 026-pageserver-s3-mvcc.md + +This RFC has broad similarities to the proposal to implement a MVCC scheme in +S3 object names, but this RFC avoids a general purpose transaction scheme in +favour of more specialized "generations" that work like a transaction ID that +always has the same lifetime as a pageserver process or tenant attachment, whichever +is shorter. + +## Requirements + +- Accommodate storage backends with no atomic or fencing capability (i.e. work within + S3's limitation that there are no atomics and clients can't be fenced) +- Don't depend on any STONITH or node fencing in the compute layer (i.e. we will not + assume that we can reliably kill and EC2 instance and have it die) +- Scoped per-tenant, not per-pageserver; for _seamless tenant migration_, we need + per-tenant granularity, and for _failover_, we likely want to spread the workload + of the failed pageserver instance to a number of peers, rather than monolithically + moving the entire workload to another machine. + We do not rule out the latter case, but should not constrain ourselves to it. + +## Design Tenets + +These are not requirements, but are ideas that guide the following design: + +- Avoid implementing another consensus system: we already have a strongly consistent + database in the control plane that can do atomic operations where needed, and we also + have a Paxos implementation in the safekeeper. +- Avoiding locking in to specific models of how failover will work (e.g. do not assume that + all the tenants on a pageserver will fail over as a unit). +- Be strictly correct when it comes to data integrity. Occasional failures of availability + are tolerable, occasional data loss is not. + +## Non Goals + +The changes in this RFC intentionally isolate the design decision of how to define +logical generations numbers and object storage format in a way that is somewhat flexible with +respect to how actual orchestration of failover works. + +This RFC intentionally does not cover: + +- Failure detection +- Orchestration of failover +- Standby modes to keep data ready for fast migration +- Intentional multi-writer operation on tenants (multi-writer scenarios are assumed to be transient split-brain situations). +- Sharding. + +The interaction between this RFC and those features is discussed in [Appendix B](#appendix-b-interoperability-with-other-features) + +## Impacted Components + +pageserver, control plane, safekeeper (optional) + +## Implementation Part 1: Correctness + +### Summary + +- A per-tenant **generation number** is introduced to uniquely identifying tenant attachments to pageserver processes. + + - This generation number increments each time the control plane modifies a tenant (`Project`)'s assigned pageserver, or when the assigned pageserver restarts. + - the control plane is the authority for generation numbers: only it may + increment a generation number. + +- **Object keys are suffixed** with the generation number +- **Safety for multiply-attached tenants** is provided by the + generation number in the object key: the competing pageservers will not + try to write to the same keys. +- **Safety in split brain for multiple nodes running with + the same node ID** is provided by the pageserver calling out to the control plane + on startup, to re-attach and thereby increment the generations of any attached tenants +- **Safety for deletions** is achieved by deferring the DELETE from S3 to a point in time where the deleting node has validated with control plane that no attachment with a higher generation has a reference to the to-be-DELETEd key. +- **The control plane is used to issue generation numbers** to avoid the need for + a built-in consensus system in the pageserver, although this could in principle + be changed without changing the storage format. + +### Generation numbers + +A generation number is associated with each tenant in the control plane, +and each time the attachment status of the tenant changes, this is incremented. +Changes in attachment status include: + +- Attaching the tenant to a different pageserver +- A pageserver restarting, and "re-attaching" its tenants on startup + +These increments of attachment generation provide invariants we need to avoid +split-brain issues in storage: + +- If two pageservers have the same tenant attached, the attachments are guaranteed to have different generation numbers, because the generation would increment + while attaching the second one. +- If there are multiple pageservers running with the same node ID, all the attachments on all pageservers are guaranteed to have different generation numbers, because the generation would increment + when the second node started and re-attached its tenants. + +As long as the infrastructure does not transparently replace an underlying +physical machine, we are totally safe. See the later [unsafe case](#unsafe-case-on-badly-behaved-infrastructure) section for details. + +### Object Key Changes + +#### Generation suffix + +All object keys (layer objects and index objects) will contain the attachment +generation as a [suffix](#why-a-generation-suffix-rather-than-prefix). +This suffix is the primary mechanism for protecting against split-brain situations, and +enabling safe multi-attachment of tenants: + +- Two pageservers running with the same node ID (e.g. after a failure, where there is + some rogue pageserver still running) will not try to write to the same objects, because at startup they will have re-attached tenants and thereby incremented + generation numbers. +- Multiple attachments (to different pageservers) of the same tenant will not try to write to the same objects, as each attachment would have a distinct generation. + +The generation is appended in hex format (8 byte string representing +u32), to all our existing key names. A u32's range limit would permit +27 restarts _per second_ over a 5 year system lifetime: orders of magnitude more than +is realistic. + +The exact meaning of the generation suffix can evolve over time if necessary, for +example if we chose to implement a failover mechanism internally to the pageservers +rather than going via the control plane. The storage format just sees it as a number, +with the only semantic property being that the highest numbered index is the latest. + +#### Index changes + +Since object keys now include a generation suffix, the index of these keys must also be updated. IndexPart currently stores keys and LSNs sufficient to reconstruct key names: this would be extended to store the generation as well. + +This will increase the size of the file, but only modestly: layers are already encoded as +their string-ized form, so the overhead is about 10 bytes per layer. This will be less if/when +the index storage format is migrated to a binary format from JSON. + +#### Visibility + +_This section doesn't describe code changes, but extends on the consequences of the +object key changes given above_ + +##### Visibility of objects to pageservers + +Pageservers can of course list objects in S3 at any time, but in practice their +visible set is based on the contents of their LayerMap, which is initialized +from the `index_part.json.???` that they load. + +Starting with the `index_part` from the most recent previous generation +(see [loading index_part](#finding-the-remote-indices-for-timelines)), a pageserver +initially has visibility of all the objects that were referenced in the loaded index. +These objects are guaranteed to remain visible until the current generation is +superseded, via pageservers in older generations avoiding deletions (see [deletion](#deletion)). + +The "most recent previous generation" is _not_ necessarily the most recent +in terms of walltime, it is the one that is readable at the time a new generation +starts. Consider the following sequence of a tenant being re-attached to different +pageserver nodes: + +- Create + attach on PS1 in generation 1 +- PS1 Do some work, write out index_part.json-0001 +- Attach to PS2 in generation 2 +- Read index_part.json-0001 +- PS2 starts doing some work... +- Attach to PS3 in generation 3 +- Read index_part.json-0001 +- **...PS2 finishes its work: now it writes index_part.json-0002** +- PS3 writes out index_part.json-0003 + +In the above sequence, the ancestry of indices is: + +``` +0001 -> 0002 + | + -> 0003 +``` + +This is not an issue for safety: if the 0002 references some object that is +not in 0001, then 0003 simply does not see it, and will re-do whatever +work was required (e.g. ingesting WAL or doing compaction). Objects referenced +by only the 0002 index will never be read by future attachment generations, and +will eventually be cleaned up by a scrub (see [scrubbing](#cleaning-up-orphan-objects-scrubbing)). + +##### Visibility of LSNs to clients + +Because index_part.json is now written with a generation suffix, which data +is visible depends on which generation the reader is operating in: + +- If one was passively reading from S3 from outside of a pageserver, the + visibility of data would depend on which index_part.json- file + one had chosen to read from. +- If two pageservers have the same tenant attached, they may have different + data visible as they're independently replaying the WAL, and maintaining + independent LayerMaps that are written to independent index_part.json files. + Data does not have to be remotely committed to be visible. +- For a pageserver writing with a stale generation, historic LSNs + remain readable until another pageserver (with a higher generation suffix) + decides to execute GC deletions. At this point, we may think of the stale + attachment's generation as having logically ended: during its existence + the generation had a consistent view of the world. +- For a newly attached pageserver, its highest visible LSN may appears to + go backwards with respect to an earlier attachment, if that earlier + attachment had not uploaded all data to S3 before the new attachment. + +### Deletion + +#### Generation number validation + +While writes are de-conflicted by writers always using their own generation number in the key, +deletions are slightly more challenging: if a pageserver A is isolated, and the true active node is +pageserver B, then it is dangerous for A to do any object deletions, even of objects that it wrote +itself, because pageserver's B metadata might reference those objects. + +We solve this by inserting a "generation validation" step between the write of a remote index +that un-links a particular object from the index, and the actual deletion of the object, such +that deletions strictly obey the following ordering: + +1. Write out index_part.json: this guarantees that any subsequent reader of the metadata will + not try and read the object we unlinked. +2. Call out to control plane to validate that the generation which we use for our attachment is still the latest. +3. If step 2 passes, it is safe to delete the object. Why? The check-in with control plane + together with our visibility rules guarantees that any later generation + will use either the exact `index_part.json` that we uploaded in step 1, or a successor + of it; not an earlier one. In both cases, the `index_part.json` doesn't reference the + key we are deleting anymore, so, the key is invisible to any later attachment generation. + Hence it's safe to delete it. + +Note that at step 2 we are only confirming that deletions of objects _no longer referenced +by the specific `index_part.json` written in step 1_ are safe. If we were attempting other deletions concurrently, +these would need their own generation validation step. + +If step 2 fails, we may leak the object. This is safe, but has a cost: see [scrubbing](#cleaning-up-orphan-objects-scrubbing). We may avoid this entirely outside of node +failures, if we do proper flushing of deletions on clean shutdown and clean migration. + +To avoid doing a huge number of control plane requests to perform generation validation, +validation of many tenants will be done in a single request, and deletions will be queued up +prior to validation: see [Persistent deletion queue](#persistent-deletion-queue) for more. + +#### `remote_consistent_lsn` updates + +Remote objects are not the only kind of deletion the pageserver does: it also indirectly deletes +WAL data, by feeding back remote_consistent_lsn to safekeepers, as a signal to the safekeepers that +they may drop data below this LSN. + +For the same reasons that deletion of objects must be guarded by an attachment generation number +validation step, updates to `remote_consistent_lsn` are subject to the same rules, using +an ordering as follows: + +1. upload the index_part that covers data up to LSN `L0` to S3 +2. Call out to control plane to validate that the generation which we use for our attachment is still the latest. +3. advance the `remote_consistent_lsn` that we advertise to the safekeepers to `L0` + +If step 2 fails, then the `remote_consistent_lsn` advertised +to safekeepers will not advance again until a pageserver +with the latest generation is ready to do so. + +**Note:** at step 3 we are not advertising the _latest_ remote_consistent_lsn, we are +advertising the value in the index_part that we uploaded in step 1. This provides +a strong ordering guarantee. + +Internally to the pageserver, each timeline will have two remote_consistent_lsn values: the one that +reflects its latest write to remote storage, and the one that reflects the most +recent validation of generation number. It is only the latter value that may +be advertised to the outside world (i.e. to the safekeeper). + +The control plane remains unaware of `remote_consistent_lsn`: it only has to validate +the freshness of generation numbers, thereby granting the pageserver permission to +share the information with the safekeeper. + +For convenience, in subsequent sections and RFCs we will use "deletion" to mean both deletion +of objects in S3, and updates to the `remote_consistent_lsn`, as updates to the remote consistent +LSN are de-facto deletions done via the safekeeper, and both kinds of deletion are subject to +the same generation validation requirement. + +### Pageserver attach/startup changes + +#### Attachment + +Calls to `/v1/tenant/{tenant_id}/attach` are augmented with an additional +`generation` field in the body. + +The pageserver does not persist this: a generation is only good for the lifetime +of a process. + +#### Finding the remote indices for timelines + +Because index files are now suffixed with generation numbers, the pageserver +cannot always GET the remote index in one request, because it can't always +know a-priori what the latest remote index is. + +Typically, the most recent generation to write an index would be our own +generation minus 1. However, this might not be the case: the previous +node might have started and acquired a generation number, and then crashed +before writing out a remote index. + +In the general case and as a fallback, the pageserver may list all the `index_part.json` +files for a timeline, sort them by generation, and pick the highest that is `<=` +its current generation for this attachment. The tenant should never load an index +with an attachment generation _newer_ than its own. +These two rules combined ensure that objects written by later generations are never visible to earlier generations. + +Note that if a given attachment picks an index part from an earlier generation (say n-2), but crashes & restarts before it writes its own generation's index part, next time it tries to pick an index part there may be an index part from generation n-1. +It would pick the n-1 index part in that case, because it's sorted higher than the previous one from generation n-2. +So, above rules guarantee no determinism in selecting the index part. +are allowed to be attached with stale attachment generations during a multiply-attached +phase in a migration, and in this instance if the old location's pageserver restarts, +it should not try and load the newer generation's index. + +To summarize, on starting a timeline, the pageserver will: + +1. Issue a GET for index_part.json- +2. If 1 failed, issue a ListObjectsv2 request for index_part.json\* and + pick the newest. + +One could optimize this further by using the control plane to record specifically +which generation most recently wrote an index_part.json, if necessary, to increase +the probability of finding the index_part.json in one GET. One could also improve +the chances by having pageservers proactively write out index_part.json after they +get a new generation ID. + +#### Re-attachment on startup + +On startup, the pageserver will call out to an new control plane `/re-attach` +API (see [Generation API](#generation-api)). This returns a list of +tenants that should be attached to the pageserver, and their generation numbers, which +the control plane will increment before returning. + +The pageserver should still scan its local disk on startup, but should _delete_ +any local content for tenants not indicated in the `/re-attach` response: their +absence is an implicit detach operation. + +**Note** if a tenant is omitted from the re-attach response, its local disk content +will be deleted. This will change in subsequent work, when the control plane gains +the concept of a secondary/standby location: a node with local content may revert +to this status and retain some local content. + +#### Cleaning up previous generations' remote indices + +Deletion of old indices is not necessary for correctness, although it is necessary +to avoid the ListObjects fallback in the previous section becoming ever more expensive. + +Once the new attachment has written out its index_part.json, it may asynchronously clean up historic index_part.json +objects that were found. + +We may choose to implement this deletion either as an explicit step after we +write out index_part for the first time in a pageserver's lifetime, or for +simplicity just do it periodically as part of the background scrub (see [scrubbing](#cleaning-up-orphan-objects-scrubbing)); + +### Control Plane Changes + +#### Store generations for attaching tenants + +- The `Project` table must store the generation number for use when + attaching the tenant to a new pageserver. +- The `/v1/tenant/:tenant_id/attach` pageserver API will require the generation number, + which the control plane can supply by simply incrementing the `Project`'s + generation number each time the tenant is attached to a different server: the same database + transaction that changes the assigned pageserver should also change the generation number. + +#### Generation API + +This section describes an API that could be provided directly by the control plane, +or built as a separate microservice. In earlier parts of the RFC, when we +discuss the control plane providing generation numbers, we are referring to this API. + +The API endpoints used by the pageserver to acquire and validate generation +numbers are quite simple, and only require access to some persistent and +linerizable storage (such as a database). + +Building this into the control plane is proposed as a least-effort option to exploit existing infrastructure and implement generation number issuance in the same transaction that mandates it (i.e., the transaction that updates the `Project` assignment to another pageserver). +However, this is not mandatory: this "Generation Number Issuer" could +be built as a microservice. In practice, we will write such a miniature service +anyway, to enable E2E pageserver/compute testing without control plane. + +The endpoints required by pageservers are: + +##### `/re-attach` + +- Request: `{node_id: }` +- Response: + - 200 `{tenants: [{id: , gen: }]}` + - 404: unknown node_id + - (Future: 429: flapping detected, perhaps nodes are fighting for the same node ID, + or perhaps this node was in a retry loop) + - (On unknown tenants, omit tenant from `tenants` array) +- Server behavior: query database for which tenants should be attached to this pageserver. + - for each tenant that should be attached, increment the attachment generation and + include the new generation in the response +- Client behavior: + - for all tenants in the response, activate with the new generation number + - for any local disk content _not_ referenced in the response, act as if we + had been asked to detach it (i.e. delete local files) + +**Note** the `node_id` in this request will change in future if we move to ephemeral +node IDs, to be replaced with some correlation ID that helps the control plane realize +if a process is running with the same storage as a previous pageserver process (e.g. +we might use EC instance ID, or we might just write some UUID to the disk the first +time we use it) + +##### `/validate` + +- Request: `{'tenants': [{tenant: , attach_gen: }, ...]}'` +- Response: + - 200 `{'tenants': [{tenant: , status: }...]}` + - (On unknown tenants, omit tenant from `tenants` array) +- Purpose: enable the pageserver to discover for the given attachments whether they are still the latest. +- Server behavior: this is a read-only operation: simply compare the generations in the request with + the generations known to the server, and set status to `true` if they match. +- Client behavior: clients must not do deletions within a tenant's remote data until they have + received a response indicating the generation they hold for the attachment is current. + +#### Use of `/load` and `/ignore` APIs + +Because the pageserver will be changed to only attach tenants on startup +based on the control plane's response to a `/re-attach` request, the load/ignore +APIs no longer make sense in their current form. + +The `/load` API becomes functionally equivalent to attach, and will be removed: +any location that used `/load` before should just attach instead. + +The `/ignore` API is equivalent to detaching, but without deleting local files. + +### Timeline/Branch creation & deletion + +All of the previous arguments for safety have described operations within +a timeline, where we may describe a sequence that includes updates to +index_part.json, and where reads and writes are coming from a postgres +endpoint (writes via the safekeeper). + +Creating or destroying timeline is a bit different, because writes +are coming from the control plane. + +We must be safe against scenarios such as: + +- A tenant is attached to pageserver B while pageserver A is + in the middle of servicing an RPC from the control plane to + create or delete a tenant. +- A pageserver A has been sent a timeline creation request + but becomes unresponsive. The tenant is attached to a + different pageserver B, and the timeline creation request + is sent there too. + +#### Timeline Creation + +If some very slow node tries to do a timeline creation _after_ +a more recent generation node has already created the timeline +and written some data into it, that must not cause harm. This +is provided in timeline creations by the way all the objects +within the timeline's remote path include a generation suffix: +a slow node in an old generation that attempts to "create" a timeline +that already exists will just emit an index_part.json with +an old generation suffix. + +Timeline IDs are never reused, so we don't have +to worry about the case of create/delete/create cycles. If they +were re-used during a disaster recovery "un-delete" of a timeline, +that special case can be handled by calling out to all available pageservers +to check that they return 404 for the timeline, and to flush their +deletion queues in case they had any deletions pending from the +timeline. + +The above makes it safe for control plane to change the assignment of +tenant to pageserver in control plane while a timeline creation is ongoing. +The reason is that the creation request against the new assigned pageserver +uses a new generation number. However, care must be taken by control plane +to ensure that a "timeline creation successul" response from some pageserver +is checked for the pageserver's generation for that timeline's tenant still being the latest. +If it is not the latest, the response does not constitute a successful timeline creation. +It is acceptable to discard such responses, the scrubber will clean up the S3 state. +It is better to issue a timelien deletion request to the stale attachment. + +#### Timeline Deletion + +Tenant/timeline deletion operations are exempt from generation validation +on deletes, and therefore don't have to go through the same deletion +queue as GC/compaction layer deletions. This is because once a +delete is issued by the control plane, it is a promise that the +control plane will keep trying until the deletion is done, so even stale +pageservers are permitted to go ahead and delete the objects. + +The implications of this for control plane are: + +- During timeline/tenant deletion, the control plane must wait for the deletion to + be truly complete (status 404) and also handle the case where the pageserver + becomes unavailable, either by waiting for a replacement with the same node_id, + or by *re-attaching the tenant elsewhere. + +- The control plane must persist its intent to delete + a timeline/tenant before issuing any RPCs, and then once it starts, it must + keep retrying until the tenant/timeline is gone. This is already handled + by using a persistent `Operation` record that is retried indefinitely. + +Timeline deletion may result in a special kind of object leak, where +the latest generation attachment completes a deletion (including erasing +all objects in the timeline path), but some slow/partitioned node is +writing into the timeline path with a stale generation number. This would +not be caught by any per-timeline scrubbing (see [scrubbing](#cleaning-up-orphan-objects-scrubbing)), since scrubbing happens on the +attached pageserver, and once the timeline is deleted it isn't attached anywhere. +This scenario should be pretty rare, and the control plane can make it even +rarer by ensuring that if a tenant is in a multi-attached state (e.g. during +migration), we wait for that to complete before processing the deletion. Beyond +that, we may implement some other top-level scrub of timelines in +an external tool, to identify any tenant/timeline paths that are not found +in the control plane database. + +#### Examples + +- Deletion, node restarts partway through: + - By the time we returned 202, we have written a remote delete marker + - Any subsequent incarnation of the same node_id will see the remote + delete marker and continue to process the deletion + - If the original pageserver is lost permanently and no replacement + with the same node_id is available, then the control plane must recover + by re-attaching the tenant to a different node. +- Creation, node becomes unresponsive partway through. + - Control plane will see HTTP request timeout, keep re-issuing + request to whoever is the latest attachment point for the tenant + until it succeeds. + - Stale nodes may be trying to execute timeline creation: they will + write out index_part.json files with + stale attachment generation: these will be eventually cleaned up + by the same mechanism as other old indices. + +### Unsafe case on badly behaved infrastructure + +This section is only relevant if running on a different environment +than EC2 machines with ephemeral disks. + +If we ever run pageservers on infrastructure that might transparently restart +a pageserver while leaving an old process running (e.g. a VM gets rescheduled +without the old one being fenced), then there is a risk of corruption, when +the control plane attaches the tenant, as follows: + +- If the control plane sends an `/attach` request to node A, then node A dies + and is replaced, and the control plane's retries the request without + incrementing that attachment ID, then it could end up with two physical nodes + both using the same generation number. +- This is not an issue when using EC2 instances with ephemeral storage, as long + as the control plane never re-uses a node ID, but it would need re-examining + if running on different infrastructure. +- To robustly protect against this class of issue, we would either: + - add a "node generation" to distinguish between different processes holding the + same node_id. + - or, dispense with static node_id entirely and issue an ephemeral ID to each + pageserver process when it starts. + +## Implementation Part 2: Optimizations + +### Persistent deletion queue + +Between writing our a new index_part.json that doesn't reference an object, +and executing the deletion, an object passes through a window where it is +only referenced in memory, and could be leaked if the pageserver is stopped +uncleanly. That introduces conflicting incentives: on the one hand, we would +like to delay and batch deletions to +1. minimize the cost of the mandatory validations calls to control plane, and +2. minimize cost for DeleteObjects requests. +On the other hand we would also like to minimize leakage by executing +deletions promptly. + +To resolve this, we may make the deletion queue persistent +and then executing these in the background at a later time. + +_Note: The deletion queue's reason for existence is optimization rather than correctness, +so there is a lot of flexibility in exactly how the it should work, +as long as it obeys the rule to validate generations before executing deletions, +so the following details are not essential to the overall RFC._ + +#### Scope + +The deletion queue will be global per pageserver, not per-tenant. There +are several reasons for this choice: + +- Use the queue as a central point to coalesce validation requests to the + control plane: this avoids individual `Timeline` objects ever touching + the control plane API, and avoids them having to know the rules about + validating deletions. This separation of concerns will avoid burdening + the already many-LoC `Timeline` type with even more responsibility. +- Decouple the deletion queue from Tenant attachment lifetime: we may + "hibernate" an inactive tenant by tearing down its `Tenant`/`Timeline` + objects in the pageserver, without having to wait for deletions to be done. +- Amortize the cost of I/O for the persistent queue, instead of having many + tiny queues. +- Coalesce deletions into a smaller number of larger DeleteObjects calls + +Because of the cost of doing I/O for persistence, and the desire to coalesce +generation validation requests across tenants, and coalesce deletions into +larger DeleteObjects requests, there will be one deletion queue per pageserver +rather than one per tenant. This has the added benefit that when deactivating +a tenant, we do not have to drain their deletion queue: deletions can proceed +for a tenant whose main `Tenant` object has been torn down. + +#### Flow of deletion + +The flow of a deletion is becomes: + +1. Need for deletion of an object (=> layer file) is identified. +2. Unlink the object from all the places that reference it (=> `index_part.json`). +3. Enqueue the deletion to a persistent queue. + Each entry is `tenant_id, attachment_generation, S3 key`. +4. Validate & execute in batches: + 4.1 For a batch of entries, call into control plane. + 4.2 For the subset of entries that passed validation, execute a `DeleteObjects` S3 DELETE request for their S3 keys. + +As outlined in the Part 1 on correctness, it is critical that deletions are only +executed once the key is not referenced anywhere in S3. +This property is obviously upheld by the scheme above. + +#### We Accept Object Leakage In Acceptable Circumcstances + +If we crash in the flow above between (2) and (3), we lose track of unreferenced object. +Further, enqueuing a single to the persistent queue may not be durable immediately to amortize cost of flush to disk. +This is acceptable for now, it can be caught by [the scrubber](#cleaning-up-orphan-objects-scrubbing). + +There are various measures we can take to improve this in the future. +1. Cap amount of time until enqueued entry becomes durable (timeout for flush-to-tisk) +2. Proactively flush: + - On graceful shutdown, as we anticipate that some or + all of our attachments may be re-assigned while we are offline. + - On tenant detach. +3. For each entry, keep track of whether it has passed (2). + Only admit entries to (4) one they have passed (2). + This requires re-writing / two queue entries (intent, commit) per deletion. + +The important take-away with any of the above is that it's not +disastrous to leak objects in exceptional circumstances. + +#### Operations that may skip the queue + +Deletions of an entire timeline are [exempt](#Timeline-Deletion) from generation number validation. Once the +control plane sends the deletion request, there is no requirement to retain the readability +of any data within the timeline, and all objects within the timeline path may be deleted +at any time from the control plane's deletion request onwards. + +Since deletions of smaller timelines won't have enough objects to compose a full sized +DeleteObjects request, it is still useful to send these through the last part of the +deletion pipeline to coalesce with other executing deletions: to enable this, the +deletion queue should expose two input channels: one for deletions that must be +processed in a generation-aware way, and a fast path for timeline deletions, where +that fast path may skip validation and the persistent queue. + +### Cleaning up orphan objects (scrubbing) + +An orphan object is any object which is no longer referenced by a running node or by metadata. + +Examples of how orphan objects arise: + +- A node PUTs a layer object, then crashes before it writes the + index_part.json that references that layer. +- A stale node carries on running for some time, and writes out an unbounded number of + objects while it believes itself to be the rightful writer for a tenant. +- A pageserver crashes between un-linking an object from the index, and persisting + the object to its deletion queue. + +Orphan objects are functionally harmless, but have a small cost due to S3 capacity consumed. We +may clean them up at some time in the future, but doing a ListObjectsv2 operation and cross +referencing with the latest metadata to identify objects which are not referenced. + +Scrubbing will be done only by an attached pageserver (not some third party process), and deletions requested during scrub will go through the same +validation as all other deletions: the attachment generation must be +fresh. This avoids the possibility of a stale pageserver incorrectly +thinking than an object written by a newer generation is stale, and deleting +it. + +It is not strictly necessary that scrubbing be done by an attached +pageserver: it could also be done externally. However, an external +scrubber would still require the same validation procedure that +a pageserver's deletion queue performs, before actually erasing +objects. + +## Operational impact + +### Availability + +Coordination of generation numbers via the control plane introduce a dependency for certain +operations: + +1. Starting new pageservers (or activating pageservers after a restart) +2. Executing enqueued deletions +3. Advertising updated `remote_consistent_lsn` to enable WAL trimming + +Item 1. would mean that some in-place restarts that previously would have resumed service even if the control plane were +unavailable, will now not resume service to users until the control plane is available. We could +avoid this by having a timeout on communication with the control plane, and after some timeout, +resume service with the previous generation numbers (assuming this was persisted to disk). However, +this is unlikely to be needed as the control plane is already an essential & highly available component. Also, having a node re-use an old generation number would complicate +reasoning about the system, as it would break the invariant that a generation number uniquely identifies +a tenant's attachment to a given pageserver _process_: it would merely identify the tenant's attachment +to the pageserver _machine_ or its _on-disk-state_. + +Item 2. is a non-issue operationally: it's harmless to delay deletions, the only impact of objects pending deletion is +the S3 capacity cost. + +Item 3. could be an issue if safekeepers are low on disk space and the control plane is unavailable for a long time. If this became an issue, +we could adjust the safekeeper to delete segments from local disk sooner, as soon as they're uploaded to S3, rather than waiting for +remote_consistent_lsn to advance. + +For a managed service, the general approach should be to make sure we are monitoring & respond fast enough +that control plane outages are bounded in time. + +There is also the fact that control plane runs in a single region. +The latency for distant regions is not a big concern for us because all request types added by this RFC are either infrequent or not in the way of the data path. +However, we lose region isolation for the operations listed above. +The ongoing work to split console and control will give us per-region control plane, and all operations in this RFC can be handled by these per-region control planes. +With that in mind, we accept the trade-offs outlined in this paragraph. + +We will also implement an "escape hatch" config generation numbers, where in a major disaster outage, +we may manually run pageservers with a hand-selected generation number, so that we can bring them online +independently of a control plane. + +### Rollout + +Although there is coupling between components, we may deploy most of the new data plane components +independently of the control plane: initially they can just use a static generation number. + +#### Phase 1 + +The pageserver is deployed with some special config to: + +- Always act like everything is generation 1 and do not wait for a control plane issued generation on attach +- Skip the places in deletion and remote_consistent_lsn updates where we would call into control plane + +#### Phase 2 + +The control plane changes are deployed: control plane will now track and increment generation numbers. + +#### Phase 3 + +The pageserver is deployed with its control-plane-dependent changes enabled: it will now require +the control plane to service re-attach requests on startup, and handle generation +validation requests. + +### On-disk backward compatibility + +Backward compatibility with existing data is straightforward: + +- When reading the index, we may assume that any layer whose metadata doesn't include + generations will have a path without generation suffix. +- When locating the index file on attachment, we may use the "fallback" listing path + and if there is only an index without generation suffix, that is the one we load. + +It is not necessary to re-write existing layers: even new index files will be able +to represent generation-less layers. + +### On-disk forward compatibility + +We will do a two phase rollout, probably over multiple releases because we will naturally +have some of the read-side code ready before the overall functionality is ready: + +1. Deploy pageservers which understand the new index format and generation suffixes + in keys, but do not write objects with generation numbers in the keys. +2. Deploy pageservers that write objects with generation numbers in the keys. + +Old pageservers will be oblivious to generation numbers. That means that they can't +read objects with generation numbers in the name. This is why we must +first step must deploy the ability to read, before the second step +starts writing them. + +# Frequently Asked Questions + +## Why a generation _suffix_ rather than _prefix_? + +The choice is motivated by object listing, since one can list by prefix but not +suffix. + +In [finding remote indices](#finding-the-remote-indices-for-timelines), we rely +on being able to do a prefix listing for `//index_part.json*`. +That relies on the prefix listing. + +The converse case of using a generation prefix and listing by generation is +not needed: one could imagine listing by generation while scrubbing (so that +a particular generation's layers could be scrubbed), but this is not part +of normal operations, and the [scrubber](#cleaning-up-orphan-objects-scrubbing) probably won't work that way anyway. + +## Wouldn't it be simpler to have a separate deletion queue per timeline? + +Functionally speaking, we could. That's how RemoteTimelineClient currently works, +but this approach does not map well to a long-lived persistent queue with +generation validation. + +Anything we do per-timeline generates tiny random I/O, on a pageserver with +tens of thousands of timelines operating: to be ready for high scale, we should: + +- A) Amortize costs where we can (e.g. a shared deletion queue) +- B) Expect to put tenants into a quiescent state while they're not + busy: i.e. we shouldn't keep a tenant alive to service its deletion queue. + +This was discussed in the [scope](#scope) part of the deletion queue section. + +# Appendix A: Examples of use in high availability/failover + +The generation numbers proposed in this RFC are adaptable to a variety of different +failover scenarios and models. The sections below sketch how they would work in practice. + +### In-place restart of a pageserver + +"In-place" here means that the restart is done before any other element in the system +has taken action in response to the node being down. + +- After restart, the node issues a re-attach request to the control plane, and + receives new generation numbers for all its attached tenants. +- Tenants may be activated with the generation number in the re-attach response. +- If any of its attachments were in fact stale (i.e. had be reassigned to another + node while this node was offline), then + - the re-attach response will inform the tenant about this by not including + the tenant of this by _not_ incrementing the generation for that attachment. + - This will implicitly block deletions in the tenant, but as an optimization + the pageserver should also proactively stop doing S3 uploads when it notices this stale-generation state. + - The control plane is expected to eventually detach this tenant from the + pageserver. + +If the control plane does not include a tenant in the re-attach response, +but there is still local state for the tenant in the filesystem, the pageserver +deletes the local state in response and does not load/active the tenant. +See the [earlier section on pageserver startup](#pageserver-attachstartup-changes) for details. +Control plane can use this mechanism to clean up a pageserver that has been +down for so long that all its tenants were migrated away before it came back +up again and asked for re-attach. + +### Failure of a pageserver + +In this context, read "failure" as the most ambiguous possible case, where +a pageserver is unavailable to clients and control plane, but may still be executing and talking +to S3. + +#### Case A: re-attachment to other nodes + +1. Let's say node 0 becomes unresponsive in a cluster of three nodes 0, 1, 2. +2. Some external mechanism notices that the node is unavailable and initiates + movement of all tenants attached to that node to a different node according + to some distribution rule. + In this example, it would mean incrementing the generation + of all tenants that were attached to node 0, as each tenant's assigned pageserver changes. +3. A tenant which is now attached to node 1 will _also_ still be attached to node + 0, from the perspective of node 0. Node 0 will still be using its old generation, + node 1 will be using a newer generation. +4. S3 writes will continue from nodes 0 and 1: there will be an index_part.json-00000001 + \_and\* an index_part.json-00000002. Objects written under the old suffix + after the new attachment was created do not matter from the rest of the system's + perspective: the endpoints are reading from the new attachment location. Objects + written by node 0 are just garbage that can be cleaned up at leisure. Node 0 will + not do any deletions because it can't synchronize with control plane, or if it could, + its deletion queue processing would get errors for the validation requests. + +#### Case B: direct node replacement with same node_id and drive + +This is the scenario we would experience if running pageservers in some dynamic +VM/container environment that would auto-replace a given node_id when it became +unresponsive, with the node's storage supplied by some network block device +that is attached to the replacement VM/container. + +1. Let's say node 0 fails, and there may be some other peers but they aren't relevant. +2. Some external mechanism notices that the node is unavailable, and creates + a "new node 0" (Node 0b) which is a physically separate server. The original node 0 + (Node 0a) may still be running, because we do not assume the environment fences nodes. +3. On startup, node 0b re-attaches and gets higher generation numbers for + all tenants. +4. S3 writes continue from nodes 0a and 0b, but the writes do not collide due to different + generation in the suffix, and the writes from node 0a are not visible to the rest + of the system because endpoints are reading only from node 0b. + +# Appendix B: interoperability with other features + +## Sharded Keyspace + +The design in this RFC maps neatly to a sharded keyspace design where subsets of the key space +for a tenant are assigned to different pageservers: + +- the "unit of work" for attachments becomes something like a TenantShard rather than a Tenant +- TenantShards get generation numbers just as Tenants do. +- Write workload (ingest, compaction) for a tenant is spread out across pageservers via + TenantShards, but each TenantShard still has exactly one valid writer at a time. + +## Read replicas + +_This section is about a passive reader of S3 pageserver state, not a postgres +read replica_ + +For historical reads to LSNs below the remote persistent LSN, any node may act as a reader at any +time: remote data is logically immutable data, and the use of deferred deletion in this RFC helps +mitigate the fact that remote data is not _physically_ immutable (i.e. the actual data for a given +page moves around as compaction happens). + +A read replica needs to be aware of generations in remote data in order to read the latest +metadata (find the index_part.json with the latest suffix). It may either query this +from the control plane, or find it with ListObjectsv2 request + +## Seamless migration + +To make tenant migration totally seamless, we will probably want to intentionally double-attach +a tenant briefly, serving reads from the old node while waiting for the new node to be ready. + +This RFC enables that double-attachment: two nodes may be attached at the same time, with the migration destination +having a higher generation number. The old node will be able to ingest and serve reads, but not +do any deletes. The new node's attachment must also avoid deleting layers that the old node may +still use. A new piece of state +will be needed for this in the control plane's definition of an attachment. + +## Warm secondary locations + +To enable faster tenant movement after a pageserver is lost, we will probably want to spend some +disk capacity on keeping standby locations populated with local disk data. + +There's no conflict between this RFC and that: implementing warm secondary locations on a per-tenant basis +would be a separate change to the control plane to store standby location(s) for a tenant. Because +the standbys do not write to S3, they do not need to be assigned generation numbers. When a tenant is +re-attached to a standby location, that would increment the tenant attachment generation and this +would work the same as any other attachment change, but with a warm cache. + +## Ephemeral node IDs + +This RFC intentionally avoids changing anything fundamental about how pageservers are identified +and registered with the control plane, to avoid coupling the implementation of pageserver split +brain protection with more fundamental changes in the management of the pageservers. + +Moving to ephemeral node IDs would provide an extra layer of +resilience in the system, as it would prevent the control plane +accidentally attaching to two physical nodes with the same +generation, if somehow there were two physical nodes with +the same node IDs (currently we rely on EC2 guarantees to +eliminate this scenario). With ephemeral node IDs, there would be +no possibility of that happening, no matter the behavior of +underlying infrastructure. + +Nothing fundamental in the pageserver's handling of generations needs to change to handle ephemeral node IDs, since we hardly use the +`node_id` anywhere. The `/re-attach` API would be extended +to enable the pageserver to obtain its ephemeral ID, and provide +some correlation identifier (e.g. EC instance ID), to help the +control plane re-attach tenants to the same physical server that +previously had them attached. diff --git a/libs/safekeeper_api/src/models.rs b/libs/safekeeper_api/src/models.rs index 0c1310eef9..eaea266f32 100644 --- a/libs/safekeeper_api/src/models.rs +++ b/libs/safekeeper_api/src/models.rs @@ -31,6 +31,8 @@ fn lsn_invalid() -> Lsn { #[serde_as] #[derive(Debug, Clone, Deserialize, Serialize)] pub struct SkTimelineInfo { + /// Term. + pub term: Option, /// Term of the last entry. pub last_log_term: Option, /// LSN of the last record. @@ -58,4 +60,6 @@ pub struct SkTimelineInfo { /// A connection string to use for WAL receiving. #[serde(default)] pub safekeeper_connstr: Option, + #[serde(default)] + pub http_connstr: Option, } diff --git a/libs/utils/Cargo.toml b/libs/utils/Cargo.toml index f01131540e..1eb9e6ab4d 100644 --- a/libs/utils/Cargo.toml +++ b/libs/utils/Cargo.toml @@ -26,6 +26,7 @@ serde_json.workspace = true signal-hook.workspace = true thiserror.workspace = true tokio.workspace = true +tokio-util.workspace = true tracing.workspace = true tracing-error.workspace = true tracing-subscriber = { workspace = true, features = ["json", "registry"] } @@ -37,6 +38,7 @@ url.workspace = true uuid.workspace = true pq_proto.workspace = true +postgres_connection.workspace = true metrics.workspace = true workspace_hack.workspace = true diff --git a/libs/utils/src/backoff.rs b/libs/utils/src/backoff.rs index 4507efe647..d50ad39585 100644 --- a/libs/utils/src/backoff.rs +++ b/libs/utils/src/backoff.rs @@ -1,18 +1,31 @@ use std::fmt::{Debug, Display}; use futures::Future; +use tokio_util::sync::CancellationToken; pub const DEFAULT_BASE_BACKOFF_SECONDS: f64 = 0.1; pub const DEFAULT_MAX_BACKOFF_SECONDS: f64 = 3.0; -pub async fn exponential_backoff(n: u32, base_increment: f64, max_seconds: f64) { +pub async fn exponential_backoff( + n: u32, + base_increment: f64, + max_seconds: f64, + cancel: &CancellationToken, +) { let backoff_duration_seconds = exponential_backoff_duration_seconds(n, base_increment, max_seconds); if backoff_duration_seconds > 0.0 { tracing::info!( "Backoff: waiting {backoff_duration_seconds} seconds before processing with the task", ); - tokio::time::sleep(std::time::Duration::from_secs_f64(backoff_duration_seconds)).await; + + drop( + tokio::time::timeout( + std::time::Duration::from_secs_f64(backoff_duration_seconds), + cancel.cancelled(), + ) + .await, + ) } } @@ -24,28 +37,57 @@ pub fn exponential_backoff_duration_seconds(n: u32, base_increment: f64, max_sec } } +/// Configure cancellation for a retried operation: when to cancel (the token), and +/// what kind of error to return on cancellation +pub struct Cancel +where + E: Display + Debug + 'static, + CF: Fn() -> E, +{ + token: CancellationToken, + on_cancel: CF, +} + +impl Cancel +where + E: Display + Debug + 'static, + CF: Fn() -> E, +{ + pub fn new(token: CancellationToken, on_cancel: CF) -> Self { + Self { token, on_cancel } + } +} + /// retries passed operation until one of the following conditions are met: /// Encountered error is considered as permanent (non-retryable) /// Retries have been exhausted. /// `is_permanent` closure should be used to provide distinction between permanent/non-permanent errors /// When attempts cross `warn_threshold` function starts to emit log warnings. /// `description` argument is added to log messages. Its value should identify the `op` is doing -pub async fn retry( +/// `cancel` argument is required: any time we are looping on retry, we should be using a CancellationToken +/// to drop out promptly on shutdown. +pub async fn retry( mut op: O, is_permanent: impl Fn(&E) -> bool, warn_threshold: u32, max_retries: u32, description: &str, + cancel: Cancel, ) -> Result where // Not std::error::Error because anyhow::Error doesnt implement it. // For context see https://github.com/dtolnay/anyhow/issues/63 - E: Display + Debug, + E: Display + Debug + 'static, O: FnMut() -> F, F: Future>, + CF: Fn() -> E, { let mut attempts = 0; loop { + if cancel.token.is_cancelled() { + return Err((cancel.on_cancel)()); + } + let result = op().await; match result { Ok(_) => { @@ -80,6 +122,7 @@ where attempts, DEFAULT_BASE_BACKOFF_SECONDS, DEFAULT_MAX_BACKOFF_SECONDS, + &cancel.token, ) .await; attempts += 1; @@ -132,6 +175,7 @@ mod tests { 1, 1, "work", + Cancel::new(CancellationToken::new(), || -> io::Error { unreachable!() }), ) .await; @@ -157,6 +201,7 @@ mod tests { 2, 2, "work", + Cancel::new(CancellationToken::new(), || -> io::Error { unreachable!() }), ) .await .unwrap(); @@ -179,6 +224,7 @@ mod tests { 2, 2, "work", + Cancel::new(CancellationToken::new(), || -> io::Error { unreachable!() }), ) .await .unwrap_err(); diff --git a/libs/utils/src/lib.rs b/libs/utils/src/lib.rs index 638dba427b..6cf829a67c 100644 --- a/libs/utils/src/lib.rs +++ b/libs/utils/src/lib.rs @@ -58,6 +58,8 @@ pub mod serde_regex; pub mod pageserver_feedback; +pub mod postgres_client; + pub mod tracing_span_assert; pub mod rate_limit; diff --git a/libs/utils/src/postgres_client.rs b/libs/utils/src/postgres_client.rs new file mode 100644 index 0000000000..dba74f5b0b --- /dev/null +++ b/libs/utils/src/postgres_client.rs @@ -0,0 +1,37 @@ +//! Postgres client connection code common to other crates (safekeeper and +//! pageserver) which depends on tenant/timeline ids and thus not fitting into +//! postgres_connection crate. + +use anyhow::Context; +use postgres_connection::{parse_host_port, PgConnectionConfig}; + +use crate::id::TenantTimelineId; + +/// Create client config for fetching WAL from safekeeper on particular timeline. +/// listen_pg_addr_str is in form host:\[port\]. +pub fn wal_stream_connection_config( + TenantTimelineId { + tenant_id, + timeline_id, + }: TenantTimelineId, + listen_pg_addr_str: &str, + auth_token: Option<&str>, + availability_zone: Option<&str>, +) -> anyhow::Result { + let (host, port) = + parse_host_port(listen_pg_addr_str).context("Unable to parse listen_pg_addr_str")?; + let port = port.unwrap_or(5432); + let mut connstr = PgConnectionConfig::new_host_port(host, port) + .extend_options([ + "-c".to_owned(), + format!("timeline_id={}", timeline_id), + format!("tenant_id={}", tenant_id), + ]) + .set_password(auth_token.map(|s| s.to_owned())); + + if let Some(availability_zone) = availability_zone { + connstr = connstr.extend_options([format!("availability_zone={}", availability_zone)]); + } + + Ok(connstr) +} diff --git a/libs/vm_monitor/Cargo.toml b/libs/vm_monitor/Cargo.toml new file mode 100644 index 0000000000..26b976830a --- /dev/null +++ b/libs/vm_monitor/Cargo.toml @@ -0,0 +1,31 @@ +[package] +name = "vm_monitor" +version = "0.1.0" +edition.workspace = true +license.workspace = true + +[[bin]] +name = "vm-monitor" +path = "./src/bin/monitor.rs" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +anyhow.workspace = true +axum.workspace = true +clap.workspace = true +futures.workspace = true +inotify.workspace = true +serde.workspace = true +serde_json.workspace = true +sysinfo.workspace = true +tokio.workspace = true +tokio-postgres.workspace = true +tokio-stream.workspace = true +tokio-util.workspace = true +tracing.workspace = true +tracing-subscriber.workspace = true +workspace_hack = { version = "0.1", path = "../../workspace_hack" } + +[target.'cfg(target_os = "linux")'.dependencies] +cgroups-rs = "0.3.3" diff --git a/libs/vm_monitor/README.md b/libs/vm_monitor/README.md new file mode 100644 index 0000000000..53cdecd9f3 --- /dev/null +++ b/libs/vm_monitor/README.md @@ -0,0 +1,34 @@ +# `vm-monitor` + +The `vm-monitor` (or just monitor) is a core component of the autoscaling system, +along with the `autoscale-scheduler` and the `autoscaler-agent`s. The monitor has +two primary roles: 1) notifying agents when immediate upscaling is necessary due +to memory conditions and 2) managing Postgres' file cache and a cgroup to carry +out upscaling and downscaling decisions. + +## More on scaling + +We scale CPU and memory using NeonVM, our in-house QEMU tool for use with Kubernetes. +To control thresholds for receiving memory usage notifications, we start Postgres +in the `neon-postgres` cgroup and set its `memory.{max,high}`. + +* See also: [`neondatabase/autoscaling`](https://github.com/neondatabase/autoscaling/) +* See also: [`neondatabase/vm-monitor`](https://github.com/neondatabase/vm-monitor/), +where initial development of the monitor happened. The repository is no longer +maintained but the commit history may be useful for debugging. + +## Structure + +The `vm-monitor` is loosely comprised of a few systems. These are: +* the server: this is just a simple `axum` server that accepts requests and +upgrades them to websocket connections. The server only allows one connection at +a time. This means that upon receiving a new connection, the server will terminate +and old one if it exists. +* the filecache: a struct that allows communication with the Postgres file cache. +On startup, we connect to the filecache and hold on to the connection for the +entire monitor lifetime. +* the cgroup watcher: the `CgroupWatcher` manages the `neon-postgres` cgroup by +listening for `memory.high` events and setting its `memory.{high,max}` values. +* the runner: the runner marries the filecache and cgroup watcher together, +communicating with the agent throught the `Dispatcher`, and then calling filecache +and cgroup watcher functions as needed to upscale and downscale diff --git a/libs/vm_monitor/src/bin/monitor.rs b/libs/vm_monitor/src/bin/monitor.rs new file mode 100644 index 0000000000..873618fc31 --- /dev/null +++ b/libs/vm_monitor/src/bin/monitor.rs @@ -0,0 +1,33 @@ +// We expose a standalone binary _and_ start the monitor in `compute_ctl` so that +// we can test the monitor as part of the entire autoscaling system in +// neondatabase/autoscaling. +// +// The monitor was previously started by vm-builder, and for testing purposes, +// we can mimic that setup with this binary. + +#[cfg(target_os = "linux")] +#[tokio::main] +async fn main() -> anyhow::Result<()> { + use clap::Parser; + use tokio_util::sync::CancellationToken; + use tracing_subscriber::EnvFilter; + use vm_monitor::Args; + + let subscriber = tracing_subscriber::fmt::Subscriber::builder() + .json() + .with_file(true) + .with_line_number(true) + .with_span_list(true) + .with_env_filter(EnvFilter::from_default_env()) + .finish(); + tracing::subscriber::set_global_default(subscriber)?; + + let args: &'static Args = Box::leak(Box::new(Args::parse())); + let token = CancellationToken::new(); + vm_monitor::start(args, token).await +} + +#[cfg(not(target_os = "linux"))] +fn main() { + panic!("the monitor requires cgroups, which are only available on linux") +} diff --git a/libs/vm_monitor/src/cgroup.rs b/libs/vm_monitor/src/cgroup.rs new file mode 100644 index 0000000000..4f529d16fd --- /dev/null +++ b/libs/vm_monitor/src/cgroup.rs @@ -0,0 +1,693 @@ +use std::{ + fmt::{Debug, Display}, + fs, + pin::pin, + sync::atomic::{AtomicU64, Ordering}, +}; + +use anyhow::{anyhow, bail, Context}; +use cgroups_rs::{ + freezer::FreezerController, + hierarchies::{self, is_cgroup2_unified_mode, UNIFIED_MOUNTPOINT}, + memory::MemController, + MaxValue, + Subsystem::{Freezer, Mem}, +}; +use inotify::{EventStream, Inotify, WatchMask}; +use tokio::sync::mpsc::{self, error::TryRecvError}; +use tokio::time::{Duration, Instant}; +use tokio_stream::{Stream, StreamExt}; +use tracing::{info, warn}; + +use crate::protocol::Resources; +use crate::MiB; + +/// Monotonically increasing counter of the number of memory.high events +/// the cgroup has experienced. +/// +/// We use this to determine if a modification to the `memory.events` file actually +/// changed the `high` field. If not, we don't care about the change. When we +/// read the file, we check the `high` field in the file against `MEMORY_EVENT_COUNT` +/// to see if it changed since last time. +pub static MEMORY_EVENT_COUNT: AtomicU64 = AtomicU64::new(0); + +/// Monotonically increasing counter that gives each cgroup event a unique id. +/// +/// This allows us to answer questions like "did this upscale arrive before this +/// memory.high?". This static is also used by the `Sequenced` type to "tag" values +/// with a sequence number. As such, prefer to used the `Sequenced` type rather +/// than this static directly. +static EVENT_SEQUENCE_NUMBER: AtomicU64 = AtomicU64::new(0); + +/// A memory event type reported in memory.events. +#[derive(Debug, Eq, PartialEq, Copy, Clone)] +pub enum MemoryEvent { + Low, + High, + Max, + Oom, + OomKill, + OomGroupKill, +} + +impl MemoryEvent { + fn as_str(&self) -> &str { + match self { + MemoryEvent::Low => "low", + MemoryEvent::High => "high", + MemoryEvent::Max => "max", + MemoryEvent::Oom => "oom", + MemoryEvent::OomKill => "oom_kill", + MemoryEvent::OomGroupKill => "oom_group_kill", + } + } +} + +impl Display for MemoryEvent { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(self.as_str()) + } +} + +/// Configuration for a `CgroupWatcher` +#[derive(Debug, Clone)] +pub struct Config { + // The target difference between the total memory reserved for the cgroup + // and the value of the cgroup's memory.high. + // + // In other words, memory.high + oom_buffer_bytes will equal the total memory that the cgroup may + // use (equal to system memory, minus whatever's taken out for the file cache). + oom_buffer_bytes: u64, + + // The amount of memory, in bytes, below a proposed new value for + // memory.high that the cgroup's memory usage must be for us to downscale + // + // In other words, we can downscale only when: + // + // memory.current + memory_high_buffer_bytes < (proposed) memory.high + // + // TODO: there's some minor issues with this approach -- in particular, that we might have + // memory in use by the kernel's page cache that we're actually ok with getting rid of. + pub(crate) memory_high_buffer_bytes: u64, + + // The maximum duration, in milliseconds, that we're allowed to pause + // the cgroup for while waiting for the autoscaler-agent to upscale us + max_upscale_wait: Duration, + + // The required minimum time, in milliseconds, that we must wait before re-freezing + // the cgroup while waiting for the autoscaler-agent to upscale us. + do_not_freeze_more_often_than: Duration, + + // The amount of memory, in bytes, that we should periodically increase memory.high + // by while waiting for the autoscaler-agent to upscale us. + // + // This exists to avoid the excessive throttling that happens when a cgroup is above its + // memory.high for too long. See more here: + // https://github.com/neondatabase/autoscaling/issues/44#issuecomment-1522487217 + memory_high_increase_by_bytes: u64, + + // The period, in milliseconds, at which we should repeatedly increase the value + // of the cgroup's memory.high while we're waiting on upscaling and memory.high + // is still being hit. + // + // Technically speaking, this actually serves as a rate limit to moderate responding to + // memory.high events, but these are roughly equivalent if the process is still allocating + // memory. + memory_high_increase_every: Duration, +} + +impl Config { + /// Calculate the new value for the cgroups memory.high based on system memory + pub fn calculate_memory_high_value(&self, total_system_mem: u64) -> u64 { + total_system_mem.saturating_sub(self.oom_buffer_bytes) + } +} + +impl Default for Config { + fn default() -> Self { + Self { + oom_buffer_bytes: 100 * MiB, + memory_high_buffer_bytes: 100 * MiB, + // while waiting for upscale, don't freeze for more than 20ms every 1s + max_upscale_wait: Duration::from_millis(20), + do_not_freeze_more_often_than: Duration::from_millis(1000), + // while waiting for upscale, increase memory.high by 10MiB every 25ms + memory_high_increase_by_bytes: 10 * MiB, + memory_high_increase_every: Duration::from_millis(25), + } + } +} + +/// Used to represent data that is associated with a certain point in time, such +/// as an upscale request or memory.high event. +/// +/// Internally, creating a `Sequenced` uses a static atomic counter to obtain +/// a unique sequence number. Sequence numbers are monotonically increasing, +/// allowing us to answer questions like "did this upscale happen after this +/// memory.high event?" by comparing the sequence numbers of the two events. +#[derive(Debug, Clone)] +pub struct Sequenced { + seqnum: u64, + data: T, +} + +impl Sequenced { + pub fn new(data: T) -> Self { + Self { + seqnum: EVENT_SEQUENCE_NUMBER.fetch_add(1, Ordering::AcqRel), + data, + } + } +} + +/// Responds to `MonitorEvents` to manage the cgroup: preventing it from being +/// OOM killed or throttling. +/// +/// The `CgroupWatcher` primarily achieves this by reading from a stream of +/// `MonitorEvent`s. See `main_signals_loop` for details on how to keep the +/// cgroup happy. +#[derive(Debug)] +pub struct CgroupWatcher { + pub config: Config, + + /// The sequence number of the last upscale. + /// + /// If we receive a memory.high event that has a _lower_ sequence number than + /// `last_upscale_seqnum`, then we know it occured before the upscale, and we + /// can safely ignore it. + /// + /// Note: Like the `events` field, this doesn't _need_ interior mutability but we + /// use it anyways so that methods take `&self`, not `&mut self`. + last_upscale_seqnum: AtomicU64, + + /// A channel on which we send messages to request upscale from the dispatcher. + upscale_requester: mpsc::Sender<()>, + + /// The actual cgroup we are watching and managing. + cgroup: cgroups_rs::Cgroup, +} + +/// Read memory.events for the desired event type. +/// +/// `path` specifies the path to the desired `memory.events` file. +/// For more info, see the `memory.events` section of the [kernel docs] +/// +fn get_event_count(path: &str, event: MemoryEvent) -> anyhow::Result { + let contents = fs::read_to_string(path) + .with_context(|| format!("failed to read memory.events from {path}"))?; + + // Then contents of the file look like: + // low 42 + // high 101 + // ... + contents + .lines() + .filter_map(|s| s.split_once(' ')) + .find(|(e, _)| *e == event.as_str()) + .ok_or_else(|| anyhow!("failed to find entry for memory.{event} events in {path}")) + .and_then(|(_, count)| { + count + .parse::() + .with_context(|| format!("failed to parse memory.{event} as u64")) + }) +} + +/// Create an event stream that produces events whenever the file at the provided +/// path is modified. +fn create_file_watcher(path: &str) -> anyhow::Result> { + info!("creating file watcher for {path}"); + let inotify = Inotify::init().context("failed to initialize file watcher")?; + inotify + .watches() + .add(path, WatchMask::MODIFY) + .with_context(|| format!("failed to start watching {path}"))?; + inotify + // The inotify docs use [0u8; 1024] so we'll just copy them. We only need + // to store one event at a time - if the event gets written over, that's + // ok. We still see that there is an event. For more information, see: + // https://man7.org/linux/man-pages/man7/inotify.7.html + .into_event_stream([0u8; 1024]) + .context("failed to start inotify event stream") +} + +impl CgroupWatcher { + /// Create a new `CgroupWatcher`. + #[tracing::instrument(skip_all, fields(%name))] + pub fn new( + name: String, + // A channel on which to send upscale requests + upscale_requester: mpsc::Sender<()>, + ) -> anyhow::Result<(Self, impl Stream>)> { + // TODO: clarify exactly why we need v2 + // Make sure cgroups v2 (aka unified) are supported + if !is_cgroup2_unified_mode() { + anyhow::bail!("cgroups v2 not supported"); + } + let cgroup = cgroups_rs::Cgroup::load(hierarchies::auto(), &name); + + // Start monitoring the cgroup for memory events. In general, for + // cgroups v2 (aka unified), metrics are reported in files like + // > `/sys/fs/cgroup/{name}/{metric}` + // We are looking for `memory.high` events, which are stored in the + // file `memory.events`. For more info, see the `memory.events` section + // of https://docs.kernel.org/admin-guide/cgroup-v2.html#memory-interface-files + let path = format!("{}/{}/memory.events", UNIFIED_MOUNTPOINT, &name); + let memory_events = create_file_watcher(&path) + .with_context(|| format!("failed to create event watcher for {path}"))? + // This would be nice with with .inspect_err followed by .ok + .filter_map(move |_| match get_event_count(&path, MemoryEvent::High) { + Ok(high) => Some(high), + Err(error) => { + // TODO: Might want to just panic here + warn!(?error, "failed to read high events count from {}", &path); + None + } + }) + // Only report the event if the memory.high count increased + .filter_map(|high| { + if MEMORY_EVENT_COUNT.fetch_max(high, Ordering::AcqRel) < high { + Some(high) + } else { + None + } + }) + .map(Sequenced::new); + + let initial_count = get_event_count( + &format!("{}/{}/memory.events", UNIFIED_MOUNTPOINT, &name), + MemoryEvent::High, + )?; + + info!(initial_count, "initial memory.high event count"); + + // Hard update `MEMORY_EVENT_COUNT` since there could have been processes + // running in the cgroup before that caused it to be non-zero. + MEMORY_EVENT_COUNT.fetch_max(initial_count, Ordering::AcqRel); + + Ok(( + Self { + cgroup, + upscale_requester, + last_upscale_seqnum: AtomicU64::new(0), + config: Default::default(), + }, + memory_events, + )) + } + + /// The entrypoint for the `CgroupWatcher`. + #[tracing::instrument(skip_all)] + pub async fn watch( + &self, + // These are ~dependency injected~ (fancy, I know) because this function + // should never return. + // -> therefore: when we tokio::spawn it, we don't await the JoinHandle. + // -> therefore: if we want to stick it in an Arc so many threads can access + // it, methods can never take mutable access. + // - note: we use the Arc strategy so that a) we can call this function + // right here and b) the runner can call the set/get_memory methods + // -> since calling recv() on a tokio::sync::mpsc::Receiver takes &mut self, + // we just pass them in here instead of holding them in fields, as that + // would require this method to take &mut self. + mut upscales: mpsc::Receiver>, + events: E, + ) -> anyhow::Result<()> + where + E: Stream>, + { + // There are several actions might do when receiving a `memory.high`, + // such as freezing the cgroup, or increasing its `memory.high`. We don't + // want to do these things too often (because postgres needs to run, and + // we only have so much memory). These timers serve as rate limits for this. + let mut wait_to_freeze = pin!(tokio::time::sleep(Duration::ZERO)); + let mut wait_to_increase_memory_high = pin!(tokio::time::sleep(Duration::ZERO)); + let mut events = pin!(events); + + // Are we waiting to be upscaled? Could be true if we request upscale due + // to a memory.high event and it does not arrive in time. + let mut waiting_on_upscale = false; + + loop { + tokio::select! { + upscale = upscales.recv() => { + let Sequenced { seqnum, data } = upscale + .context("failed to listen on upscale notification channel")?; + self.last_upscale_seqnum.store(seqnum, Ordering::Release); + info!(cpu = data.cpu, mem_bytes = data.mem, "received upscale"); + } + event = events.next() => { + let Some(Sequenced { seqnum, .. }) = event else { + bail!("failed to listen for memory.high events") + }; + // The memory.high came before our last upscale, so we consider + // it resolved + if self.last_upscale_seqnum.fetch_max(seqnum, Ordering::AcqRel) > seqnum { + info!( + "received memory.high event, but it came before our last upscale -> ignoring it" + ); + continue; + } + + // The memory.high came after our latest upscale. We don't + // want to do anything yet, so peek the next event in hopes + // that it's an upscale. + if let Some(upscale_num) = self + .upscaled(&mut upscales) + .context("failed to check if we were upscaled")? + { + if upscale_num > seqnum { + info!( + "received memory.high event, but it came before our last upscale -> ignoring it" + ); + continue; + } + } + + // If it's been long enough since we last froze, freeze the + // cgroup and request upscale + if wait_to_freeze.is_elapsed() { + info!("received memory.high event -> requesting upscale"); + waiting_on_upscale = self + .handle_memory_high_event(&mut upscales) + .await + .context("failed to handle upscale")?; + wait_to_freeze + .as_mut() + .reset(Instant::now() + self.config.do_not_freeze_more_often_than); + continue; + } + + // Ok, we can't freeze, just request upscale + if !waiting_on_upscale { + info!("received memory.high event, but too soon to refreeze -> requesting upscale"); + + // Make check to make sure we haven't been upscaled in the + // meantine (can happen if the agent independently decides + // to upscale us again) + if self + .upscaled(&mut upscales) + .context("failed to check if we were upscaled")? + .is_some() + { + info!("no need to request upscaling because we got upscaled"); + continue; + } + self.upscale_requester + .send(()) + .await + .context("failed to request upscale")?; + continue; + } + + // Shoot, we can't freeze or and we're still waiting on upscale, + // increase memory.high to reduce throttling + if wait_to_increase_memory_high.is_elapsed() { + info!( + "received memory.high event, \ + but too soon to refreeze and already requested upscale \ + -> increasing memory.high" + ); + + // Make check to make sure we haven't been upscaled in the + // meantine (can happen if the agent independently decides + // to upscale us again) + if self + .upscaled(&mut upscales) + .context("failed to check if we were upscaled")? + .is_some() + { + info!("no need to increase memory.high because got upscaled"); + continue; + } + + // Request upscale anyways (the agent will handle deduplicating + // requests) + self.upscale_requester + .send(()) + .await + .context("failed to request upscale")?; + + let memory_high = + self.get_high_bytes().context("failed to get memory.high")?; + let new_high = memory_high + self.config.memory_high_increase_by_bytes; + info!( + current_high_bytes = memory_high, + new_high_bytes = new_high, + "updating memory.high" + ); + self.set_high_bytes(new_high) + .context("failed to set memory.high")?; + wait_to_increase_memory_high + .as_mut() + .reset(Instant::now() + self.config.memory_high_increase_every) + } + + // we can't do anything + } + }; + } + } + + /// Handle a `memory.high`, returning whether we are still waiting on upscale + /// by the time the function returns. + /// + /// The general plan for handling a `memory.high` event is as follows: + /// 1. Freeze the cgroup + /// 2. Start a timer for `self.config.max_upscale_wait` + /// 3. Request upscale + /// 4. After the timer elapses or we receive upscale, thaw the cgroup. + /// 5. Return whether or not we are still waiting for upscale. If we are, + /// we'll increase the cgroups memory.high to avoid getting oom killed + #[tracing::instrument(skip_all)] + async fn handle_memory_high_event( + &self, + upscales: &mut mpsc::Receiver>, + ) -> anyhow::Result { + // Immediately freeze the cgroup before doing anything else. + info!("received memory.high event -> freezing cgroup"); + self.freeze().context("failed to freeze cgroup")?; + + // We'll use this for logging durations + let start_time = Instant::now(); + + // Await the upscale until we have to unfreeze + let timed = + tokio::time::timeout(self.config.max_upscale_wait, self.await_upscale(upscales)); + + // Request the upscale + info!( + wait = ?self.config.max_upscale_wait, + "sending request for immediate upscaling", + ); + self.upscale_requester + .send(()) + .await + .context("failed to request upscale")?; + + let waiting_on_upscale = match timed.await { + Ok(Ok(())) => { + info!(elapsed = ?start_time.elapsed(), "received upscale in time"); + false + } + // **important**: unfreeze the cgroup before ?-reporting the error + Ok(Err(e)) => { + info!("error waiting for upscale -> thawing cgroup"); + self.thaw() + .context("failed to thaw cgroup after errored waiting for upscale")?; + Err(e.context("failed to await upscale"))? + } + Err(_) => { + info!(elapsed = ?self.config.max_upscale_wait, "timed out waiting for upscale"); + true + } + }; + + info!("thawing cgroup"); + self.thaw().context("failed to thaw cgroup")?; + + Ok(waiting_on_upscale) + } + + /// Checks whether we were just upscaled, returning the upscale's sequence + /// number if so. + #[tracing::instrument(skip_all)] + fn upscaled( + &self, + upscales: &mut mpsc::Receiver>, + ) -> anyhow::Result> { + let Sequenced { seqnum, data } = match upscales.try_recv() { + Ok(upscale) => upscale, + Err(TryRecvError::Empty) => return Ok(None), + Err(TryRecvError::Disconnected) => { + bail!("upscale notification channel was disconnected") + } + }; + + // Make sure to update the last upscale sequence number + self.last_upscale_seqnum.store(seqnum, Ordering::Release); + info!(cpu = data.cpu, mem_bytes = data.mem, "received upscale"); + Ok(Some(seqnum)) + } + + /// Await an upscale event, discarding any `memory.high` events received in + /// the process. + /// + /// This is used in `handle_memory_high_event`, where we need to listen + /// for upscales in particular so we know if we can thaw the cgroup early. + #[tracing::instrument(skip_all)] + async fn await_upscale( + &self, + upscales: &mut mpsc::Receiver>, + ) -> anyhow::Result<()> { + let Sequenced { seqnum, .. } = upscales + .recv() + .await + .context("error listening for upscales")?; + + self.last_upscale_seqnum.store(seqnum, Ordering::Release); + Ok(()) + } + + /// Get the cgroup's name. + pub fn path(&self) -> &str { + self.cgroup.path() + } +} + +/// Represents a set of limits we apply to a cgroup to control memory usage. +/// +/// Setting these values also affects the thresholds for receiving usage alerts. +#[derive(Debug)] +pub struct MemoryLimits { + high: u64, + max: u64, +} + +impl MemoryLimits { + pub fn new(high: u64, max: u64) -> Self { + Self { max, high } + } +} + +// Methods for manipulating the actual cgroup +impl CgroupWatcher { + /// Get a handle on the freezer subsystem. + fn freezer(&self) -> anyhow::Result<&FreezerController> { + if let Some(Freezer(freezer)) = self + .cgroup + .subsystems() + .iter() + .find(|sub| matches!(sub, Freezer(_))) + { + Ok(freezer) + } else { + anyhow::bail!("could not find freezer subsystem") + } + } + + /// Attempt to freeze the cgroup. + pub fn freeze(&self) -> anyhow::Result<()> { + self.freezer() + .context("failed to get freezer subsystem")? + .freeze() + .context("failed to freeze") + } + + /// Attempt to thaw the cgroup. + pub fn thaw(&self) -> anyhow::Result<()> { + self.freezer() + .context("failed to get freezer subsystem")? + .thaw() + .context("failed to thaw") + } + + /// Get a handle on the memory subsystem. + /// + /// Note: this method does not require `self.memory_update_lock` because + /// getting a handle to the subsystem does not access any of the files we + /// care about, such as memory.high and memory.events + fn memory(&self) -> anyhow::Result<&MemController> { + if let Some(Mem(memory)) = self + .cgroup + .subsystems() + .iter() + .find(|sub| matches!(sub, Mem(_))) + { + Ok(memory) + } else { + anyhow::bail!("could not find memory subsystem") + } + } + + /// Get cgroup current memory usage. + pub fn current_memory_usage(&self) -> anyhow::Result { + Ok(self + .memory() + .context("failed to get memory subsystem")? + .memory_stat() + .usage_in_bytes) + } + + /// Set cgroup memory.high threshold. + pub fn set_high_bytes(&self, bytes: u64) -> anyhow::Result<()> { + self.memory() + .context("failed to get memory subsystem")? + .set_mem(cgroups_rs::memory::SetMemory { + low: None, + high: Some(MaxValue::Value(u64::min(bytes, i64::MAX as u64) as i64)), + min: None, + max: None, + }) + .context("failed to set memory.high") + } + + /// Set cgroup memory.high and memory.max. + pub fn set_limits(&self, limits: &MemoryLimits) -> anyhow::Result<()> { + info!( + limits.high, + limits.max, + path = self.path(), + "writing new memory limits", + ); + self.memory() + .context("failed to get memory subsystem while setting memory limits")? + .set_mem(cgroups_rs::memory::SetMemory { + min: None, + low: None, + high: Some(MaxValue::Value( + u64::min(limits.high, i64::MAX as u64) as i64 + )), + max: Some(MaxValue::Value(u64::min(limits.max, i64::MAX as u64) as i64)), + }) + .context("failed to set memory limits") + } + + /// Given some amount of available memory, set the desired cgroup memory limits + pub fn set_memory_limits(&mut self, available_memory: u64) -> anyhow::Result<()> { + let new_high = self.config.calculate_memory_high_value(available_memory); + let limits = MemoryLimits::new(new_high, available_memory); + info!( + path = self.path(), + memory = ?limits, + "setting cgroup memory", + ); + self.set_limits(&limits) + .context("failed to set cgroup memory limits")?; + Ok(()) + } + + /// Get memory.high threshold. + pub fn get_high_bytes(&self) -> anyhow::Result { + let high = self + .memory() + .context("failed to get memory subsystem while getting memory statistics")? + .get_mem() + .map(|mem| mem.high) + .context("failed to get memory statistics from subsystem")?; + match high { + Some(MaxValue::Max) => Ok(i64::MAX as u64), + Some(MaxValue::Value(high)) => Ok(high as u64), + None => anyhow::bail!("failed to read memory.high from memory subsystem"), + } + } +} diff --git a/libs/vm_monitor/src/dispatcher.rs b/libs/vm_monitor/src/dispatcher.rs new file mode 100644 index 0000000000..109a68fff1 --- /dev/null +++ b/libs/vm_monitor/src/dispatcher.rs @@ -0,0 +1,153 @@ +//! Managing the websocket connection and other signals in the monitor. +//! +//! Contains types that manage the interaction (not data interchange, see `protocol`) +//! between agent and monitor, allowing us to to process and send messages in a +//! straightforward way. The dispatcher also manages that signals that come from +//! the cgroup (requesting upscale), and the signals that go to the cgroup +//! (notifying it of upscale). + +use anyhow::{bail, Context}; +use axum::extract::ws::{Message, WebSocket}; +use futures::{ + stream::{SplitSink, SplitStream}, + SinkExt, StreamExt, +}; +use tokio::sync::mpsc; +use tracing::info; + +use crate::cgroup::Sequenced; +use crate::protocol::{ + OutboundMsg, ProtocolRange, ProtocolResponse, ProtocolVersion, Resources, PROTOCOL_MAX_VERSION, + PROTOCOL_MIN_VERSION, +}; + +/// The central handler for all communications in the monitor. +/// +/// The dispatcher has two purposes: +/// 1. Manage the connection to the agent, sending and receiving messages. +/// 2. Communicate with the cgroup manager, notifying it when upscale is received, +/// and sending a message to the agent when the cgroup manager requests +/// upscale. +#[derive(Debug)] +pub struct Dispatcher { + /// We read agent messages of of `source` + pub(crate) source: SplitStream, + + /// We send messages to the agent through `sink` + sink: SplitSink, + + /// Used to notify the cgroup when we are upscaled. + pub(crate) notify_upscale_events: mpsc::Sender>, + + /// When the cgroup requests upscale it will send on this channel. In response + /// we send an `UpscaleRequst` to the agent. + pub(crate) request_upscale_events: mpsc::Receiver<()>, + + /// The protocol version we have agreed to use with the agent. This is negotiated + /// during the creation of the dispatcher, and should be the highest shared protocol + /// version. + /// + // NOTE: currently unused, but will almost certainly be used in the futures + // as the protocol changes + #[allow(unused)] + pub(crate) proto_version: ProtocolVersion, +} + +impl Dispatcher { + /// Creates a new dispatcher using the passed-in connection. + /// + /// Performs a negotiation with the agent to determine the highest protocol + /// version that both support. This consists of two steps: + /// 1. Wait for the agent to sent the range of protocols it supports. + /// 2. Send a protocol version that works for us as well, or an error if there + /// is no compatible version. + pub async fn new( + stream: WebSocket, + notify_upscale_events: mpsc::Sender>, + request_upscale_events: mpsc::Receiver<()>, + ) -> anyhow::Result { + let (mut sink, mut source) = stream.split(); + + // Figure out the highest protocol version we both support + info!("waiting for agent to send protocol version range"); + let Some(message) = source.next().await else { + bail!("websocket connection closed while performing protocol handshake") + }; + + let message = message.context("failed to read protocol version range off connection")?; + + let Message::Text(message_text) = message else { + // All messages should be in text form, since we don't do any + // pinging/ponging. See nhooyr/websocket's implementation and the + // agent for more info + bail!("received non-text message during proocol handshake: {message:?}") + }; + + let monitor_range = ProtocolRange { + min: PROTOCOL_MIN_VERSION, + max: PROTOCOL_MAX_VERSION, + }; + + let agent_range: ProtocolRange = serde_json::from_str(&message_text) + .context("failed to deserialize protocol version range")?; + + info!(range = ?agent_range, "received protocol version range"); + + let highest_shared_version = match monitor_range.highest_shared_version(&agent_range) { + Ok(version) => { + sink.send(Message::Text( + serde_json::to_string(&ProtocolResponse::Version(version)).unwrap(), + )) + .await + .context("failed to notify agent of negotiated protocol version")?; + version + } + Err(e) => { + sink.send(Message::Text( + serde_json::to_string(&ProtocolResponse::Error(format!( + "Received protocol version range {} which does not overlap with {}", + agent_range, monitor_range + ))) + .unwrap(), + )) + .await + .context("failed to notify agent of no overlap between protocol version ranges")?; + Err(e).context("error determining suitable protocol version range")? + } + }; + + Ok(Self { + sink, + source, + notify_upscale_events, + request_upscale_events, + proto_version: highest_shared_version, + }) + } + + /// Notify the cgroup manager that we have received upscale and wait for + /// the acknowledgement. + #[tracing::instrument(skip_all, fields(?resources))] + pub async fn notify_upscale(&self, resources: Sequenced) -> anyhow::Result<()> { + self.notify_upscale_events + .send(resources) + .await + .context("failed to send resources and oneshot sender across channel") + } + + /// Send a message to the agent. + /// + /// Although this function is small, it has one major benefit: it is the only + /// way to send data accross the connection, and you can only pass in a proper + /// `MonitorMessage`. Without safeguards like this, it's easy to accidentally + /// serialize the wrong thing and send it, since `self.sink.send` will take + /// any string. + pub async fn send(&mut self, message: OutboundMsg) -> anyhow::Result<()> { + info!(?message, "sending message"); + let json = serde_json::to_string(&message).context("failed to serialize message")?; + self.sink + .send(Message::Text(json)) + .await + .context("stream error sending message") + } +} diff --git a/libs/vm_monitor/src/filecache.rs b/libs/vm_monitor/src/filecache.rs new file mode 100644 index 0000000000..3a860500f1 --- /dev/null +++ b/libs/vm_monitor/src/filecache.rs @@ -0,0 +1,316 @@ +//! Logic for configuring and scaling the Postgres file cache. + +use std::num::NonZeroU64; + +use crate::MiB; +use anyhow::{anyhow, Context}; +use tokio_postgres::{types::ToSql, Client, NoTls, Row}; +use tokio_util::sync::CancellationToken; +use tracing::{error, info}; + +/// Manages Postgres' file cache by keeping a connection open. +#[derive(Debug)] +pub struct FileCacheState { + client: Client, + conn_str: String, + pub(crate) config: FileCacheConfig, + + /// A token for cancelling spawned threads during shutdown. + token: CancellationToken, +} + +#[derive(Debug)] +pub struct FileCacheConfig { + /// Whether the file cache is *actually* stored in memory (e.g. by writing to + /// a tmpfs or shmem file). If true, the size of the file cache will be counted against the + /// memory available for the cgroup. + pub(crate) in_memory: bool, + + /// The size of the file cache, in terms of the size of the resource it consumes + /// (currently: only memory) + /// + /// For example, setting `resource_multipler = 0.75` gives the cache a target size of 75% of total + /// resources. + /// + /// This value must be strictly between 0 and 1. + resource_multiplier: f64, + + /// The required minimum amount of memory, in bytes, that must remain available + /// after subtracting the file cache. + /// + /// This value must be non-zero. + min_remaining_after_cache: NonZeroU64, + + /// Controls the rate of increase in the file cache's size as it grows from zero + /// (when total resources equals min_remaining_after_cache) to the desired size based on + /// `resource_multiplier`. + /// + /// A `spread_factor` of zero means that all additional resources will go to the cache until it + /// reaches the desired size. Setting `spread_factor` to N roughly means "for every 1 byte added to + /// the cache's size, N bytes are reserved for the rest of the system, until the cache gets to + /// its desired size". + /// + /// This value must be >= 0, and must retain an increase that is more than what would be given by + /// `resource_multiplier`. For example, setting `resource_multiplier` = 0.75 but `spread_factor` = 1 + /// would be invalid, because `spread_factor` would induce only 50% usage - never reaching the 75% + /// as desired by `resource_multiplier`. + /// + /// `spread_factor` is too large if `(spread_factor + 1) * resource_multiplier >= 1`. + spread_factor: f64, +} + +impl FileCacheConfig { + pub fn default_in_memory() -> Self { + Self { + in_memory: true, + // 75 % + resource_multiplier: 0.75, + // 640 MiB; (512 + 128) + min_remaining_after_cache: NonZeroU64::new(640 * MiB).unwrap(), + // ensure any increase in file cache size is split 90-10 with 10% to other memory + spread_factor: 0.1, + } + } + + pub fn default_on_disk() -> Self { + Self { + in_memory: false, + resource_multiplier: 0.75, + // 256 MiB - lower than when in memory because overcommitting is safe; if we don't have + // memory, the kernel will just evict from its page cache, rather than e.g. killing + // everything. + min_remaining_after_cache: NonZeroU64::new(256 * MiB).unwrap(), + spread_factor: 0.1, + } + } + + /// Make sure fields of the config are consistent. + pub fn validate(&self) -> anyhow::Result<()> { + // Single field validity + anyhow::ensure!( + 0.0 < self.resource_multiplier && self.resource_multiplier < 1.0, + "resource_multiplier must be between 0.0 and 1.0 exclusive, got {}", + self.resource_multiplier + ); + anyhow::ensure!( + self.spread_factor >= 0.0, + "spread_factor must be >= 0, got {}", + self.spread_factor + ); + + // Check that `resource_multiplier` and `spread_factor` are valid w.r.t. each other. + // + // As shown in `calculate_cache_size`, we have two lines resulting from `resource_multiplier` and + // `spread_factor`, respectively. They are: + // + // `total` `min_remaining_after_cache` + // size = ————————————————————— - ————————————————————————————— + // `spread_factor` + 1 `spread_factor` + 1 + // + // and + // + // size = `resource_multiplier` × total + // + // .. where `total` is the total resources. These are isomorphic to the typical 'y = mx + b' + // form, with y = "size" and x = "total". + // + // These lines intersect at: + // + // `min_remaining_after_cache` + // ——————————————————————————————————————————————————— + // 1 - `resource_multiplier` × (`spread_factor` + 1) + // + // We want to ensure that this value (a) exists, and (b) is >= `min_remaining_after_cache`. This is + // guaranteed when '`resource_multiplier` × (`spread_factor` + 1)' is less than 1. + // (We also need it to be >= 0, but that's already guaranteed.) + + let intersect_factor = self.resource_multiplier * (self.spread_factor + 1.0); + anyhow::ensure!( + intersect_factor < 1.0, + "incompatible resource_multipler and spread_factor" + ); + Ok(()) + } + + /// Calculate the desired size of the cache, given the total memory + pub fn calculate_cache_size(&self, total: u64) -> u64 { + // *Note*: all units are in bytes, until the very last line. + let available = total.saturating_sub(self.min_remaining_after_cache.get()); + if available == 0 { + return 0; + } + + // Conversions to ensure we don't overflow from floating-point ops + let size_from_spread = + i64::max(0, (available as f64 / (1.0 + self.spread_factor)) as i64) as u64; + + let size_from_normal = (total as f64 * self.resource_multiplier) as u64; + + let byte_size = u64::min(size_from_spread, size_from_normal); + + // The file cache operates in units of mebibytes, so the sizes we produce should + // be rounded to a mebibyte. We round down to be conservative. + byte_size / MiB * MiB + } +} + +impl FileCacheState { + /// Connect to the file cache. + #[tracing::instrument(skip_all, fields(%conn_str, ?config))] + pub async fn new( + conn_str: &str, + config: FileCacheConfig, + token: CancellationToken, + ) -> anyhow::Result { + config.validate().context("file cache config is invalid")?; + + info!(conn_str, "connecting to Postgres file cache"); + let client = FileCacheState::connect(conn_str, token.clone()) + .await + .context("failed to connect to postgres file cache")?; + + let conn_str = conn_str.to_string(); + Ok(Self { + client, + config, + conn_str, + token, + }) + } + + /// Connect to Postgres. + /// + /// Aborts the spawned thread if the kill signal is received. This is not + /// a method as it is called in [`FileCacheState::new`]. + #[tracing::instrument(skip_all, fields(%conn_str))] + async fn connect(conn_str: &str, token: CancellationToken) -> anyhow::Result { + let (client, conn) = tokio_postgres::connect(conn_str, NoTls) + .await + .context("failed to connect to pg client")?; + + // The connection object performs the actual communication with the database, + // so spawn it off to run on its own. See tokio-postgres docs. + crate::spawn_with_cancel( + token, + |res| { + if let Err(error) = res { + error!(%error, "postgres error") + } + }, + conn, + ); + + Ok(client) + } + + /// Execute a query with a retry if necessary. + /// + /// If the initial query fails, we restart the database connection and attempt + /// if again. + #[tracing::instrument(skip_all, fields(%statement))] + pub async fn query_with_retry( + &mut self, + statement: &str, + params: &[&(dyn ToSql + Sync)], + ) -> anyhow::Result> { + match self + .client + .query(statement, params) + .await + .context("failed to execute query") + { + Ok(rows) => Ok(rows), + Err(e) => { + error!(error = ?e, "postgres error: {e} -> retrying"); + + let client = FileCacheState::connect(&self.conn_str, self.token.clone()) + .await + .context("failed to connect to postgres file cache")?; + info!("successfully reconnected to postgres client"); + + // Replace the old client and attempt the query with the new one + self.client = client; + self.client + .query(statement, params) + .await + .context("failed to execute query a second time") + } + } + } + + /// Get the current size of the file cache. + #[tracing::instrument(skip_all)] + pub async fn get_file_cache_size(&mut self) -> anyhow::Result { + self.query_with_retry( + // The file cache GUC variable is in MiB, but the conversion with + // pg_size_bytes means that the end result we get is in bytes. + "SELECT pg_size_bytes(current_setting('neon.file_cache_size_limit'));", + &[], + ) + .await + .context("failed to query pg for file cache size")? + .first() + .ok_or_else(|| anyhow!("file cache size query returned no rows"))? + // pg_size_bytes returns a bigint which is the same as an i64. + .try_get::<_, i64>(0) + // Since the size of the table is not negative, the cast is sound. + .map(|bytes| bytes as u64) + .context("failed to extract file cache size from query result") + } + + /// Attempt to set the file cache size, returning the size it was actually + /// set to. + #[tracing::instrument(skip_all, fields(%num_bytes))] + pub async fn set_file_cache_size(&mut self, num_bytes: u64) -> anyhow::Result { + let max_bytes = self + // The file cache GUC variable is in MiB, but the conversion with pg_size_bytes + // means that the end result we get is in bytes. + .query_with_retry( + "SELECT pg_size_bytes(current_setting('neon.max_file_cache_size'));", + &[], + ) + .await + .context("failed to query pg for max file cache size")? + .first() + .ok_or_else(|| anyhow!("max file cache size query returned no rows"))? + .try_get::<_, i64>(0) + .map(|bytes| bytes as u64) + .context("failed to extract max file cache size from query result")?; + + let max_mb = max_bytes / MiB; + let num_mb = u64::min(num_bytes, max_bytes) / MiB; + + let capped = if num_bytes > max_bytes { + " (capped by maximum size)" + } else { + "" + }; + + info!( + size = num_mb, + max = max_mb, + "updating file cache size {capped}", + ); + + // note: even though the normal ways to get the cache size produce values with trailing "MB" + // (hence why we call pg_size_bytes in `get_file_cache_size`'s query), the format + // it expects to set the value is "integer number of MB" without trailing units. + // For some reason, this *really* wasn't working with normal arguments, so that's + // why we're constructing the query here. + self.client + .query( + &format!("ALTER SYSTEM SET neon.file_cache_size_limit = {};", num_mb), + &[], + ) + .await + .context("failed to change file cache size limit")?; + + // must use pg_reload_conf to have the settings change take effect + self.client + .execute("SELECT pg_reload_conf();", &[]) + .await + .context("failed to reload config")?; + + Ok(num_mb * MiB) + } +} diff --git a/libs/vm_monitor/src/lib.rs b/libs/vm_monitor/src/lib.rs new file mode 100644 index 0000000000..2eee31c965 --- /dev/null +++ b/libs/vm_monitor/src/lib.rs @@ -0,0 +1,215 @@ +#![cfg(target_os = "linux")] + +use anyhow::Context; +use axum::{ + extract::{ws::WebSocket, State, WebSocketUpgrade}, + response::Response, +}; +use axum::{routing::get, Router, Server}; +use clap::Parser; +use futures::Future; +use std::{fmt::Debug, time::Duration}; +use sysinfo::{RefreshKind, System, SystemExt}; +use tokio::{sync::broadcast, task::JoinHandle}; +use tokio_util::sync::CancellationToken; +use tracing::{error, info}; + +use runner::Runner; + +// Code that interfaces with agent +pub mod dispatcher; +pub mod protocol; + +pub mod cgroup; +pub mod filecache; +pub mod runner; + +/// The vm-monitor is an autoscaling component started by compute_ctl. +/// +/// It carries out autoscaling decisions (upscaling/downscaling) and responds to +/// memory pressure by making requests to the autoscaler-agent. +#[derive(Debug, Parser)] +pub struct Args { + /// The name of the cgroup we should monitor for memory.high events. This + /// is the cgroup that postgres should be running in. + #[arg(short, long)] + pub cgroup: Option, + + /// The connection string for the Postgres file cache we should manage. + #[arg(short, long)] + pub pgconnstr: Option, + + /// Flag to signal that the Postgres file cache is on disk (i.e. not in memory aside from the + /// kernel's page cache), and therefore should not count against available memory. + // + // NB: Ideally this flag would directly refer to whether the file cache is in memory (rather + // than a roundabout way, via whether it's on disk), but in order to be backwards compatible + // during the switch away from an in-memory file cache, we had to default to the previous + // behavior. + #[arg(long)] + pub file_cache_on_disk: bool, + + /// The address we should listen on for connection requests. For the + /// agent, this is 0.0.0.0:10301. For the informant, this is 127.0.0.1:10369. + #[arg(short, long)] + pub addr: String, +} + +impl Args { + pub fn addr(&self) -> &str { + &self.addr + } +} + +/// The number of bytes in one mebibyte. +#[allow(non_upper_case_globals)] +const MiB: u64 = 1 << 20; + +/// Convert a quantity in bytes to a quantity in mebibytes, generally for display +/// purposes. (Most calculations in this crate use bytes directly) +pub fn bytes_to_mebibytes(bytes: u64) -> f32 { + (bytes as f32) / (MiB as f32) +} + +pub fn get_total_system_memory() -> u64 { + System::new_with_specifics(RefreshKind::new().with_memory()).total_memory() +} + +/// Global app state for the Axum server +#[derive(Debug, Clone)] +pub struct ServerState { + /// Used to close old connections. + /// + /// When a new connection is made, we send a message signalling to the old + /// connection to close. + pub sender: broadcast::Sender<()>, + + /// Used to cancel all spawned threads in the monitor. + pub token: CancellationToken, + + // The CLI args + pub args: &'static Args, +} + +/// Spawn a thread that may get cancelled by the provided [`CancellationToken`]. +/// +/// This is mainly meant to be called with futures that will be pending for a very +/// long time, or are not mean to return. If it is not desirable for the future to +/// ever resolve, such as in the case of [`cgroup::CgroupWatcher::watch`], the error can +/// be logged with `f`. +pub fn spawn_with_cancel( + token: CancellationToken, + f: F, + future: T, +) -> JoinHandle> +where + T: Future + Send + 'static, + T::Output: Send + 'static, + F: FnOnce(&T::Output) + Send + 'static, +{ + tokio::spawn(async move { + tokio::select! { + _ = token.cancelled() => { + info!("received global kill signal"); + None + } + res = future => { + f(&res); + Some(res) + } + } + }) +} + +/// The entrypoint to the binary. +/// +/// Set up tracing, parse arguments, and start an http server. +pub async fn start(args: &'static Args, token: CancellationToken) -> anyhow::Result<()> { + // This channel is used to close old connections. When a new connection is + // made, we send a message signalling to the old connection to close. + let (sender, _) = tokio::sync::broadcast::channel::<()>(1); + + let app = Router::new() + // This route gets upgraded to a websocket connection. We only support + // one connection at a time, which we enforce by killing old connections + // when we receive a new one. + .route("/monitor", get(ws_handler)) + .with_state(ServerState { + sender, + token, + args, + }); + + let addr = args.addr(); + let bound = Server::try_bind(&addr.parse().expect("parsing address should not fail")) + .with_context(|| format!("failed to bind to {addr}"))?; + + info!(addr, "server bound"); + + bound + .serve(app.into_make_service()) + .await + .context("server exited")?; + + Ok(()) +} + +/// Handles incoming websocket connections. +/// +/// If we are already to connected to an agent, we kill that old connection +/// and accept the new one. +#[tracing::instrument(name = "/monitor", skip_all, fields(?args))] +pub async fn ws_handler( + ws: WebSocketUpgrade, + State(ServerState { + sender, + token, + args, + }): State, +) -> Response { + // Kill the old monitor + info!("closing old connection if there is one"); + let _ = sender.send(()); + + // Start the new one. Wow, the cycle of death and rebirth + let closer = sender.subscribe(); + ws.on_upgrade(|ws| start_monitor(ws, args, closer, token)) +} + +/// Starts the monitor. If startup fails or the monitor exits, an error will +/// be logged and our internal state will be reset to allow for new connections. +#[tracing::instrument(skip_all, fields(?args))] +async fn start_monitor( + ws: WebSocket, + args: &Args, + kill: broadcast::Receiver<()>, + token: CancellationToken, +) { + info!("accepted new websocket connection -> starting monitor"); + let timeout = Duration::from_secs(4); + let monitor = tokio::time::timeout( + timeout, + Runner::new(Default::default(), args, ws, kill, token), + ) + .await; + let mut monitor = match monitor { + Ok(Ok(monitor)) => monitor, + Ok(Err(error)) => { + error!(?error, "failed to create monitor"); + return; + } + Err(_) => { + error!( + ?timeout, + "creating monitor timed out (probably waiting to receive protocol range)" + ); + return; + } + }; + info!("connected to agent"); + + match monitor.run().await { + Ok(()) => info!("monitor was killed due to new connection"), + Err(e) => error!(error = ?e, "monitor terminated unexpectedly"), + } +} diff --git a/libs/vm_monitor/src/protocol.rs b/libs/vm_monitor/src/protocol.rs new file mode 100644 index 0000000000..5f07435503 --- /dev/null +++ b/libs/vm_monitor/src/protocol.rs @@ -0,0 +1,241 @@ +//! Types representing protocols and actual agent-monitor messages. +//! +//! The pervasive use of serde modifiers throughout this module is to ease +//! serialization on the go side. Because go does not have enums (which model +//! messages well), it is harder to model messages, and we accomodate that with +//! serde. +//! +//! *Note*: the agent sends and receives messages in different ways. +//! +//! The agent serializes messages in the form and then sends them. The use +//! of `#[serde(tag = "type", content = "content")]` allows us to use `Type` +//! to determine how to deserialize `Content`. +//! ```ignore +//! struct { +//! Content any +//! Type string +//! Id uint64 +//! } +//! ``` +//! and receives messages in the form: +//! ```ignore +//! struct { +//! {fields embedded} +//! Type string +//! Id uint64 +//! } +//! ``` +//! After reading the type field, the agent will decode the entire message +//! again, this time into the correct type using the embedded fields. +//! Because the agent cannot just extract the json contained in a certain field +//! (it initially deserializes to `map[string]interface{}`), we keep the fields +//! at the top level, so the entire piece of json can be deserialized into a struct, +//! such as a `DownscaleResult`, with the `Type` and `Id` fields ignored. + +use core::fmt; +use std::cmp; + +use serde::{de::Error, Deserialize, Serialize}; + +/// A Message we send to the agent. +#[derive(Serialize, Deserialize, Debug, Clone)] +pub struct OutboundMsg { + #[serde(flatten)] + pub(crate) inner: OutboundMsgKind, + pub(crate) id: usize, +} + +impl OutboundMsg { + pub fn new(inner: OutboundMsgKind, id: usize) -> Self { + Self { inner, id } + } +} + +/// The different underlying message types we can send to the agent. +#[derive(Serialize, Deserialize, Debug, Clone)] +#[serde(tag = "type")] +pub enum OutboundMsgKind { + /// Indicates that the agent sent an invalid message, i.e, we couldn't + /// properly deserialize it. + InvalidMessage { error: String }, + /// Indicates that we experienced an internal error while processing a message. + /// For example, if a cgroup operation fails while trying to handle an upscale, + /// we return `InternalError`. + InternalError { error: String }, + /// Returned to the agent once we have finished handling an upscale. If the + /// handling was unsuccessful, an `InternalError` will get returned instead. + /// *Note*: this is a struct variant because of the way go serializes struct{} + UpscaleConfirmation {}, + /// Indicates to the monitor that we are urgently requesting resources. + /// *Note*: this is a struct variant because of the way go serializes struct{} + UpscaleRequest {}, + /// Returned to the agent once we have finished attempting to downscale. If + /// an error occured trying to do so, an `InternalError` will get returned instead. + /// However, if we are simply unsuccessful (for example, do to needing the resources), + /// that gets included in the `DownscaleResult`. + DownscaleResult { + // FIXME for the future (once the informant is deprecated) + // As of the time of writing, the agent/informant version of this struct is + // called api.DownscaleResult. This struct has uppercase fields which are + // serialized as such. Thus, we serialize using uppercase names so we don't + // have to make a breaking change to the agent<->informant protocol. Once + // the informant has been superseded by the monitor, we can add the correct + // struct tags to api.DownscaleResult without causing a breaking change, + // since we don't need to support the agent<->informant protocol anymore. + #[serde(rename = "Ok")] + ok: bool, + #[serde(rename = "Status")] + status: String, + }, + /// Part of the bidirectional heartbeat. The heartbeat is initiated by the + /// agent. + /// *Note*: this is a struct variant because of the way go serializes struct{} + HealthCheck {}, +} + +/// A message received form the agent. +#[derive(Serialize, Deserialize, Debug, Clone)] +pub struct InboundMsg { + #[serde(flatten)] + pub(crate) inner: InboundMsgKind, + pub(crate) id: usize, +} + +/// The different underlying message types we can receive from the agent. +#[derive(Serialize, Deserialize, Debug, Clone)] +#[serde(tag = "type", content = "content")] +pub enum InboundMsgKind { + /// Indicates that the we sent an invalid message, i.e, we couldn't + /// properly deserialize it. + InvalidMessage { error: String }, + /// Indicates that the informan experienced an internal error while processing + /// a message. For example, if it failed to request upsacle from the agent, it + /// would return an `InternalError`. + InternalError { error: String }, + /// Indicates to us that we have been granted more resources. We should respond + /// with an `UpscaleConfirmation` when done handling the resources (increasins + /// file cache size, cgorup memory limits). + UpscaleNotification { granted: Resources }, + /// A request to reduce resource usage. We should response with a `DownscaleResult`, + /// when done. + DownscaleRequest { target: Resources }, + /// Part of the bidirectional heartbeat. The heartbeat is initiated by the + /// agent. + /// *Note*: this is a struct variant because of the way go serializes struct{} + HealthCheck {}, +} + +/// Represents the resources granted to a VM. +#[derive(Serialize, Deserialize, Debug, Clone, Copy)] +// Renamed because the agent has multiple resources types: +// `Resources` (milliCPU/memory slots) +// `Allocation` (vCPU/bytes) <- what we correspond to +#[serde(rename(serialize = "Allocation", deserialize = "Allocation"))] +pub struct Resources { + /// Number of vCPUs + pub(crate) cpu: f64, + /// Bytes of memory + pub(crate) mem: u64, +} + +impl Resources { + pub fn new(cpu: f64, mem: u64) -> Self { + Self { cpu, mem } + } +} + +pub const PROTOCOL_MIN_VERSION: ProtocolVersion = ProtocolVersion::V1_0; +pub const PROTOCOL_MAX_VERSION: ProtocolVersion = ProtocolVersion::V1_0; + +#[derive(Debug, Clone, Copy, PartialEq, PartialOrd, Ord, Eq, Serialize, Deserialize)] +pub struct ProtocolVersion(u8); + +impl ProtocolVersion { + /// Represents v1.0 of the agent<-> monitor protocol - the initial version + /// + /// Currently the latest version. + const V1_0: ProtocolVersion = ProtocolVersion(1); +} + +impl fmt::Display for ProtocolVersion { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match *self { + ProtocolVersion(0) => f.write_str(""), + ProtocolVersion::V1_0 => f.write_str("v1.0"), + other => write!(f, ""), + } + } +} + +/// A set of protocol bounds that determines what we are speaking. +/// +/// These bounds are inclusive. +#[derive(Debug)] +pub struct ProtocolRange { + pub min: ProtocolVersion, + pub max: ProtocolVersion, +} + +// Use a custom deserialize impl to ensure that `self.min <= self.max` +impl<'de> Deserialize<'de> for ProtocolRange { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + #[derive(Deserialize)] + struct InnerProtocolRange { + min: ProtocolVersion, + max: ProtocolVersion, + } + let InnerProtocolRange { min, max } = InnerProtocolRange::deserialize(deserializer)?; + if min > max { + Err(D::Error::custom(format!( + "min version = {min} is greater than max version = {max}", + ))) + } else { + Ok(ProtocolRange { min, max }) + } + } +} + +impl fmt::Display for ProtocolRange { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + if self.min == self.max { + f.write_fmt(format_args!("{}", self.max)) + } else { + f.write_fmt(format_args!("{} to {}", self.min, self.max)) + } + } +} + +impl ProtocolRange { + /// Find the highest shared version between two `ProtocolRange`'s + pub fn highest_shared_version(&self, other: &Self) -> anyhow::Result { + // We first have to make sure the ranges are overlapping. Once we know + // this, we can merge the ranges by taking the max of the mins and the + // mins of the maxes. + if self.min > other.max { + anyhow::bail!( + "Non-overlapping bounds: other.max = {} was less than self.min = {}", + other.max, + self.min, + ) + } else if self.max < other.min { + anyhow::bail!( + "Non-overlappinng bounds: self.max = {} was less than other.min = {}", + self.max, + other.min + ) + } else { + Ok(cmp::min(self.max, other.max)) + } + } +} + +/// We send this to the monitor after negotiating which protocol to use +#[derive(Serialize, Debug)] +#[serde(rename_all = "camelCase")] +pub enum ProtocolResponse { + Error(String), + Version(ProtocolVersion), +} diff --git a/libs/vm_monitor/src/runner.rs b/libs/vm_monitor/src/runner.rs new file mode 100644 index 0000000000..82055fda2e --- /dev/null +++ b/libs/vm_monitor/src/runner.rs @@ -0,0 +1,460 @@ +//! Exposes the `Runner`, which handles messages received from agent and +//! sends upscale requests. +//! +//! This is the "Monitor" part of the monitor binary and is the main entrypoint for +//! all functionality. + +use std::sync::Arc; +use std::{fmt::Debug, mem}; + +use anyhow::{bail, Context}; +use axum::extract::ws::{Message, WebSocket}; +use futures::StreamExt; +use tokio::sync::broadcast; +use tokio::sync::mpsc; +use tokio_util::sync::CancellationToken; +use tracing::{error, info, warn}; + +use crate::cgroup::{CgroupWatcher, MemoryLimits, Sequenced}; +use crate::dispatcher::Dispatcher; +use crate::filecache::{FileCacheConfig, FileCacheState}; +use crate::protocol::{InboundMsg, InboundMsgKind, OutboundMsg, OutboundMsgKind, Resources}; +use crate::{bytes_to_mebibytes, get_total_system_memory, spawn_with_cancel, Args, MiB}; + +/// Central struct that interacts with agent, dispatcher, and cgroup to handle +/// signals from the agent. +#[derive(Debug)] +pub struct Runner { + config: Config, + filecache: Option, + cgroup: Option>, + dispatcher: Dispatcher, + + /// We "mint" new message ids by incrementing this counter and taking the value. + /// + /// **Note**: This counter is always odd, so that we avoid collisions between the IDs generated + /// by us vs the autoscaler-agent. + counter: usize, + + /// A signal to kill the main thread produced by `self.run()`. This is triggered + /// when the server receives a new connection. When the thread receives the + /// signal off this channel, it will gracefully shutdown. + kill: broadcast::Receiver<()>, +} + +/// Configuration for a `Runner` +#[derive(Debug)] +pub struct Config { + /// `sys_buffer_bytes` gives the estimated amount of memory, in bytes, that the kernel uses before + /// handing out the rest to userspace. This value is the estimated difference between the + /// *actual* physical memory and the amount reported by `grep MemTotal /proc/meminfo`. + /// + /// For more information, refer to `man 5 proc`, which defines MemTotal as "Total usable RAM + /// (i.e., physical RAM minus a few reserved bits and the kernel binary code)". + /// + /// We only use `sys_buffer_bytes` when calculating the system memory from the *external* memory + /// size, rather than the self-reported memory size, according to the kernel. + /// + /// TODO: this field is only necessary while we still have to trust the autoscaler-agent's + /// upscale resource amounts (because we might not *actually* have been upscaled yet). This field + /// should be removed once we have a better solution there. + sys_buffer_bytes: u64, +} + +impl Default for Config { + fn default() -> Self { + Self { + sys_buffer_bytes: 100 * MiB, + } + } +} + +impl Runner { + /// Create a new monitor. + #[tracing::instrument(skip_all, fields(?config, ?args))] + pub async fn new( + config: Config, + args: &Args, + ws: WebSocket, + kill: broadcast::Receiver<()>, + token: CancellationToken, + ) -> anyhow::Result { + anyhow::ensure!( + config.sys_buffer_bytes != 0, + "invalid monitor Config: sys_buffer_bytes cannot be 0" + ); + + // *NOTE*: the dispatcher and cgroup manager talk through these channels + // so make sure they each get the correct half, nothing is droppped, etc. + let (notified_send, notified_recv) = mpsc::channel(1); + let (requesting_send, requesting_recv) = mpsc::channel(1); + + let dispatcher = Dispatcher::new(ws, notified_send, requesting_recv) + .await + .context("error creating new dispatcher")?; + + let mut state = Runner { + config, + filecache: None, + cgroup: None, + dispatcher, + counter: 1, // NB: must be odd, see the comment about the field for more. + kill, + }; + + let mut file_cache_reserved_bytes = 0; + let mem = get_total_system_memory(); + + // We need to process file cache initialization before cgroup initialization, so that the memory + // allocated to the file cache is appropriately taken into account when we decide the cgroup's + // memory limits. + if let Some(connstr) = &args.pgconnstr { + info!("initializing file cache"); + let config = match args.file_cache_on_disk { + true => FileCacheConfig::default_on_disk(), + false => FileCacheConfig::default_in_memory(), + }; + + let mut file_cache = FileCacheState::new(connstr, config, token.clone()) + .await + .context("failed to create file cache")?; + + let size = file_cache + .get_file_cache_size() + .await + .context("error getting file cache size")?; + + let new_size = file_cache.config.calculate_cache_size(mem); + info!( + initial = bytes_to_mebibytes(size), + new = bytes_to_mebibytes(new_size), + "setting initial file cache size", + ); + + // note: even if size == new_size, we want to explicitly set it, just + // to make sure that we have the permissions to do so + let actual_size = file_cache + .set_file_cache_size(new_size) + .await + .context("failed to set file cache size, possibly due to inadequate permissions")?; + if actual_size != new_size { + info!("file cache size actually got set to {actual_size}") + } + // Mark the resources given to the file cache as reserved, but only if it's in memory. + if !args.file_cache_on_disk { + file_cache_reserved_bytes = actual_size; + } + + state.filecache = Some(file_cache); + } + + if let Some(name) = &args.cgroup { + let (mut cgroup, cgroup_event_stream) = + CgroupWatcher::new(name.clone(), requesting_send) + .context("failed to create cgroup manager")?; + + let available = mem - file_cache_reserved_bytes; + + cgroup + .set_memory_limits(available) + .context("failed to set cgroup memory limits")?; + + let cgroup = Arc::new(cgroup); + + // Some might call this . . . cgroup v2 + let cgroup_clone = Arc::clone(&cgroup); + + spawn_with_cancel(token, |_| error!("cgroup watcher terminated"), async move { + cgroup_clone.watch(notified_recv, cgroup_event_stream).await + }); + + state.cgroup = Some(cgroup); + } else { + // *NOTE*: We need to forget the sender so that its drop impl does not get ran. + // This allows us to poll it in `Monitor::run` regardless of whether we + // are managing a cgroup or not. If we don't forget it, all receives will + // immediately return an error because the sender is droped and it will + // claim all select! statements, effectively turning `Monitor::run` into + // `loop { fail to receive }`. + mem::forget(requesting_send); + } + + Ok(state) + } + + /// Attempt to downscale filecache + cgroup + #[tracing::instrument(skip_all, fields(?target))] + pub async fn try_downscale(&mut self, target: Resources) -> anyhow::Result<(bool, String)> { + // Nothing to adjust + if self.cgroup.is_none() && self.filecache.is_none() { + info!("no action needed for downscale (no cgroup or file cache enabled)"); + return Ok(( + true, + "monitor is not managing cgroup or file cache".to_string(), + )); + } + + let requested_mem = target.mem; + let usable_system_memory = requested_mem.saturating_sub(self.config.sys_buffer_bytes); + let expected_file_cache_mem_usage = self + .filecache + .as_ref() + .map(|file_cache| file_cache.config.calculate_cache_size(usable_system_memory)) + .unwrap_or(0); + let mut new_cgroup_mem_high = 0; + if let Some(cgroup) = &self.cgroup { + new_cgroup_mem_high = cgroup + .config + .calculate_memory_high_value(usable_system_memory - expected_file_cache_mem_usage); + + let current = cgroup + .current_memory_usage() + .context("failed to fetch cgroup memory")?; + + if new_cgroup_mem_high < current + cgroup.config.memory_high_buffer_bytes { + let status = format!( + "{}: {} MiB (new high) < {} (current usage) + {} (buffer)", + "calculated memory.high too low", + bytes_to_mebibytes(new_cgroup_mem_high), + bytes_to_mebibytes(current), + bytes_to_mebibytes(cgroup.config.memory_high_buffer_bytes) + ); + + info!(status, "discontinuing downscale"); + + return Ok((false, status)); + } + } + + // The downscaling has been approved. Downscale the file cache, then the cgroup. + let mut status = vec![]; + let mut file_cache_mem_usage = 0; + if let Some(file_cache) = &mut self.filecache { + let actual_usage = file_cache + .set_file_cache_size(expected_file_cache_mem_usage) + .await + .context("failed to set file cache size")?; + if file_cache.config.in_memory { + file_cache_mem_usage = actual_usage; + } + let message = format!( + "set file cache size to {} MiB (in memory = {})", + bytes_to_mebibytes(actual_usage), + file_cache.config.in_memory, + ); + info!("downscale: {message}"); + status.push(message); + } + + if let Some(cgroup) = &self.cgroup { + let available_memory = usable_system_memory - file_cache_mem_usage; + + if file_cache_mem_usage != expected_file_cache_mem_usage { + new_cgroup_mem_high = cgroup.config.calculate_memory_high_value(available_memory); + } + + let limits = MemoryLimits::new( + // new_cgroup_mem_high is initialized to 0 but it is guarancontextd to not be here + // since it is properly initialized in the previous cgroup if let block + new_cgroup_mem_high, + available_memory, + ); + cgroup + .set_limits(&limits) + .context("failed to set cgroup memory limits")?; + + let message = format!( + "set cgroup memory.high to {} MiB, of new max {} MiB", + bytes_to_mebibytes(new_cgroup_mem_high), + bytes_to_mebibytes(available_memory) + ); + info!("downscale: {message}"); + status.push(message); + } + + // TODO: make this status thing less jank + let status = status.join("; "); + Ok((true, status)) + } + + /// Handle new resources + #[tracing::instrument(skip_all, fields(?resources))] + pub async fn handle_upscale(&mut self, resources: Resources) -> anyhow::Result<()> { + if self.filecache.is_none() && self.cgroup.is_none() { + info!("no action needed for upscale (no cgroup or file cache enabled)"); + return Ok(()); + } + + let new_mem = resources.mem; + let usable_system_memory = new_mem.saturating_sub(self.config.sys_buffer_bytes); + + // Get the file cache's expected contribution to the memory usage + let mut file_cache_mem_usage = 0; + if let Some(file_cache) = &mut self.filecache { + let expected_usage = file_cache.config.calculate_cache_size(usable_system_memory); + info!( + target = bytes_to_mebibytes(expected_usage), + total = bytes_to_mebibytes(new_mem), + "updating file cache size", + ); + + let actual_usage = file_cache + .set_file_cache_size(expected_usage) + .await + .context("failed to set file cache size")?; + if file_cache.config.in_memory { + file_cache_mem_usage = actual_usage; + } + + if actual_usage != expected_usage { + warn!( + "file cache was set to a different size that we wanted: target = {} Mib, actual= {} Mib", + bytes_to_mebibytes(expected_usage), + bytes_to_mebibytes(actual_usage) + ) + } + } + + if let Some(cgroup) = &self.cgroup { + let available_memory = usable_system_memory - file_cache_mem_usage; + let new_cgroup_mem_high = cgroup.config.calculate_memory_high_value(available_memory); + info!( + target = bytes_to_mebibytes(new_cgroup_mem_high), + total = bytes_to_mebibytes(new_mem), + name = cgroup.path(), + "updating cgroup memory.high", + ); + let limits = MemoryLimits::new(new_cgroup_mem_high, available_memory); + cgroup + .set_limits(&limits) + .context("failed to set file cache size")?; + } + + Ok(()) + } + + /// Take in a message and perform some action, such as downscaling or upscaling, + /// and return a message to be send back. + #[tracing::instrument(skip_all, fields(%id, message = ?inner))] + pub async fn process_message( + &mut self, + InboundMsg { inner, id }: InboundMsg, + ) -> anyhow::Result> { + match inner { + InboundMsgKind::UpscaleNotification { granted } => { + self.handle_upscale(granted) + .await + .context("failed to handle upscale")?; + self.dispatcher + .notify_upscale(Sequenced::new(granted)) + .await + .context("failed to notify notify cgroup of upscale")?; + Ok(Some(OutboundMsg::new( + OutboundMsgKind::UpscaleConfirmation {}, + id, + ))) + } + InboundMsgKind::DownscaleRequest { target } => self + .try_downscale(target) + .await + .context("failed to downscale") + .map(|(ok, status)| { + Some(OutboundMsg::new( + OutboundMsgKind::DownscaleResult { ok, status }, + id, + )) + }), + InboundMsgKind::InvalidMessage { error } => { + warn!( + %error, id, "received notification of an invalid message we sent" + ); + Ok(None) + } + InboundMsgKind::InternalError { error } => { + warn!(error, id, "agent experienced an internal error"); + Ok(None) + } + InboundMsgKind::HealthCheck {} => { + Ok(Some(OutboundMsg::new(OutboundMsgKind::HealthCheck {}, id))) + } + } + } + + // TODO: don't propagate errors, probably just warn!? + #[tracing::instrument(skip_all)] + pub async fn run(&mut self) -> anyhow::Result<()> { + info!("starting dispatcher"); + loop { + tokio::select! { + signal = self.kill.recv() => { + match signal { + Ok(()) => return Ok(()), + Err(e) => bail!("failed to receive kill signal: {e}") + } + } + // we need to propagate an upscale request + request = self.dispatcher.request_upscale_events.recv() => { + if request.is_none() { + bail!("failed to listen for upscale event from cgroup") + } + info!("cgroup asking for upscale; forwarding request"); + self.counter += 2; // Increment, preserving parity (i.e. keep the + // counter odd). See the field comment for more. + self.dispatcher + .send(OutboundMsg::new(OutboundMsgKind::UpscaleRequest {}, self.counter)) + .await + .context("failed to send message")?; + } + // there is a message from the agent + msg = self.dispatcher.source.next() => { + if let Some(msg) = msg { + // Don't use 'message' as a key as the string also uses + // that for its key + info!(?msg, "received message"); + match msg { + Ok(msg) => { + let message: InboundMsg = match msg { + Message::Text(text) => { + serde_json::from_str(&text).context("failed to deserialize text message")? + } + other => { + warn!( + // Don't use 'message' as a key as the + // string also uses that for its key + msg = ?other, + "agent should only send text messages but received different type" + ); + continue + }, + }; + + let out = match self.process_message(message.clone()).await { + Ok(Some(out)) => out, + Ok(None) => continue, + Err(e) => { + let error = e.to_string(); + warn!(?error, "error handling message"); + OutboundMsg::new( + OutboundMsgKind::InternalError { + error + }, + message.id + ) + } + }; + + self.dispatcher + .send(out) + .await + .context("failed to send message")?; + } + Err(e) => warn!("{e}"), + } + } else { + anyhow::bail!("dispatcher connection closed") + } + } + } + } + } +} diff --git a/pageserver/Cargo.toml b/pageserver/Cargo.toml index d709fac5cb..bbdd8b1e99 100644 --- a/pageserver/Cargo.toml +++ b/pageserver/Cargo.toml @@ -51,6 +51,7 @@ serde.workspace = true serde_json = { workspace = true, features = ["raw_value"] } serde_with.workspace = true signal-hook.workspace = true +smallvec = { workspace = true, features = ["write"] } svg_fmt.workspace = true sync_wrapper.workspace = true tokio-tar.workspace = true diff --git a/pageserver/benches/bench_layer_map.rs b/pageserver/benches/bench_layer_map.rs index f7a5832844..735f358d8b 100644 --- a/pageserver/benches/bench_layer_map.rs +++ b/pageserver/benches/bench_layer_map.rs @@ -215,7 +215,6 @@ fn bench_sequential(c: &mut Criterion) { TimelineId::generate(), zero.add(10 * i32)..zero.add(10 * i32 + 1), Lsn(i), - false, 0, ); updates.insert_historic(layer); diff --git a/pageserver/ctl/src/layer_map_analyzer.rs b/pageserver/ctl/src/layer_map_analyzer.rs index 9d9a1fa151..29bd6ce598 100644 --- a/pageserver/ctl/src/layer_map_analyzer.rs +++ b/pageserver/ctl/src/layer_map_analyzer.rs @@ -10,7 +10,7 @@ use std::{fs, path::Path, str}; use pageserver::page_cache::PAGE_SZ; use pageserver::repository::{Key, KEY_SIZE}; -use pageserver::tenant::block_io::{BlockReader, FileBlockReader}; +use pageserver::tenant::block_io::FileBlockReader; use pageserver::tenant::disk_btree::{DiskBtreeReader, VisitDirection}; use pageserver::tenant::storage_layer::delta_layer::{Summary, DELTA_KEY_SIZE}; use pageserver::tenant::storage_layer::range_overlaps; @@ -97,7 +97,7 @@ pub(crate) fn parse_filename(name: &str) -> Option { // Finds the max_holes largest holes, ignoring any that are smaller than MIN_HOLE_LENGTH" async fn get_holes(path: &Path, max_holes: usize) -> Result> { let file = FileBlockReader::new(VirtualFile::open(path)?); - let summary_blk = file.read_blk(0)?; + let summary_blk = file.read_blk(0).await?; let actual_summary = Summary::des_prefix(summary_blk.as_ref())?; let tree_reader = DiskBtreeReader::<_, DELTA_KEY_SIZE>::new( actual_summary.index_start_blk, diff --git a/pageserver/ctl/src/layers.rs b/pageserver/ctl/src/layers.rs index 2d178a42bd..2af54902f7 100644 --- a/pageserver/ctl/src/layers.rs +++ b/pageserver/ctl/src/layers.rs @@ -44,13 +44,11 @@ pub(crate) enum LayerCmd { } async fn read_delta_file(path: impl AsRef) -> Result<()> { - use pageserver::tenant::block_io::BlockReader; - let path = path.as_ref(); virtual_file::init(10); page_cache::init(100); let file = FileBlockReader::new(VirtualFile::open(path)?); - let summary_blk = file.read_blk(0)?; + let summary_blk = file.read_blk(0).await?; let actual_summary = Summary::des_prefix(summary_blk.as_ref())?; let tree_reader = DiskBtreeReader::<_, DELTA_KEY_SIZE>::new( actual_summary.index_start_blk, @@ -70,7 +68,7 @@ async fn read_delta_file(path: impl AsRef) -> Result<()> { }, ) .await?; - let cursor = BlockCursor::new(&file); + let cursor = BlockCursor::new_fileblockreader_virtual(&file); for (k, v) in all { let value = cursor.read_blob(v.pos()).await?; println!("key:{} value_len:{}", k, value.len()); diff --git a/pageserver/src/metrics.rs b/pageserver/src/metrics.rs index 2bdf880373..3153d1c80e 100644 --- a/pageserver/src/metrics.rs +++ b/pageserver/src/metrics.rs @@ -6,7 +6,7 @@ use metrics::{ HistogramVec, IntCounter, IntCounterVec, IntGauge, IntGaugeVec, UIntGauge, UIntGaugeVec, }; use once_cell::sync::Lazy; -use strum::VariantNames; +use strum::{EnumCount, IntoEnumIterator, VariantNames}; use strum_macros::{EnumVariantNames, IntoStaticStr}; use utils::id::{TenantId, TimelineId}; @@ -570,23 +570,160 @@ pub(crate) static STORAGE_IO_SIZE: Lazy = Lazy::new(|| { .expect("failed to define a metric") }); -const SMGR_QUERY_TIME_OPERATIONS: &[&str] = &[ - "get_rel_exists", - "get_rel_size", - "get_page_at_lsn", - "get_db_size", -]; +#[derive(Debug)] +struct GlobalAndPerTimelineHistogram { + global: Histogram, + per_tenant_timeline: Histogram, +} -pub static SMGR_QUERY_TIME: Lazy = Lazy::new(|| { +impl GlobalAndPerTimelineHistogram { + fn observe(&self, value: f64) { + self.global.observe(value); + self.per_tenant_timeline.observe(value); + } +} + +struct GlobalAndPerTimelineHistogramTimer<'a> { + h: &'a GlobalAndPerTimelineHistogram, + start: std::time::Instant, +} + +impl<'a> Drop for GlobalAndPerTimelineHistogramTimer<'a> { + fn drop(&mut self) { + let elapsed = self.start.elapsed(); + self.h.observe(elapsed.as_secs_f64()); + } +} + +#[derive( + Debug, + Clone, + Copy, + IntoStaticStr, + strum_macros::EnumCount, + strum_macros::EnumIter, + strum_macros::FromRepr, +)] +#[strum(serialize_all = "snake_case")] +pub enum SmgrQueryType { + GetRelExists, + GetRelSize, + GetPageAtLsn, + GetDbSize, +} + +#[derive(Debug)] +pub struct SmgrQueryTimePerTimeline { + metrics: [GlobalAndPerTimelineHistogram; SmgrQueryType::COUNT], +} + +static SMGR_QUERY_TIME_PER_TENANT_TIMELINE: Lazy = Lazy::new(|| { register_histogram_vec!( "pageserver_smgr_query_seconds", - "Time spent on smgr query handling", + "Time spent on smgr query handling, aggegated by query type and tenant/timeline.", &["smgr_query_type", "tenant_id", "timeline_id"], CRITICAL_OP_BUCKETS.into(), ) .expect("failed to define a metric") }); +static SMGR_QUERY_TIME_GLOBAL: Lazy = Lazy::new(|| { + register_histogram_vec!( + "pageserver_smgr_query_seconds_global", + "Time spent on smgr query handling, aggregated by query type.", + &["smgr_query_type"], + CRITICAL_OP_BUCKETS.into(), + ) + .expect("failed to define a metric") +}); + +impl SmgrQueryTimePerTimeline { + pub(crate) fn new(tenant_id: &TenantId, timeline_id: &TimelineId) -> Self { + let tenant_id = tenant_id.to_string(); + let timeline_id = timeline_id.to_string(); + let metrics = std::array::from_fn(|i| { + let op = SmgrQueryType::from_repr(i).unwrap(); + let global = SMGR_QUERY_TIME_GLOBAL + .get_metric_with_label_values(&[op.into()]) + .unwrap(); + let per_tenant_timeline = SMGR_QUERY_TIME_PER_TENANT_TIMELINE + .get_metric_with_label_values(&[op.into(), &tenant_id, &timeline_id]) + .unwrap(); + GlobalAndPerTimelineHistogram { + global, + per_tenant_timeline, + } + }); + Self { metrics } + } + pub(crate) fn start_timer(&self, op: SmgrQueryType) -> impl Drop + '_ { + let metric = &self.metrics[op as usize]; + GlobalAndPerTimelineHistogramTimer { + h: metric, + start: std::time::Instant::now(), + } + } +} + +#[cfg(test)] +mod smgr_query_time_tests { + use strum::IntoEnumIterator; + use utils::id::{TenantId, TimelineId}; + + // Regression test, we used hard-coded string constants before using an enum. + #[test] + fn op_label_name() { + use super::SmgrQueryType::*; + let expect: [(super::SmgrQueryType, &'static str); 4] = [ + (GetRelExists, "get_rel_exists"), + (GetRelSize, "get_rel_size"), + (GetPageAtLsn, "get_page_at_lsn"), + (GetDbSize, "get_db_size"), + ]; + for (op, expect) in expect { + let actual: &'static str = op.into(); + assert_eq!(actual, expect); + } + } + + #[test] + fn basic() { + let ops: Vec<_> = super::SmgrQueryType::iter().collect(); + + for op in &ops { + let tenant_id = TenantId::generate(); + let timeline_id = TimelineId::generate(); + let metrics = super::SmgrQueryTimePerTimeline::new(&tenant_id, &timeline_id); + + let get_counts = || { + let global: u64 = ops + .iter() + .map(|op| metrics.metrics[*op as usize].global.get_sample_count()) + .sum(); + let per_tenant_timeline: u64 = ops + .iter() + .map(|op| { + metrics.metrics[*op as usize] + .per_tenant_timeline + .get_sample_count() + }) + .sum(); + (global, per_tenant_timeline) + }; + + let (pre_global, pre_per_tenant_timeline) = get_counts(); + assert_eq!(pre_per_tenant_timeline, 0); + + let timer = metrics.start_timer(*op); + drop(timer); + + let (post_global, post_per_tenant_timeline) = get_counts(); + assert_eq!(post_per_tenant_timeline, 1); + assert!(post_global > pre_global); + } + } +} + // keep in sync with control plane Go code so that we can validate // compute's basebackup_ms metric with our perspective in the context of SLI/SLO. static COMPUTE_STARTUP_BUCKETS: Lazy<[f64; 28]> = Lazy::new(|| { @@ -1070,6 +1207,12 @@ impl Drop for TimelineMetrics { .write() .unwrap() .remove(tenant_id, timeline_id); + + // The following metrics are born outside of the TimelineMetrics lifecycle but still + // removed at the end of it. The idea is to have the metrics outlive the + // entity during which they're observed, e.g., the smgr metrics shall + // outlive an individual smgr connection, but not the timeline. + for op in StorageTimeOperation::VARIANTS { let _ = STORAGE_TIME_SUM_PER_TIMELINE.remove_label_values(&[op, tenant_id, timeline_id]); @@ -1081,8 +1224,12 @@ impl Drop for TimelineMetrics { let _ = STORAGE_IO_SIZE.remove_label_values(&[op, tenant_id, timeline_id]); } - for op in SMGR_QUERY_TIME_OPERATIONS { - let _ = SMGR_QUERY_TIME.remove_label_values(&[op, tenant_id, timeline_id]); + for op in SmgrQueryType::iter() { + let _ = SMGR_QUERY_TIME_PER_TENANT_TIMELINE.remove_label_values(&[ + op.into(), + tenant_id, + timeline_id, + ]); } } } diff --git a/pageserver/src/page_cache.rs b/pageserver/src/page_cache.rs index e1e696ddad..fb1c5fc485 100644 --- a/pageserver/src/page_cache.rs +++ b/pageserver/src/page_cache.rs @@ -75,10 +75,7 @@ use std::{ collections::{hash_map::Entry, HashMap}, convert::TryInto, - sync::{ - atomic::{AtomicU64, AtomicU8, AtomicUsize, Ordering}, - RwLock, RwLockReadGuard, RwLockWriteGuard, TryLockError, - }, + sync::atomic::{AtomicU64, AtomicU8, AtomicUsize, Ordering}, }; use anyhow::Context; @@ -162,7 +159,7 @@ struct Version { } struct Slot { - inner: RwLock, + inner: tokio::sync::RwLock, usage_count: AtomicU8, } @@ -203,6 +200,11 @@ impl Slot { Err(usage_count) => usage_count, } } + + /// Sets the usage count to a specific value. + fn set_usage_count(&self, count: u8) { + self.usage_count.store(count, Ordering::Relaxed); + } } pub struct PageCache { @@ -215,9 +217,9 @@ pub struct PageCache { /// /// If you add support for caching different kinds of objects, each object kind /// can have a separate mapping map, next to this field. - materialized_page_map: RwLock>>, + materialized_page_map: std::sync::RwLock>>, - immutable_page_map: RwLock>, + immutable_page_map: std::sync::RwLock>, /// The actual buffers with their metadata. slots: Box<[Slot]>, @@ -233,7 +235,7 @@ pub struct PageCache { /// PageReadGuard is a "lease" on a buffer, for reading. The page is kept locked /// until the guard is dropped. /// -pub struct PageReadGuard<'i>(RwLockReadGuard<'i, SlotInner>); +pub struct PageReadGuard<'i>(tokio::sync::RwLockReadGuard<'i, SlotInner>); impl std::ops::Deref for PageReadGuard<'_> { type Target = [u8; PAGE_SZ]; @@ -260,9 +262,10 @@ impl AsRef<[u8; PAGE_SZ]> for PageReadGuard<'_> { /// to initialize. /// pub struct PageWriteGuard<'i> { - inner: RwLockWriteGuard<'i, SlotInner>, + inner: tokio::sync::RwLockWriteGuard<'i, SlotInner>, // Are the page contents currently valid? + // Used to mark pages as invalid that are assigned but not yet filled with data. valid: bool, } @@ -337,7 +340,7 @@ impl PageCache { /// The 'lsn' is an upper bound, this will return the latest version of /// the given block, but not newer than 'lsn'. Returns the actual LSN of the /// returned page. - pub fn lookup_materialized_page( + pub async fn lookup_materialized_page( &self, tenant_id: TenantId, timeline_id: TimelineId, @@ -357,7 +360,7 @@ impl PageCache { lsn, }; - if let Some(guard) = self.try_lock_for_read(&mut cache_key) { + if let Some(guard) = self.try_lock_for_read(&mut cache_key).await { if let CacheKey::MaterializedPage { hash_key: _, lsn: available_lsn, @@ -384,7 +387,7 @@ impl PageCache { /// /// Store an image of the given page in the cache. /// - pub fn memorize_materialized_page( + pub async fn memorize_materialized_page( &self, tenant_id: TenantId, timeline_id: TimelineId, @@ -401,7 +404,7 @@ impl PageCache { lsn, }; - match self.lock_for_write(&cache_key)? { + match self.lock_for_write(&cache_key).await? { WriteBufResult::Found(write_guard) => { // We already had it in cache. Another thread must've put it there // concurrently. Check that it had the same contents that we @@ -419,31 +422,14 @@ impl PageCache { // Section 1.2: Public interface functions for working with immutable file pages. - pub fn read_immutable_buf(&self, file_id: FileId, blkno: u32) -> anyhow::Result { + pub async fn read_immutable_buf( + &self, + file_id: FileId, + blkno: u32, + ) -> anyhow::Result { let mut cache_key = CacheKey::ImmutableFilePage { file_id, blkno }; - self.lock_for_read(&mut cache_key) - } - - /// Immediately drop all buffers belonging to given file - pub fn drop_buffers_for_immutable(&self, drop_file_id: FileId) { - for slot_idx in 0..self.slots.len() { - let slot = &self.slots[slot_idx]; - - let mut inner = slot.inner.write().unwrap(); - if let Some(key) = &inner.key { - match key { - CacheKey::ImmutableFilePage { file_id, blkno: _ } - if *file_id == drop_file_id => - { - // remove mapping for old buffer - self.remove_mapping(key); - inner.key = None; - } - _ => {} - } - } - } + self.lock_for_read(&mut cache_key).await } // @@ -463,14 +449,14 @@ impl PageCache { /// /// If no page is found, returns None and *cache_key is left unmodified. /// - fn try_lock_for_read(&self, cache_key: &mut CacheKey) -> Option { + async fn try_lock_for_read(&self, cache_key: &mut CacheKey) -> Option { let cache_key_orig = cache_key.clone(); if let Some(slot_idx) = self.search_mapping(cache_key) { // The page was found in the mapping. Lock the slot, and re-check // that it's still what we expected (because we released the mapping // lock already, another thread could have evicted the page) let slot = &self.slots[slot_idx]; - let inner = slot.inner.read().unwrap(); + let inner = slot.inner.read().await; if inner.key.as_ref() == Some(cache_key) { slot.inc_usage_count(); return Some(PageReadGuard(inner)); @@ -511,7 +497,7 @@ impl PageCache { /// } /// ``` /// - fn lock_for_read(&self, cache_key: &mut CacheKey) -> anyhow::Result { + async fn lock_for_read(&self, cache_key: &mut CacheKey) -> anyhow::Result { let (read_access, hit) = match cache_key { CacheKey::MaterializedPage { .. } => { unreachable!("Materialized pages use lookup_materialized_page") @@ -526,7 +512,7 @@ impl PageCache { let mut is_first_iteration = true; loop { // First check if the key already exists in the cache. - if let Some(read_guard) = self.try_lock_for_read(cache_key) { + if let Some(read_guard) = self.try_lock_for_read(cache_key).await { if is_first_iteration { hit.inc(); } @@ -556,7 +542,7 @@ impl PageCache { // Make the slot ready let slot = &self.slots[slot_idx]; inner.key = Some(cache_key.clone()); - slot.usage_count.store(1, Ordering::Relaxed); + slot.set_usage_count(1); return Ok(ReadBufResult::NotFound(PageWriteGuard { inner, @@ -569,13 +555,13 @@ impl PageCache { /// found, returns None. /// /// When locking a page for writing, the search criteria is always "exact". - fn try_lock_for_write(&self, cache_key: &CacheKey) -> Option { + async fn try_lock_for_write(&self, cache_key: &CacheKey) -> Option { if let Some(slot_idx) = self.search_mapping_for_write(cache_key) { // The page was found in the mapping. Lock the slot, and re-check // that it's still what we expected (because we don't released the mapping // lock already, another thread could have evicted the page) let slot = &self.slots[slot_idx]; - let inner = slot.inner.write().unwrap(); + let inner = slot.inner.write().await; if inner.key.as_ref() == Some(cache_key) { slot.inc_usage_count(); return Some(PageWriteGuard { inner, valid: true }); @@ -588,10 +574,10 @@ impl PageCache { /// /// Similar to lock_for_read(), but the returned buffer is write-locked and /// may be modified by the caller even if it's already found in the cache. - fn lock_for_write(&self, cache_key: &CacheKey) -> anyhow::Result { + async fn lock_for_write(&self, cache_key: &CacheKey) -> anyhow::Result { loop { // First check if the key already exists in the cache. - if let Some(write_guard) = self.try_lock_for_write(cache_key) { + if let Some(write_guard) = self.try_lock_for_write(cache_key).await { return Ok(WriteBufResult::Found(write_guard)); } @@ -617,7 +603,7 @@ impl PageCache { // Make the slot ready let slot = &self.slots[slot_idx]; inner.key = Some(cache_key.clone()); - slot.usage_count.store(1, Ordering::Relaxed); + slot.set_usage_count(1); return Ok(WriteBufResult::NotFound(PageWriteGuard { inner, @@ -772,7 +758,7 @@ impl PageCache { /// Find a slot to evict. /// /// On return, the slot is empty and write-locked. - fn find_victim(&self) -> anyhow::Result<(usize, RwLockWriteGuard)> { + fn find_victim(&self) -> anyhow::Result<(usize, tokio::sync::RwLockWriteGuard)> { let iter_limit = self.slots.len() * 10; let mut iters = 0; loop { @@ -784,10 +770,7 @@ impl PageCache { if slot.dec_usage_count() == 0 { let mut inner = match slot.inner.try_write() { Ok(inner) => inner, - Err(TryLockError::Poisoned(err)) => { - anyhow::bail!("buffer lock was poisoned: {err:?}") - } - Err(TryLockError::WouldBlock) => { + Err(_err) => { // If we have looped through the whole buffer pool 10 times // and still haven't found a victim buffer, something's wrong. // Maybe all the buffers were in locked. That could happen in @@ -816,6 +799,8 @@ impl PageCache { fn new(num_pages: usize) -> Self { assert!(num_pages > 0, "page cache size must be > 0"); + // We use Box::leak here and into_boxed_slice to avoid leaking uninitialized + // memory that Vec's might contain. let page_buffer = Box::leak(vec![0u8; num_pages * PAGE_SZ].into_boxed_slice()); let size_metrics = &crate::metrics::PAGE_CACHE_SIZE; @@ -829,7 +814,7 @@ impl PageCache { let buf: &mut [u8; PAGE_SZ] = chunk.try_into().unwrap(); Slot { - inner: RwLock::new(SlotInner { key: None, buf }), + inner: tokio::sync::RwLock::new(SlotInner { key: None, buf }), usage_count: AtomicU8::new(0), } }) diff --git a/pageserver/src/page_service.rs b/pageserver/src/page_service.rs index 35dd5ecdb5..72a66d51a6 100644 --- a/pageserver/src/page_service.rs +++ b/pageserver/src/page_service.rs @@ -50,7 +50,8 @@ use crate::basebackup; use crate::config::PageServerConf; use crate::context::{DownloadBehavior, RequestContext}; use crate::import_datadir::import_wal_from_tar; -use crate::metrics::{LIVE_CONNECTIONS_COUNT, SMGR_QUERY_TIME}; +use crate::metrics; +use crate::metrics::LIVE_CONNECTIONS_COUNT; use crate::task_mgr; use crate::task_mgr::TaskKind; use crate::tenant; @@ -306,39 +307,6 @@ async fn page_service_conn_main( } } -struct PageRequestMetrics { - get_rel_exists: metrics::Histogram, - get_rel_size: metrics::Histogram, - get_page_at_lsn: metrics::Histogram, - get_db_size: metrics::Histogram, -} - -impl PageRequestMetrics { - fn new(tenant_id: &TenantId, timeline_id: &TimelineId) -> Self { - let tenant_id = tenant_id.to_string(); - let timeline_id = timeline_id.to_string(); - - let get_rel_exists = - SMGR_QUERY_TIME.with_label_values(&["get_rel_exists", &tenant_id, &timeline_id]); - - let get_rel_size = - SMGR_QUERY_TIME.with_label_values(&["get_rel_size", &tenant_id, &timeline_id]); - - let get_page_at_lsn = - SMGR_QUERY_TIME.with_label_values(&["get_page_at_lsn", &tenant_id, &timeline_id]); - - let get_db_size = - SMGR_QUERY_TIME.with_label_values(&["get_db_size", &tenant_id, &timeline_id]); - - Self { - get_rel_exists, - get_rel_size, - get_page_at_lsn, - get_db_size, - } - } -} - struct PageServerHandler { _conf: &'static PageServerConf, broker_client: storage_broker::BrokerClientChannel, @@ -406,7 +374,7 @@ impl PageServerHandler { pgb.write_message_noflush(&BeMessage::CopyBothResponse)?; pgb.flush().await?; - let metrics = PageRequestMetrics::new(&tenant_id, &timeline_id); + let metrics = metrics::SmgrQueryTimePerTimeline::new(&tenant_id, &timeline_id); loop { let msg = tokio::select! { @@ -446,21 +414,21 @@ impl PageServerHandler { let response = match neon_fe_msg { PagestreamFeMessage::Exists(req) => { - let _timer = metrics.get_rel_exists.start_timer(); + let _timer = metrics.start_timer(metrics::SmgrQueryType::GetRelExists); self.handle_get_rel_exists_request(&timeline, &req, &ctx) .await } PagestreamFeMessage::Nblocks(req) => { - let _timer = metrics.get_rel_size.start_timer(); + let _timer = metrics.start_timer(metrics::SmgrQueryType::GetRelSize); self.handle_get_nblocks_request(&timeline, &req, &ctx).await } PagestreamFeMessage::GetPage(req) => { - let _timer = metrics.get_page_at_lsn.start_timer(); + let _timer = metrics.start_timer(metrics::SmgrQueryType::GetPageAtLsn); self.handle_get_page_at_lsn_request(&timeline, &req, &ctx) .await } PagestreamFeMessage::DbSize(req) => { - let _timer = metrics.get_db_size.start_timer(); + let _timer = metrics.start_timer(metrics::SmgrQueryType::GetDbSize); self.handle_db_size_request(&timeline, &req, &ctx).await } }; @@ -984,8 +952,8 @@ where false }; - metrics::metric_vec_duration::observe_async_block_duration_by_result( - &*crate::metrics::BASEBACKUP_QUERY_TIME, + ::metrics::metric_vec_duration::observe_async_block_duration_by_result( + &*metrics::BASEBACKUP_QUERY_TIME, async move { self.handle_basebackup_request( pgb, diff --git a/pageserver/src/tenant.rs b/pageserver/src/tenant.rs index 808aeede79..ee365a7a3a 100644 --- a/pageserver/src/tenant.rs +++ b/pageserver/src/tenant.rs @@ -73,7 +73,6 @@ use crate::tenant::remote_timeline_client::index::IndexPart; use crate::tenant::remote_timeline_client::MaybeDeletedIndexPart; use crate::tenant::storage_layer::DeltaLayer; use crate::tenant::storage_layer::ImageLayer; -use crate::tenant::storage_layer::Layer; use crate::InitializationOrder; use crate::tenant::timeline::delete::DeleteTimelineFlow; @@ -428,13 +427,53 @@ impl Tenant { init_order, CreateTimelineCause::Load, )?; - let new_disk_consistent_lsn = timeline.get_disk_consistent_lsn(); + let disk_consistent_lsn = timeline.get_disk_consistent_lsn(); anyhow::ensure!( - new_disk_consistent_lsn.is_valid(), + disk_consistent_lsn.is_valid(), "Timeline {tenant_id}/{timeline_id} has invalid disk_consistent_lsn" ); + assert_eq!( + disk_consistent_lsn, + up_to_date_metadata.disk_consistent_lsn(), + "these are used interchangeably" + ); + + // Save the metadata file to local disk. + if !picked_local { + save_metadata( + self.conf, + &tenant_id, + &timeline_id, + up_to_date_metadata, + first_save, + ) + .context("save_metadata")?; + } + + let index_part = remote_startup_data.as_ref().map(|x| &x.index_part); + + if let Some(index_part) = index_part { + timeline + .remote_client + .as_ref() + .unwrap() + .init_upload_queue(index_part)?; + } else if self.remote_storage.is_some() { + // No data on the remote storage, but we have local metadata file. We can end up + // here with timeline_create being interrupted before finishing index part upload. + // By doing what we do here, the index part upload is retried. + // If control plane retries timeline creation in the meantime, the mgmt API handler + // for timeline creation will coalesce on the upload we queue here. + let rtc = timeline.remote_client.as_ref().unwrap(); + rtc.init_upload_queue_for_empty_remote(up_to_date_metadata)?; + rtc.schedule_index_upload_for_metadata_update(up_to_date_metadata)?; + } + timeline - .load_layer_map(new_disk_consistent_lsn) + .load_layer_map( + disk_consistent_lsn, + remote_startup_data.map(|x| x.index_part), + ) .await .with_context(|| { format!("Failed to load layermap for timeline {tenant_id}/{timeline_id}") @@ -458,19 +497,6 @@ impl Tenant { } }; - if self.remote_storage.is_some() { - // Reconcile local state with remote storage, downloading anything that's - // missing locally, and scheduling uploads for anything that's missing - // in remote storage. - timeline - .reconcile_with_remote( - up_to_date_metadata, - remote_startup_data.as_ref().map(|r| &r.index_part), - ) - .await - .context("failed to reconcile with remote")? - } - // Sanity check: a timeline should have some content. anyhow::ensure!( ancestor.is_some() @@ -485,18 +511,6 @@ impl Tenant { "Timeline has no ancestor and no layer files" ); - // Save the metadata file to local disk. - if !picked_local { - save_metadata( - self.conf, - &tenant_id, - &timeline_id, - up_to_date_metadata, - first_save, - ) - .context("save_metadata")?; - } - Ok(()) } @@ -691,10 +705,7 @@ impl Tenant { debug!("successfully downloaded index part for timeline {timeline_id}"); match index_part { MaybeDeletedIndexPart::IndexPart(index_part) => { - timeline_ancestors.insert( - timeline_id, - index_part.parse_metadata().context("parse_metadata")?, - ); + timeline_ancestors.insert(timeline_id, index_part.metadata.clone()); remote_index_and_client.insert(timeline_id, (index_part, client)); } MaybeDeletedIndexPart::Deleted(index_part) => { @@ -746,7 +757,7 @@ impl Tenant { DeleteTimelineFlow::resume_deletion( Arc::clone(self), timeline_id, - &index_part.parse_metadata().context("parse_metadata")?, + &index_part.metadata, Some(remote_timeline_client), self.deletion_queue_client.clone(), None, @@ -1313,10 +1324,7 @@ impl Tenant { } }; - let remote_metadata = index_part - .parse_metadata() - .context("parse_metadata") - .map_err(LoadLocalTimelineError::Load)?; + let remote_metadata = index_part.metadata.clone(); ( Some(RemoteStartupData { index_part, @@ -4122,7 +4130,7 @@ mod tests { let mut found_error_message = false; let mut err_source = err.source(); while let Some(source) = err_source { - if source.to_string() == "metadata checksum mismatch" { + if source.to_string().contains("metadata checksum mismatch") { found_error_message = true; break; } diff --git a/pageserver/src/tenant/blob_io.rs b/pageserver/src/tenant/blob_io.rs index fd211e8276..f5ff15b50c 100644 --- a/pageserver/src/tenant/blob_io.rs +++ b/pageserver/src/tenant/blob_io.rs @@ -12,14 +12,11 @@ //! len >= 128: 1XXXXXXX XXXXXXXX XXXXXXXX XXXXXXXX //! use crate::page_cache::PAGE_SZ; -use crate::tenant::block_io::{BlockCursor, BlockReader}; +use crate::tenant::block_io::BlockCursor; use std::cmp::min; use std::io::{Error, ErrorKind}; -impl BlockCursor -where - R: BlockReader, -{ +impl<'a> BlockCursor<'a> { /// Read a blob into a new buffer. pub async fn read_blob(&self, offset: u64) -> Result, std::io::Error> { let mut buf = Vec::new(); @@ -36,7 +33,7 @@ where let mut blknum = (offset / PAGE_SZ as u64) as u32; let mut off = (offset % PAGE_SZ as u64) as usize; - let mut buf = self.read_blk(blknum)?; + let mut buf = self.read_blk(blknum).await?; // peek at the first byte, to determine if it's a 1- or 4-byte length let first_len_byte = buf[off]; @@ -52,7 +49,7 @@ where // it is split across two pages len_buf[..thislen].copy_from_slice(&buf[off..PAGE_SZ]); blknum += 1; - buf = self.read_blk(blknum)?; + buf = self.read_blk(blknum).await?; len_buf[thislen..].copy_from_slice(&buf[0..4 - thislen]); off = 4 - thislen; } else { @@ -73,7 +70,7 @@ where if page_remain == 0 { // continue on next page blknum += 1; - buf = self.read_blk(blknum)?; + buf = self.read_blk(blknum).await?; off = 0; page_remain = PAGE_SZ; } diff --git a/pageserver/src/tenant/block_io.rs b/pageserver/src/tenant/block_io.rs index 503e5bd4e6..69d5b49c6d 100644 --- a/pageserver/src/tenant/block_io.rs +++ b/pageserver/src/tenant/block_io.rs @@ -2,8 +2,12 @@ //! Low-level Block-oriented I/O functions //! +use super::ephemeral_file::EphemeralFile; +use super::storage_layer::delta_layer::{Adapter, DeltaLayerInner}; use crate::page_cache::{self, PageReadGuard, ReadBufResult, PAGE_SZ}; +use crate::virtual_file::VirtualFile; use bytes::Bytes; +use std::fs::File; use std::ops::{Deref, DerefMut}; use std::os::unix::fs::FileExt; @@ -13,32 +17,20 @@ use std::os::unix::fs::FileExt; /// There are currently two implementations: EphemeralFile, and FileBlockReader /// below. pub trait BlockReader { - /// - /// Read a block. Returns a "lease" object that can be used to - /// access to the contents of the page. (For the page cache, the - /// lease object represents a lock on the buffer.) - /// - fn read_blk(&self, blknum: u32) -> Result; - /// /// Create a new "cursor" for reading from this reader. /// /// A cursor caches the last accessed page, allowing for faster /// access if the same block is accessed repeatedly. - fn block_cursor(&self) -> BlockCursor<&Self> - where - Self: Sized, - { - BlockCursor::new(self) - } + fn block_cursor(&self) -> BlockCursor<'_>; } impl BlockReader for &B where B: BlockReader, { - fn read_blk(&self, blknum: u32) -> Result { - (*self).read_blk(blknum) + fn block_cursor(&self) -> BlockCursor<'_> { + (*self).block_cursor() } } @@ -47,7 +39,7 @@ pub enum BlockLease<'a> { PageReadGuard(PageReadGuard<'static>), EphemeralFileMutableTail(&'a [u8; PAGE_SZ]), #[cfg(test)] - Rc(std::rc::Rc<[u8; PAGE_SZ]>), + Arc(std::sync::Arc<[u8; PAGE_SZ]>), } impl From> for BlockLease<'static> { @@ -57,9 +49,9 @@ impl From> for BlockLease<'static> { } #[cfg(test)] -impl<'a> From> for BlockLease<'a> { - fn from(value: std::rc::Rc<[u8; PAGE_SZ]>) -> Self { - BlockLease::Rc(value) +impl<'a> From> for BlockLease<'a> { + fn from(value: std::sync::Arc<[u8; PAGE_SZ]>) -> Self { + BlockLease::Arc(value) } } @@ -71,7 +63,35 @@ impl<'a> Deref for BlockLease<'a> { BlockLease::PageReadGuard(v) => v.deref(), BlockLease::EphemeralFileMutableTail(v) => v, #[cfg(test)] - BlockLease::Rc(v) => v.deref(), + BlockLease::Arc(v) => v.deref(), + } + } +} + +/// Provides the ability to read blocks from different sources, +/// similar to using traits for this purpose. +/// +/// Unlike traits, we also support the read function to be async though. +pub(crate) enum BlockReaderRef<'a> { + FileBlockReaderVirtual(&'a FileBlockReader), + FileBlockReaderFile(&'a FileBlockReader), + EphemeralFile(&'a EphemeralFile), + Adapter(Adapter<&'a DeltaLayerInner>), + #[cfg(test)] + TestDisk(&'a super::disk_btree::tests::TestDisk), +} + +impl<'a> BlockReaderRef<'a> { + #[inline(always)] + async fn read_blk(&self, blknum: u32) -> Result { + use BlockReaderRef::*; + match self { + FileBlockReaderVirtual(r) => r.read_blk(blknum).await, + FileBlockReaderFile(r) => r.read_blk(blknum).await, + EphemeralFile(r) => r.read_blk(blknum).await, + Adapter(r) => r.read_blk(blknum).await, + #[cfg(test)] + TestDisk(r) => r.read_blk(blknum), } } } @@ -93,23 +113,29 @@ impl<'a> Deref for BlockLease<'a> { /// // do stuff with 'buf' /// ``` /// -pub struct BlockCursor -where - R: BlockReader, -{ - reader: R, +pub struct BlockCursor<'a> { + reader: BlockReaderRef<'a>, } -impl BlockCursor -where - R: BlockReader, -{ - pub fn new(reader: R) -> Self { +impl<'a> BlockCursor<'a> { + pub(crate) fn new(reader: BlockReaderRef<'a>) -> Self { BlockCursor { reader } } + // Needed by cli + pub fn new_fileblockreader_virtual(reader: &'a FileBlockReader) -> Self { + BlockCursor { + reader: BlockReaderRef::FileBlockReaderVirtual(reader), + } + } - pub fn read_blk(&self, blknum: u32) -> Result { - self.reader.read_blk(blknum) + /// Read a block. + /// + /// Returns a "lease" object that can be used to + /// access to the contents of the page. (For the page cache, the + /// lease object represents a lock on the buffer.) + #[inline(always)] + pub async fn read_blk(&self, blknum: u32) -> Result { + self.reader.read_blk(blknum).await } } @@ -139,17 +165,17 @@ where assert!(buf.len() == PAGE_SZ); self.file.read_exact_at(buf, blkno as u64 * PAGE_SZ as u64) } -} - -impl BlockReader for FileBlockReader -where - F: FileExt, -{ - fn read_blk(&self, blknum: u32) -> Result { + /// Read a block. + /// + /// Returns a "lease" object that can be used to + /// access to the contents of the page. (For the page cache, the + /// lease object represents a lock on the buffer.) + pub async fn read_blk(&self, blknum: u32) -> Result { let cache = page_cache::get(); loop { match cache .read_immutable_buf(self.file_id, blknum) + .await .map_err(|e| { std::io::Error::new( std::io::ErrorKind::Other, @@ -170,6 +196,18 @@ where } } +impl BlockReader for FileBlockReader { + fn block_cursor(&self) -> BlockCursor<'_> { + BlockCursor::new(BlockReaderRef::FileBlockReaderFile(self)) + } +} + +impl BlockReader for FileBlockReader { + fn block_cursor(&self) -> BlockCursor<'_> { + BlockCursor::new(BlockReaderRef::FileBlockReaderVirtual(self)) + } +} + /// /// Trait for block-oriented output /// diff --git a/pageserver/src/tenant/delete.rs b/pageserver/src/tenant/delete.rs index de509cd3de..5c23af8036 100644 --- a/pageserver/src/tenant/delete.rs +++ b/pageserver/src/tenant/delete.rs @@ -7,6 +7,7 @@ use anyhow::Context; use pageserver_api::models::TenantState; use remote_storage::{DownloadError, GenericRemoteStorage, RemotePath}; use tokio::sync::OwnedMutexGuard; +use tokio_util::sync::CancellationToken; use tracing::{error, info, instrument, warn, Instrument, Span}; use utils::{ @@ -82,6 +83,8 @@ async fn create_remote_delete_mark( FAILED_UPLOAD_WARN_THRESHOLD, FAILED_REMOTE_OP_RETRIES, "mark_upload", + // TODO: use a cancellation token (https://github.com/neondatabase/neon/issues/5066) + backoff::Cancel::new(CancellationToken::new(), || unreachable!()), ) .await .context("mark_upload")?; @@ -171,6 +174,8 @@ async fn remove_tenant_remote_delete_mark( FAILED_UPLOAD_WARN_THRESHOLD, FAILED_REMOTE_OP_RETRIES, "remove_tenant_remote_delete_mark", + // TODO: use a cancellation token (https://github.com/neondatabase/neon/issues/5066) + backoff::Cancel::new(CancellationToken::new(), || unreachable!()), ) .await .context("remove_tenant_remote_delete_mark")?; @@ -252,6 +257,8 @@ pub(crate) async fn remote_delete_mark_exists( SHOULD_RESUME_DELETION_FETCH_MARK_ATTEMPTS, SHOULD_RESUME_DELETION_FETCH_MARK_ATTEMPTS, "fetch_tenant_deletion_mark", + // TODO: use a cancellation token (https://github.com/neondatabase/neon/issues/5066) + backoff::Cancel::new(CancellationToken::new(), || unreachable!()), ) .await; diff --git a/pageserver/src/tenant/disk_btree.rs b/pageserver/src/tenant/disk_btree.rs index 2064a62c56..44d6b4f87e 100644 --- a/pageserver/src/tenant/disk_btree.rs +++ b/pageserver/src/tenant/disk_btree.rs @@ -259,9 +259,10 @@ where { let mut stack = Vec::new(); stack.push((self.root_blk, None)); + let block_cursor = self.reader.block_cursor(); while let Some((node_blknum, opt_iter)) = stack.pop() { // Locate the node. - let node_buf = self.reader.read_blk(self.start_blk + node_blknum)?; + let node_buf = block_cursor.read_blk(self.start_blk + node_blknum).await?; let node = OnDiskNode::deparse(node_buf.as_ref())?; let prefix_len = node.prefix_len as usize; @@ -353,8 +354,10 @@ where stack.push((self.root_blk, String::new(), 0, 0, 0)); + let block_cursor = self.reader.block_cursor(); + while let Some((blknum, path, depth, child_idx, key_off)) = stack.pop() { - let blk = self.reader.read_blk(self.start_blk + blknum)?; + let blk = block_cursor.read_blk(self.start_blk + blknum).await?; let buf: &[u8] = blk.as_ref(); let node = OnDiskNode::::deparse(buf)?; @@ -683,27 +686,30 @@ impl BuildNode { } #[cfg(test)] -mod tests { +pub(crate) mod tests { use super::*; - use crate::tenant::block_io::BlockLease; + use crate::tenant::block_io::{BlockCursor, BlockLease, BlockReaderRef}; use rand::Rng; use std::collections::BTreeMap; use std::sync::atomic::{AtomicUsize, Ordering}; #[derive(Clone, Default)] - struct TestDisk { + pub(crate) struct TestDisk { blocks: Vec, } impl TestDisk { fn new() -> Self { Self::default() } - } - impl BlockReader for TestDisk { - fn read_blk(&self, blknum: u32) -> io::Result { + pub(crate) fn read_blk(&self, blknum: u32) -> io::Result { let mut buf = [0u8; PAGE_SZ]; buf.copy_from_slice(&self.blocks[blknum as usize]); - Ok(std::rc::Rc::new(buf).into()) + Ok(std::sync::Arc::new(buf).into()) + } + } + impl BlockReader for TestDisk { + fn block_cursor(&self) -> BlockCursor<'_> { + BlockCursor::new(BlockReaderRef::TestDisk(self)) } } impl BlockWriter for &mut TestDisk { diff --git a/pageserver/src/tenant/ephemeral_file.rs b/pageserver/src/tenant/ephemeral_file.rs index 5de9c24d90..31db3869d9 100644 --- a/pageserver/src/tenant/ephemeral_file.rs +++ b/pageserver/src/tenant/ephemeral_file.rs @@ -3,8 +3,7 @@ use crate::config::PageServerConf; use crate::page_cache::{self, PAGE_SZ}; -use crate::tenant::blob_io::BlobWriter; -use crate::tenant::block_io::{BlockLease, BlockReader}; +use crate::tenant::block_io::{BlockCursor, BlockLease, BlockReader}; use crate::virtual_file::VirtualFile; use std::cmp::min; use std::fs::OpenOptions; @@ -22,7 +21,7 @@ pub struct EphemeralFile { _tenant_id: TenantId, _timeline_id: TimelineId, file: VirtualFile, - size: u64, + len: u64, /// An ephemeral file is append-only. /// We keep the last page, which can still be modified, in [`Self::mutable_tail`]. /// The other pages, which can no longer be modified, are accessed through the page cache. @@ -53,27 +52,57 @@ impl EphemeralFile { _tenant_id: tenant_id, _timeline_id: timeline_id, file, - size: 0, + len: 0, mutable_tail: [0u8; PAGE_SZ], }) } - pub(crate) fn size(&self) -> u64 { - self.size + pub(crate) fn len(&self) -> u64 { + self.len } -} -/// Does the given filename look like an ephemeral file? -pub fn is_ephemeral_file(filename: &str) -> bool { - if let Some(rest) = filename.strip_prefix("ephemeral-") { - rest.parse::().is_ok() - } else { - false + pub(crate) async fn read_blk(&self, blknum: u32) -> Result { + let flushed_blknums = 0..self.len / PAGE_SZ as u64; + if flushed_blknums.contains(&(blknum as u64)) { + let cache = page_cache::get(); + loop { + match cache + .read_immutable_buf(self.page_cache_file_id, blknum) + .await + .map_err(|e| { + std::io::Error::new( + std::io::ErrorKind::Other, + // order path before error because error is anyhow::Error => might have many contexts + format!( + "ephemeral file: read immutable page #{}: {}: {:#}", + blknum, + self.file.path.display(), + e, + ), + ) + })? { + page_cache::ReadBufResult::Found(guard) => { + return Ok(BlockLease::PageReadGuard(guard)) + } + page_cache::ReadBufResult::NotFound(mut write_guard) => { + let buf: &mut [u8] = write_guard.deref_mut(); + debug_assert_eq!(buf.len(), PAGE_SZ); + self.file + .read_exact_at(&mut buf[..], blknum as u64 * PAGE_SZ as u64)?; + write_guard.mark_valid(); + + // Swap for read lock + continue; + } + }; + } + } else { + debug_assert_eq!(blknum as u64, self.len / PAGE_SZ as u64); + Ok(BlockLease::EphemeralFileMutableTail(&self.mutable_tail)) + } } -} -impl BlobWriter for EphemeralFile { - fn write_blob(&mut self, srcbuf: &[u8]) -> Result { + pub(crate) async fn write_blob(&mut self, srcbuf: &[u8]) -> Result { struct Writer<'a> { ephemeral_file: &'a mut EphemeralFile, /// The block to which the next [`push_bytes`] will write. @@ -84,13 +113,13 @@ impl BlobWriter for EphemeralFile { impl<'a> Writer<'a> { fn new(ephemeral_file: &'a mut EphemeralFile) -> io::Result> { Ok(Writer { - blknum: (ephemeral_file.size / PAGE_SZ as u64) as u32, - off: (ephemeral_file.size % PAGE_SZ as u64) as usize, + blknum: (ephemeral_file.len / PAGE_SZ as u64) as u32, + off: (ephemeral_file.len % PAGE_SZ as u64) as usize, ephemeral_file, }) } #[inline(always)] - fn push_bytes(&mut self, src: &[u8]) -> Result<(), io::Error> { + async fn push_bytes(&mut self, src: &[u8]) -> Result<(), io::Error> { let mut src_remaining = src; while !src_remaining.is_empty() { let dst_remaining = &mut self.ephemeral_file.mutable_tail[self.off..]; @@ -107,10 +136,13 @@ impl BlobWriter for EphemeralFile { // Pre-warm the page cache with what we just wrote. // This isn't necessary for coherency/correctness, but it's how we've always done it. let cache = page_cache::get(); - match cache.read_immutable_buf( - self.ephemeral_file.page_cache_file_id, - self.blknum, - ) { + match cache + .read_immutable_buf( + self.ephemeral_file.page_cache_file_id, + self.blknum, + ) + .await + { Ok(page_cache::ReadBufResult::Found(_guard)) => { // This function takes &mut self, so, it shouldn't be possible to reach this point. unreachable!("we just wrote blknum {} and this function takes &mut self, so, no concurrent read_blk is possible", self.blknum); @@ -154,39 +186,47 @@ impl BlobWriter for EphemeralFile { } } - let pos = self.size; + let pos = self.len; let mut writer = Writer::new(self)?; // Write the length field if srcbuf.len() < 0x80 { // short one-byte length header let len_buf = [srcbuf.len() as u8]; - writer.push_bytes(&len_buf)?; + writer.push_bytes(&len_buf).await?; } else { let mut len_buf = u32::to_be_bytes(srcbuf.len() as u32); len_buf[0] |= 0x80; - writer.push_bytes(&len_buf)?; + writer.push_bytes(&len_buf).await?; } // Write the payload - writer.push_bytes(srcbuf)?; + writer.push_bytes(srcbuf).await?; if srcbuf.len() < 0x80 { - self.size += 1; + self.len += 1; } else { - self.size += 4; + self.len += 4; } - self.size += srcbuf.len() as u64; + self.len += srcbuf.len() as u64; Ok(pos) } } +/// Does the given filename look like an ephemeral file? +pub fn is_ephemeral_file(filename: &str) -> bool { + if let Some(rest) = filename.strip_prefix("ephemeral-") { + rest.parse::().is_ok() + } else { + false + } +} + impl Drop for EphemeralFile { fn drop(&mut self) { - // drop all pages from page cache - let cache = page_cache::get(); - cache.drop_buffers_for_immutable(self.page_cache_file_id); + // There might still be pages in the [`crate::page_cache`] for this file. + // We leave them there, [`crate::page_cache::PageCache::find_victim`] will evict them when needed. // unlink the file let res = std::fs::remove_file(&self.file.path); @@ -207,52 +247,15 @@ impl Drop for EphemeralFile { } impl BlockReader for EphemeralFile { - fn read_blk(&self, blknum: u32) -> Result { - let flushed_blknums = 0..self.size / PAGE_SZ as u64; - if flushed_blknums.contains(&(blknum as u64)) { - let cache = page_cache::get(); - loop { - match cache - .read_immutable_buf(self.page_cache_file_id, blknum) - .map_err(|e| { - std::io::Error::new( - std::io::ErrorKind::Other, - // order path before error because error is anyhow::Error => might have many contexts - format!( - "ephemeral file: read immutable page #{}: {}: {:#}", - blknum, - self.file.path.display(), - e, - ), - ) - })? { - page_cache::ReadBufResult::Found(guard) => { - return Ok(BlockLease::PageReadGuard(guard)) - } - page_cache::ReadBufResult::NotFound(mut write_guard) => { - let buf: &mut [u8] = write_guard.deref_mut(); - debug_assert_eq!(buf.len(), PAGE_SZ); - self.file - .read_exact_at(&mut buf[..], blknum as u64 * PAGE_SZ as u64)?; - write_guard.mark_valid(); - - // Swap for read lock - continue; - } - }; - } - } else { - debug_assert_eq!(blknum as u64, self.size / PAGE_SZ as u64); - Ok(BlockLease::EphemeralFileMutableTail(&self.mutable_tail)) - } + fn block_cursor(&self) -> super::block_io::BlockCursor<'_> { + BlockCursor::new(super::block_io::BlockReaderRef::EphemeralFile(self)) } } #[cfg(test)] mod tests { use super::*; - use crate::tenant::blob_io::BlobWriter; - use crate::tenant::block_io::BlockCursor; + use crate::tenant::block_io::{BlockCursor, BlockReaderRef}; use rand::{thread_rng, RngCore}; use std::fs; use std::str::FromStr; @@ -280,12 +283,12 @@ mod tests { let mut file = EphemeralFile::create(conf, tenant_id, timeline_id)?; - let pos_foo = file.write_blob(b"foo")?; + let pos_foo = file.write_blob(b"foo").await?; assert_eq!( b"foo", file.block_cursor().read_blob(pos_foo).await?.as_slice() ); - let pos_bar = file.write_blob(b"bar")?; + let pos_bar = file.write_blob(b"bar").await?; assert_eq!( b"foo", file.block_cursor().read_blob(pos_foo).await?.as_slice() @@ -298,17 +301,17 @@ mod tests { let mut blobs = Vec::new(); for i in 0..10000 { let data = Vec::from(format!("blob{}", i).as_bytes()); - let pos = file.write_blob(&data)?; + let pos = file.write_blob(&data).await?; blobs.push((pos, data)); } // also test with a large blobs for i in 0..100 { let data = format!("blob{}", i).as_bytes().repeat(100); - let pos = file.write_blob(&data)?; + let pos = file.write_blob(&data).await?; blobs.push((pos, data)); } - let cursor = BlockCursor::new(&file); + let cursor = BlockCursor::new(BlockReaderRef::EphemeralFile(&file)); for (pos, expected) in blobs { let actual = cursor.read_blob(pos).await?; assert_eq!(actual, expected); @@ -318,7 +321,7 @@ mod tests { let mut large_data = Vec::new(); large_data.resize(20000, 0); thread_rng().fill_bytes(&mut large_data); - let pos_large = file.write_blob(&large_data)?; + let pos_large = file.write_blob(&large_data).await?; let result = file.block_cursor().read_blob(pos_large).await?; assert_eq!(result, large_data); diff --git a/pageserver/src/tenant/layer_map.rs b/pageserver/src/tenant/layer_map.rs index 59813b19d5..b8185da010 100644 --- a/pageserver/src/tenant/layer_map.rs +++ b/pageserver/src/tenant/layer_map.rs @@ -50,7 +50,6 @@ use crate::context::RequestContext; use crate::keyspace::KeyPartitioning; use crate::repository::Key; use crate::tenant::storage_layer::InMemoryLayer; -use crate::tenant::storage_layer::Layer; use anyhow::Result; use std::collections::VecDeque; use std::ops::Range; diff --git a/pageserver/src/tenant/metadata.rs b/pageserver/src/tenant/metadata.rs index a27602c335..dbf2d5ac37 100644 --- a/pageserver/src/tenant/metadata.rs +++ b/pageserver/src/tenant/metadata.rs @@ -12,7 +12,7 @@ use std::fs::{File, OpenOptions}; use std::io::{self, Write}; use anyhow::{bail, ensure, Context}; -use serde::{Deserialize, Serialize}; +use serde::{de::Error, Deserialize, Serialize, Serializer}; use thiserror::Error; use tracing::info_span; use utils::bin_ser::SerializeError; @@ -232,6 +232,28 @@ impl TimelineMetadata { } } +impl<'de> Deserialize<'de> for TimelineMetadata { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + let bytes = Vec::::deserialize(deserializer)?; + Self::from_bytes(bytes.as_slice()).map_err(|e| D::Error::custom(format!("{e}"))) + } +} + +impl Serialize for TimelineMetadata { + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + let bytes = self + .to_bytes() + .map_err(|e| serde::ser::Error::custom(format!("{e}")))?; + bytes.serialize(serializer) + } +} + /// Save timeline metadata to file pub fn save_metadata( conf: &'static PageServerConf, diff --git a/pageserver/src/tenant/remote_timeline_client.rs b/pageserver/src/tenant/remote_timeline_client.rs index ab116de090..13d50d90b8 100644 --- a/pageserver/src/tenant/remote_timeline_client.rs +++ b/pageserver/src/tenant/remote_timeline_client.rs @@ -139,7 +139,7 @@ //! - Initiate upload queue with that [`IndexPart`]. //! - Reschedule all lost operations by comparing the local filesystem state //! and remote state as per [`IndexPart`]. This is done in -//! [`Tenant::timeline_init_and_sync`] and [`Timeline::reconcile_with_remote`]. +//! [`Tenant::timeline_init_and_sync`]. //! //! Note that if we crash during file deletion between the index update //! that removes the file from the list of files, and deleting the remote file, @@ -176,7 +176,6 @@ //! transitioning it from `TenantState::Attaching` to `TenantState::Active` state. //! This starts the timelines' WAL-receivers and the tenant's GC & Compaction loops. //! -//! Most of the above steps happen in [`Timeline::reconcile_with_remote`] or its callers. //! We keep track of the fact that a client is in `Attaching` state in a marker //! file on the local disk. This is critical because, when we restart the pageserver, //! we do not want to do the `List timelines` step for each tenant that has already @@ -196,14 +195,14 @@ //! not created and the uploads are skipped. //! Theoretically, it should be ok to remove and re-add remote storage configuration to //! the pageserver config at any time, since it doesn't make a difference to -//! `reconcile_with_remote`. +//! [`Timeline::load_layer_map`]. //! Of course, the remote timeline dir must not change while we have de-configured //! remote storage, i.e., the pageserver must remain the owner of the given prefix //! in remote storage. //! But note that we don't test any of this right now. //! //! [`Tenant::timeline_init_and_sync`]: super::Tenant::timeline_init_and_sync -//! [`Timeline::reconcile_with_remote`]: super::Timeline::reconcile_with_remote +//! [`Timeline::load_layer_map`]: super::Timeline::load_layer_map mod download; pub mod index; @@ -214,6 +213,7 @@ use chrono::{NaiveDateTime, Utc}; // re-export these pub use download::{is_temp_download_file, list_remote_timelines}; use scopeguard::ScopeGuard; +use tokio_util::sync::CancellationToken; use utils::backoff::{ self, exponential_backoff, DEFAULT_BASE_BACKOFF_SECONDS, DEFAULT_MAX_BACKOFF_SECONDS, }; @@ -235,6 +235,7 @@ use crate::metrics::{ RemoteTimelineClientMetricsCallTrackSize, REMOTE_ONDEMAND_DOWNLOADED_BYTES, REMOTE_ONDEMAND_DOWNLOADED_LAYERS, }; +use crate::task_mgr::shutdown_token; use crate::tenant::debug_assert_current_span_has_tenant_and_timeline_id; use crate::tenant::remote_timeline_client::index::LayerFileMetadata; use crate::{ @@ -356,6 +357,10 @@ impl RemoteTimelineClient { let mut upload_queue = self.upload_queue.lock().unwrap(); upload_queue.initialize_with_current_remote_index_part(index_part)?; self.update_remote_physical_size_gauge(Some(index_part)); + info!( + "initialized upload queue from remote index with {} layer files", + index_part.layer_metadata.len() + ); Ok(()) } @@ -368,6 +373,7 @@ impl RemoteTimelineClient { let mut upload_queue = self.upload_queue.lock().unwrap(); upload_queue.initialize_empty_remote(local_metadata)?; self.update_remote_physical_size_gauge(None); + info!("initialized upload queue as empty"); Ok(()) } @@ -538,8 +544,7 @@ impl RemoteTimelineClient { // ahead of what's _actually_ on the remote during index upload. upload_queue.latest_metadata = metadata.clone(); - let metadata_bytes = upload_queue.latest_metadata.to_bytes()?; - self.schedule_index_upload(upload_queue, metadata_bytes); + self.schedule_index_upload(upload_queue, upload_queue.latest_metadata.clone()); Ok(()) } @@ -559,8 +564,7 @@ impl RemoteTimelineClient { let upload_queue = guard.initialized_mut()?; if upload_queue.latest_files_changes_since_metadata_upload_scheduled > 0 { - let metadata_bytes = upload_queue.latest_metadata.to_bytes()?; - self.schedule_index_upload(upload_queue, metadata_bytes); + self.schedule_index_upload(upload_queue, upload_queue.latest_metadata.clone()); } Ok(()) @@ -570,7 +574,7 @@ impl RemoteTimelineClient { fn schedule_index_upload( self: &Arc, upload_queue: &mut UploadQueueInitialized, - metadata_bytes: Vec, + metadata: TimelineMetadata, ) { info!( "scheduling metadata upload with {} files ({} changed)", @@ -583,7 +587,7 @@ impl RemoteTimelineClient { let index_part = IndexPart::new( upload_queue.latest_files.clone(), disk_consistent_lsn, - metadata_bytes, + metadata, ); let op = UploadOp::UploadMetadata(index_part, disk_consistent_lsn); self.calls_unfinished_metric_begin(&op); @@ -642,17 +646,18 @@ impl RemoteTimelineClient { // Deleting layers doesn't affect the values stored in TimelineMetadata, // so we don't need update it. Just serialize it. - let metadata_bytes = upload_queue.latest_metadata.to_bytes()?; + let metadata = upload_queue.latest_metadata.clone(); // Update the remote index file, removing the to-be-deleted files from the index, // before deleting the actual files. for name in names { - upload_queue.latest_files.remove(name); - upload_queue.latest_files_changes_since_metadata_upload_scheduled += 1; + if upload_queue.latest_files.remove(name).is_some() { + upload_queue.latest_files_changes_since_metadata_upload_scheduled += 1; + } } if upload_queue.latest_files_changes_since_metadata_upload_scheduled > 0 { - self.schedule_index_upload(upload_queue, metadata_bytes); + self.schedule_index_upload(upload_queue, metadata); } } @@ -752,7 +757,7 @@ impl RemoteTimelineClient { pausable_failpoint!("persist_deleted_index_part"); backoff::retry( - || async { + || { upload::upload_index_part( self.conf, &self.storage_impl, @@ -760,7 +765,6 @@ impl RemoteTimelineClient { &self.timeline_id, &index_part_with_deleted_at, ) - .await }, |_e| false, 1, @@ -769,6 +773,8 @@ impl RemoteTimelineClient { // when executed as part of tenant deletion this happens in the background 2, "persist_index_part_with_deleted_flag", + // TODO: use a cancellation token (https://github.com/neondatabase/neon/issues/5066) + backoff::Cancel::new(CancellationToken::new(), || unreachable!()), ) .await?; @@ -852,6 +858,7 @@ impl RemoteTimelineClient { FAILED_DOWNLOAD_WARN_THRESHOLD, FAILED_REMOTE_OP_RETRIES, "list_prefixes", + backoff::Cancel::new(shutdown_token(), || anyhow::anyhow!("Cancelled!")), ) .await .context("list prefixes")?; @@ -1044,6 +1051,15 @@ impl RemoteTimelineClient { .await } UploadOp::UploadMetadata(ref index_part, _lsn) => { + let mention_having_future_layers = if cfg!(feature = "testing") { + index_part + .layer_metadata + .keys() + .any(|x| x.is_in_future(*_lsn)) + } else { + false + }; + let res = upload::upload_index_part( self.conf, &self.storage_impl, @@ -1061,6 +1077,10 @@ impl RemoteTimelineClient { .await; if res.is_ok() { self.update_remote_physical_size_gauge(Some(index_part)); + if mention_having_future_layers { + // find rationale near crate::tenant::timeline::init::cleanup_future_layer + tracing::info!(disk_consistent_lsn=%_lsn, "uploaded an index_part.json with future layers -- this is ok! if shutdown now, expect future layer cleanup"); + } } res } @@ -1097,14 +1117,13 @@ impl RemoteTimelineClient { } // sleep until it's time to retry, or we're cancelled - tokio::select! { - _ = task_mgr::shutdown_watcher() => { }, - _ = exponential_backoff( - retries, - DEFAULT_BASE_BACKOFF_SECONDS, - DEFAULT_MAX_BACKOFF_SECONDS, - ) => { }, - }; + exponential_backoff( + retries, + DEFAULT_BASE_BACKOFF_SECONDS, + DEFAULT_MAX_BACKOFF_SECONDS, + &shutdown_token(), + ) + .await; } } } @@ -1536,8 +1555,7 @@ mod tests { &layer_file_name_2.file_name(), ], ); - let downloaded_metadata = index_part.parse_metadata().unwrap(); - assert_eq!(downloaded_metadata, metadata); + assert_eq!(index_part.metadata, metadata); // Schedule upload and then a deletion. Check that the deletion is queued client diff --git a/pageserver/src/tenant/remote_timeline_client/download.rs b/pageserver/src/tenant/remote_timeline_client/download.rs index 831dc62f33..ef74640902 100644 --- a/pageserver/src/tenant/remote_timeline_client/download.rs +++ b/pageserver/src/tenant/remote_timeline_client/download.rs @@ -11,6 +11,7 @@ use std::time::Duration; use anyhow::{anyhow, Context}; use tokio::fs; use tokio::io::AsyncWriteExt; +use tokio_util::sync::CancellationToken; use utils::{backoff, crashsafe}; use crate::config::PageServerConf; @@ -266,6 +267,10 @@ where FAILED_DOWNLOAD_WARN_THRESHOLD, FAILED_REMOTE_OP_RETRIES, description, + // TODO: use a cancellation token (https://github.com/neondatabase/neon/issues/5066) + backoff::Cancel::new(CancellationToken::new(), || -> DownloadError { + unreachable!() + }), ) .await } diff --git a/pageserver/src/tenant/remote_timeline_client/index.rs b/pageserver/src/tenant/remote_timeline_client/index.rs index 8985ab0865..28177b097f 100644 --- a/pageserver/src/tenant/remote_timeline_client/index.rs +++ b/pageserver/src/tenant/remote_timeline_client/index.rs @@ -77,7 +77,9 @@ pub struct IndexPart { // private because internally we would read from metadata instead. #[serde_as(as = "DisplayFromStr")] disk_consistent_lsn: Lsn, - metadata_bytes: Vec, + + #[serde(rename = "metadata_bytes")] + pub metadata: TimelineMetadata, } impl IndexPart { @@ -95,7 +97,7 @@ impl IndexPart { pub fn new( layers_and_metadata: HashMap, disk_consistent_lsn: Lsn, - metadata_bytes: Vec, + metadata: TimelineMetadata, ) -> Self { let mut timeline_layers = HashSet::with_capacity(layers_and_metadata.len()); let mut layer_metadata = HashMap::with_capacity(layers_and_metadata.len()); @@ -111,14 +113,10 @@ impl IndexPart { timeline_layers, layer_metadata, disk_consistent_lsn, - metadata_bytes, + metadata, deleted_at: None, } } - - pub fn parse_metadata(&self) -> anyhow::Result { - TimelineMetadata::from_bytes(&self.metadata_bytes) - } } impl TryFrom<&UploadQueueInitialized> for IndexPart { @@ -126,12 +124,12 @@ impl TryFrom<&UploadQueueInitialized> for IndexPart { fn try_from(upload_queue: &UploadQueueInitialized) -> Result { let disk_consistent_lsn = upload_queue.latest_metadata.disk_consistent_lsn(); - let metadata_bytes = upload_queue.latest_metadata.to_bytes()?; + let metadata = upload_queue.latest_metadata.clone(); Ok(Self::new( upload_queue.latest_files.clone(), disk_consistent_lsn, - metadata_bytes, + metadata, )) } } @@ -182,7 +180,7 @@ mod tests { }) ]), disk_consistent_lsn: "0/16960E8".parse::().unwrap(), - metadata_bytes: [113,11,159,210,0,54,0,4,0,0,0,0,1,105,96,232,1,0,0,0,0,1,105,96,112,0,0,0,0,0,0,0,0,0,0,0,0,0,1,105,96,112,0,0,0,0,1,105,96,112,0,0,0,14,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0].to_vec(), + metadata: TimelineMetadata::from_bytes(&[113,11,159,210,0,54,0,4,0,0,0,0,1,105,96,232,1,0,0,0,0,1,105,96,112,0,0,0,0,0,0,0,0,0,0,0,0,0,1,105,96,112,0,0,0,0,1,105,96,112,0,0,0,14,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]).unwrap(), deleted_at: None, }; @@ -201,7 +199,7 @@ mod tests { "000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__00000000016B59D8-00000000016B5A51": { "file_size": 9007199254741001 } }, "disk_consistent_lsn":"0/16960E8", - "metadata_bytes":[112,11,159,210,0,54,0,4,0,0,0,0,1,105,96,232,1,0,0,0,0,1,105,96,112,0,0,0,0,0,0,0,0,0,0,0,0,0,1,105,96,112,0,0,0,0,1,105,96,112,0,0,0,14,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0] + "metadata_bytes":[113,11,159,210,0,54,0,4,0,0,0,0,1,105,96,232,1,0,0,0,0,1,105,96,112,0,0,0,0,0,0,0,0,0,0,0,0,0,1,105,96,112,0,0,0,0,1,105,96,112,0,0,0,14,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0] }"#; let expected = IndexPart { @@ -219,7 +217,7 @@ mod tests { }) ]), disk_consistent_lsn: "0/16960E8".parse::().unwrap(), - metadata_bytes: [112,11,159,210,0,54,0,4,0,0,0,0,1,105,96,232,1,0,0,0,0,1,105,96,112,0,0,0,0,0,0,0,0,0,0,0,0,0,1,105,96,112,0,0,0,0,1,105,96,112,0,0,0,14,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0].to_vec(), + metadata: TimelineMetadata::from_bytes(&[113,11,159,210,0,54,0,4,0,0,0,0,1,105,96,232,1,0,0,0,0,1,105,96,112,0,0,0,0,0,0,0,0,0,0,0,0,0,1,105,96,112,0,0,0,0,1,105,96,112,0,0,0,14,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]).unwrap(), deleted_at: None, }; @@ -238,7 +236,7 @@ mod tests { "000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__00000000016B59D8-00000000016B5A51": { "file_size": 9007199254741001 } }, "disk_consistent_lsn":"0/16960E8", - "metadata_bytes":[112,11,159,210,0,54,0,4,0,0,0,0,1,105,96,232,1,0,0,0,0,1,105,96,112,0,0,0,0,0,0,0,0,0,0,0,0,0,1,105,96,112,0,0,0,0,1,105,96,112,0,0,0,14,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0], + "metadata_bytes":[113,11,159,210,0,54,0,4,0,0,0,0,1,105,96,232,1,0,0,0,0,1,105,96,112,0,0,0,0,0,0,0,0,0,0,0,0,0,1,105,96,112,0,0,0,0,1,105,96,112,0,0,0,14,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0], "deleted_at": "2023-07-31T09:00:00.123" }"#; @@ -257,7 +255,7 @@ mod tests { }) ]), disk_consistent_lsn: "0/16960E8".parse::().unwrap(), - metadata_bytes: [112,11,159,210,0,54,0,4,0,0,0,0,1,105,96,232,1,0,0,0,0,1,105,96,112,0,0,0,0,0,0,0,0,0,0,0,0,0,1,105,96,112,0,0,0,0,1,105,96,112,0,0,0,14,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0].to_vec(), + metadata: TimelineMetadata::from_bytes(&[113,11,159,210,0,54,0,4,0,0,0,0,1,105,96,232,1,0,0,0,0,1,105,96,112,0,0,0,0,0,0,0,0,0,0,0,0,0,1,105,96,112,0,0,0,0,1,105,96,112,0,0,0,14,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]).unwrap(), deleted_at: Some(chrono::NaiveDateTime::parse_from_str( "2023-07-31T09:00:00.123000000", "%Y-%m-%dT%H:%M:%S.%f").unwrap()) }; @@ -281,7 +279,7 @@ mod tests { timeline_layers: HashSet::new(), layer_metadata: HashMap::new(), disk_consistent_lsn: "0/2532648".parse::().unwrap(), - metadata_bytes: [ + metadata: TimelineMetadata::from_bytes(&[ 136, 151, 49, 208, 0, 70, 0, 4, 0, 0, 0, 0, 2, 83, 38, 72, 1, 0, 0, 0, 0, 2, 83, 38, 32, 1, 87, 198, 240, 135, 97, 119, 45, 125, 38, 29, 155, 161, 140, 141, 255, 210, 0, 0, 0, 0, 2, 83, 38, 72, 0, 0, 0, 0, 1, 73, 240, 192, 0, 0, 0, 0, 1, 73, @@ -302,8 +300,8 @@ mod tests { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - ] - .to_vec(), + ]) + .unwrap(), deleted_at: None, }; diff --git a/pageserver/src/tenant/storage_layer.rs b/pageserver/src/tenant/storage_layer.rs index 4107161aa1..a39e041eaf 100644 --- a/pageserver/src/tenant/storage_layer.rs +++ b/pageserver/src/tenant/storage_layer.rs @@ -41,8 +41,6 @@ pub use inmemory_layer::InMemoryLayer; pub use layer_desc::{PersistentLayerDesc, PersistentLayerKey}; pub use remote_layer::RemoteLayer; -use super::timeline::layer_manager::LayerManager; - pub fn range_overlaps(a: &Range, b: &Range) -> bool where T: PartialOrd, @@ -175,16 +173,9 @@ impl LayerAccessStats { /// /// [`LayerLoad`]: LayerResidenceEventReason::LayerLoad /// [`record_residence_event`]: Self::record_residence_event - pub(crate) fn for_loading_layer( - layer_map_lock_held_witness: &LayerManager, - status: LayerResidenceStatus, - ) -> Self { + pub(crate) fn for_loading_layer(status: LayerResidenceStatus) -> Self { let new = LayerAccessStats(Mutex::new(LayerAccessStatsLocked::default())); - new.record_residence_event( - layer_map_lock_held_witness, - status, - LayerResidenceEventReason::LayerLoad, - ); + new.record_residence_event(status, LayerResidenceEventReason::LayerLoad); new } @@ -197,7 +188,6 @@ impl LayerAccessStats { /// [`record_residence_event`]: Self::record_residence_event pub(crate) fn clone_for_residence_change( &self, - layer_map_lock_held_witness: &LayerManager, new_status: LayerResidenceStatus, ) -> LayerAccessStats { let clone = { @@ -205,11 +195,7 @@ impl LayerAccessStats { inner.clone() }; let new = LayerAccessStats(Mutex::new(clone)); - new.record_residence_event( - layer_map_lock_held_witness, - new_status, - LayerResidenceEventReason::ResidenceChange, - ); + new.record_residence_event(new_status, LayerResidenceEventReason::ResidenceChange); new } @@ -229,7 +215,6 @@ impl LayerAccessStats { /// pub(crate) fn record_residence_event( &self, - _layer_map_lock_held_witness: &LayerManager, status: LayerResidenceStatus, reason: LayerResidenceEventReason, ) { @@ -344,23 +329,6 @@ impl LayerAccessStats { /// are used in (timeline). #[async_trait::async_trait] pub trait Layer: std::fmt::Debug + std::fmt::Display + Send + Sync + 'static { - /// Range of keys that this layer covers - fn get_key_range(&self) -> Range; - - /// Inclusive start bound of the LSN range that this layer holds - /// Exclusive end bound of the LSN range that this layer holds. - /// - /// - For an open in-memory layer, this is MAX_LSN. - /// - For a frozen in-memory layer or a delta layer, this is a valid end bound. - /// - An image layer represents snapshot at one LSN, so end_lsn is always the snapshot LSN + 1 - fn get_lsn_range(&self) -> Range; - - /// Does this layer only contain some data for the key-range (incremental), - /// or does it contain a version of every page? This is important to know - /// for garbage collecting old layers: an incremental layer depends on - /// the previous non-incremental layer. - fn is_incremental(&self) -> bool; - /// /// Return data needed to reconstruct given page at LSN. /// @@ -380,9 +348,6 @@ pub trait Layer: std::fmt::Debug + std::fmt::Display + Send + Sync + 'static { reconstruct_data: &mut ValueReconstructState, ctx: &RequestContext, ) -> Result; - - /// Dump summary of the contents of the layer to stdout - async fn dump(&self, verbose: bool, ctx: &RequestContext) -> Result<()>; } /// Get a layer descriptor from a layer. @@ -467,7 +432,6 @@ pub mod tests { TimelineId::from_array([0; 16]), value.key_range, value.lsn, - false, 233, ) } diff --git a/pageserver/src/tenant/storage_layer/delta_layer.rs b/pageserver/src/tenant/storage_layer/delta_layer.rs index 6094a7db52..d9df346a14 100644 --- a/pageserver/src/tenant/storage_layer/delta_layer.rs +++ b/pageserver/src/tenant/storage_layer/delta_layer.rs @@ -239,8 +239,54 @@ impl std::fmt::Debug for DeltaLayerInner { #[async_trait::async_trait] impl Layer for DeltaLayer { - /// debugging function to print out the contents of the layer - async fn dump(&self, verbose: bool, ctx: &RequestContext) -> Result<()> { + async fn get_value_reconstruct_data( + &self, + key: Key, + lsn_range: Range, + reconstruct_state: &mut ValueReconstructState, + ctx: &RequestContext, + ) -> anyhow::Result { + self.get_value_reconstruct_data(key, lsn_range, reconstruct_state, ctx) + .await + } +} +/// Boilerplate to implement the Layer trait, always use layer_desc for persistent layers. +impl std::fmt::Display for DeltaLayer { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", self.layer_desc().short_id()) + } +} + +impl AsLayerDesc for DeltaLayer { + fn layer_desc(&self) -> &PersistentLayerDesc { + &self.desc + } +} + +impl PersistentLayer for DeltaLayer { + fn downcast_delta_layer(self: Arc) -> Option> { + Some(self) + } + + fn local_path(&self) -> Option { + self.local_path() + } + + fn delete_resident_layer_file(&self) -> Result<()> { + self.delete_resident_layer_file() + } + + fn info(&self, reset: LayerAccessStatsReset) -> HistoricLayerInfo { + self.info(reset) + } + + fn access_stats(&self) -> &LayerAccessStats { + self.access_stats() + } +} + +impl DeltaLayer { + pub(crate) async fn dump(&self, verbose: bool, ctx: &RequestContext) -> Result<()> { println!( "----- delta layer for ten {} tli {} keys {}-{} lsn {}-{} size {} ----", self.desc.tenant_id, @@ -272,30 +318,28 @@ impl Layer for DeltaLayer { tree_reader.dump().await?; - let keys = DeltaLayerInner::load_keys(&Ref(&**inner)).await?; + let keys = DeltaLayerInner::load_keys(&inner).await?; // A subroutine to dump a single blob - let dump_blob = |val: ValueRef<_>| -> _ { - async move { - let buf = val.reader.read_blob(val.blob_ref.pos()).await?; - let val = Value::des(&buf)?; - let desc = match val { - Value::Image(img) => { - format!(" img {} bytes", img.len()) - } - Value::WalRecord(rec) => { - let wal_desc = walrecord::describe_wal_record(&rec)?; - format!( - " rec {} bytes will_init: {} {}", - buf.len(), - rec.will_init(), - wal_desc - ) - } - }; - Ok(desc) - } - }; + async fn dump_blob(val: ValueRef<'_>) -> Result { + let buf = val.reader.read_blob(val.blob_ref.pos()).await?; + let val = Value::des(&buf)?; + let desc = match val { + Value::Image(img) => { + format!(" img {} bytes", img.len()) + } + Value::WalRecord(rec) => { + let wal_desc = walrecord::describe_wal_record(&rec)?; + format!( + " rec {} bytes will_init: {} {}", + buf.len(), + rec.will_init(), + wal_desc + ) + } + }; + Ok(desc) + } for entry in keys { let DeltaEntry { key, lsn, val, .. } = entry; @@ -312,7 +356,7 @@ impl Layer for DeltaLayer { Ok(()) } - async fn get_value_reconstruct_data( + pub(crate) async fn get_value_reconstruct_data( &self, key: Key, lsn_range: Range, @@ -331,52 +375,19 @@ impl Layer for DeltaLayer { .await } - /// Boilerplate to implement the Layer trait, always use layer_desc for persistent layers. - fn get_key_range(&self) -> Range { - self.layer_desc().key_range.clone() - } - - /// Boilerplate to implement the Layer trait, always use layer_desc for persistent layers. - fn get_lsn_range(&self) -> Range { - self.layer_desc().lsn_range.clone() - } - - /// Boilerplate to implement the Layer trait, always use layer_desc for persistent layers. - fn is_incremental(&self) -> bool { - self.layer_desc().is_incremental - } -} -/// Boilerplate to implement the Layer trait, always use layer_desc for persistent layers. -impl std::fmt::Display for DeltaLayer { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "{}", self.layer_desc().short_id()) - } -} - -impl AsLayerDesc for DeltaLayer { - fn layer_desc(&self) -> &PersistentLayerDesc { - &self.desc - } -} - -impl PersistentLayer for DeltaLayer { - fn downcast_delta_layer(self: Arc) -> Option> { - Some(self) - } - - fn local_path(&self) -> Option { + pub(crate) fn local_path(&self) -> Option { Some(self.path()) } - fn delete_resident_layer_file(&self) -> Result<()> { + pub(crate) fn delete_resident_layer_file(&self) -> Result<()> { // delete underlying file fs::remove_file(self.path())?; Ok(()) } - fn info(&self, reset: LayerAccessStatsReset) -> HistoricLayerInfo { - let layer_file_name = self.filename().file_name(); - let lsn_range = self.get_lsn_range(); + pub(crate) fn info(&self, reset: LayerAccessStatsReset) -> HistoricLayerInfo { + let layer_file_name = self.layer_desc().filename().file_name(); + let lsn_range = self.layer_desc().lsn_range.clone(); let access_stats = self.access_stats.as_api_model(reset); @@ -390,12 +401,10 @@ impl PersistentLayer for DeltaLayer { } } - fn access_stats(&self) -> &LayerAccessStats { + pub(crate) fn access_stats(&self) -> &LayerAccessStats { &self.access_stats } -} -impl DeltaLayer { fn path_for( path_or_conf: &PathOrConf, tenant_id: &TenantId, @@ -458,7 +467,7 @@ impl DeltaLayer { PathOrConf::Path(_) => None, }; - let loaded = DeltaLayerInner::load(&path, summary)?; + let loaded = DeltaLayerInner::load(&path, summary).await?; if let PathOrConf::Path(ref path) = self.path_or_conf { // not production code @@ -541,17 +550,12 @@ impl DeltaLayer { /// Loads all keys stored in the layer. Returns key, lsn, value size and value reference. /// /// The value can be obtained via the [`ValueRef::load`] function. - pub(crate) async fn load_keys( - &self, - ctx: &RequestContext, - ) -> Result>>> { + pub(crate) async fn load_keys(&self, ctx: &RequestContext) -> Result>> { let inner = self .load(LayerAccessKind::KeyIter, ctx) .await .context("load delta layer keys")?; - - let inner = Ref(&**inner); - DeltaLayerInner::load_keys(&inner) + DeltaLayerInner::load_keys(inner) .await .context("Layer index is corrupted") } @@ -837,12 +841,15 @@ impl Drop for DeltaLayerWriter { } impl DeltaLayerInner { - pub(super) fn load(path: &std::path::Path, summary: Option) -> anyhow::Result { + pub(super) async fn load( + path: &std::path::Path, + summary: Option, + ) -> anyhow::Result { let file = VirtualFile::open(path) .with_context(|| format!("Failed to open file '{}'", path.display()))?; let file = FileBlockReader::new(file); - let summary_blk = file.read_blk(0)?; + let summary_blk = file.read_blk(0).await?; let actual_summary = Summary::des_prefix(summary_blk.as_ref())?; if let Some(mut expected_summary) = summary { @@ -947,14 +954,14 @@ impl DeltaLayerInner { pub(super) async fn load_keys + Clone>( this: &T, - ) -> Result>> { + ) -> Result>> { let dl = this.as_ref(); let file = &dl.file; let tree_reader = DiskBtreeReader::<_, DELTA_KEY_SIZE>::new(dl.index_start_blk, dl.index_root_blk, file); - let mut all_keys: Vec> = Vec::new(); + let mut all_keys: Vec> = Vec::new(); tree_reader .visit( @@ -964,7 +971,9 @@ impl DeltaLayerInner { let delta_key = DeltaKey::from_slice(key); let val_ref = ValueRef { blob_ref: BlobRef(value), - reader: BlockCursor::new(Adapter(this.clone())), + reader: BlockCursor::new(crate::tenant::block_io::BlockReaderRef::Adapter( + Adapter(dl), + )), }; let pos = BlobRef(value).pos(); if let Some(last) = all_keys.last_mut() { @@ -993,43 +1002,23 @@ impl DeltaLayerInner { } } -/// Cloneable borrow wrapper to make borrows behave like smart pointers. -/// -/// Shared references are trivially copyable. This wrapper avoids (confusion) to otherwise attempt -/// cloning DeltaLayerInner. -pub(crate) struct Ref(T); - -impl<'a, T> AsRef for Ref<&'a T> { - fn as_ref(&self) -> &T { - self.0 - } -} - -impl<'a, T> Clone for Ref<&'a T> { - fn clone(&self) -> Self { - *self - } -} - -impl<'a, T> Copy for Ref<&'a T> {} - /// A set of data associated with a delta layer key and its value -pub struct DeltaEntry> { +pub struct DeltaEntry<'a> { pub key: Key, pub lsn: Lsn, /// Size of the stored value pub size: u64, /// Reference to the on-disk value - pub val: ValueRef, + pub val: ValueRef<'a>, } /// Reference to an on-disk value -pub struct ValueRef> { +pub struct ValueRef<'a> { blob_ref: BlobRef, - reader: BlockCursor>, + reader: BlockCursor<'a>, } -impl> ValueRef { +impl<'a> ValueRef<'a> { /// Loads the value from disk pub async fn load(&self) -> Result { // theoretically we *could* record an access time for each, but it does not really matter @@ -1039,10 +1028,10 @@ impl> ValueRef { } } -struct Adapter>(T); +pub(crate) struct Adapter(T); -impl> BlockReader for Adapter { - fn read_blk(&self, blknum: u32) -> Result { - self.0.as_ref().file.read_blk(blknum) +impl> Adapter { + pub(crate) async fn read_blk(&self, blknum: u32) -> Result { + self.0.as_ref().file.read_blk(blknum).await } } diff --git a/pageserver/src/tenant/storage_layer/filename.rs b/pageserver/src/tenant/storage_layer/filename.rs index 843bb1f631..b52c20a7c6 100644 --- a/pageserver/src/tenant/storage_layer/filename.rs +++ b/pageserver/src/tenant/storage_layer/filename.rs @@ -212,9 +212,20 @@ pub enum LayerFileName { } impl LayerFileName { - pub fn file_name(&self) -> String { + pub(crate) fn file_name(&self) -> String { self.to_string() } + + /// Determines if this layer file is considered to be in future meaning we will discard these + /// layers during timeline initialization from the given disk_consistent_lsn. + pub(crate) fn is_in_future(&self, disk_consistent_lsn: Lsn) -> bool { + use LayerFileName::*; + match self { + Image(file_name) if file_name.lsn > disk_consistent_lsn => true, + Delta(file_name) if file_name.lsn_range.end > disk_consistent_lsn + 1 => true, + _ => false, + } + } } impl fmt::Display for LayerFileName { @@ -263,8 +274,8 @@ impl serde::Serialize for LayerFileName { S: serde::Serializer, { match self { - Self::Image(fname) => serializer.serialize_str(&fname.to_string()), - Self::Delta(fname) => serializer.serialize_str(&fname.to_string()), + Self::Image(fname) => serializer.collect_str(fname), + Self::Delta(fname) => serializer.collect_str(fname), } } } diff --git a/pageserver/src/tenant/storage_layer/image_layer.rs b/pageserver/src/tenant/storage_layer/image_layer.rs index 3fe600e5ff..b1fc257092 100644 --- a/pageserver/src/tenant/storage_layer/image_layer.rs +++ b/pageserver/src/tenant/storage_layer/image_layer.rs @@ -169,8 +169,52 @@ impl std::fmt::Debug for ImageLayerInner { #[async_trait::async_trait] impl Layer for ImageLayer { - /// debugging function to print out the contents of the layer - async fn dump(&self, verbose: bool, ctx: &RequestContext) -> Result<()> { + /// Look up given page in the file + async fn get_value_reconstruct_data( + &self, + key: Key, + lsn_range: Range, + reconstruct_state: &mut ValueReconstructState, + ctx: &RequestContext, + ) -> anyhow::Result { + self.get_value_reconstruct_data(key, lsn_range, reconstruct_state, ctx) + .await + } +} + +/// Boilerplate to implement the Layer trait, always use layer_desc for persistent layers. +impl std::fmt::Display for ImageLayer { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", self.layer_desc().short_id()) + } +} + +impl AsLayerDesc for ImageLayer { + fn layer_desc(&self) -> &PersistentLayerDesc { + &self.desc + } +} + +impl PersistentLayer for ImageLayer { + fn local_path(&self) -> Option { + self.local_path() + } + + fn delete_resident_layer_file(&self) -> Result<()> { + self.delete_resident_layer_file() + } + + fn info(&self, reset: LayerAccessStatsReset) -> HistoricLayerInfo { + self.info(reset) + } + + fn access_stats(&self) -> &LayerAccessStats { + self.access_stats() + } +} + +impl ImageLayer { + pub(crate) async fn dump(&self, verbose: bool, ctx: &RequestContext) -> Result<()> { println!( "----- image layer for ten {} tli {} key {}-{} at {} is_incremental {} size {} ----", self.desc.tenant_id, @@ -178,7 +222,7 @@ impl Layer for ImageLayer { self.desc.key_range.start, self.desc.key_range.end, self.lsn, - self.desc.is_incremental, + self.desc.is_incremental(), self.desc.file_size ); @@ -203,8 +247,7 @@ impl Layer for ImageLayer { Ok(()) } - /// Look up given page in the file - async fn get_value_reconstruct_data( + pub(crate) async fn get_value_reconstruct_data( &self, key: Key, lsn_range: Range, @@ -225,65 +268,33 @@ impl Layer for ImageLayer { .with_context(|| format!("read {}", self.path().display())) } - /// Boilerplate to implement the Layer trait, always use layer_desc for persistent layers. - fn get_key_range(&self) -> Range { - self.layer_desc().key_range.clone() - } - - /// Boilerplate to implement the Layer trait, always use layer_desc for persistent layers. - fn get_lsn_range(&self) -> Range { - self.layer_desc().lsn_range.clone() - } - - /// Boilerplate to implement the Layer trait, always use layer_desc for persistent layers. - fn is_incremental(&self) -> bool { - self.layer_desc().is_incremental - } -} - -/// Boilerplate to implement the Layer trait, always use layer_desc for persistent layers. -impl std::fmt::Display for ImageLayer { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "{}", self.layer_desc().short_id()) - } -} - -impl AsLayerDesc for ImageLayer { - fn layer_desc(&self) -> &PersistentLayerDesc { - &self.desc - } -} - -impl PersistentLayer for ImageLayer { - fn local_path(&self) -> Option { + pub(crate) fn local_path(&self) -> Option { Some(self.path()) } - fn delete_resident_layer_file(&self) -> Result<()> { + pub(crate) fn delete_resident_layer_file(&self) -> Result<()> { // delete underlying file fs::remove_file(self.path())?; Ok(()) } - fn info(&self, reset: LayerAccessStatsReset) -> HistoricLayerInfo { - let layer_file_name = self.filename().file_name(); - let lsn_range = self.get_lsn_range(); + pub(crate) fn info(&self, reset: LayerAccessStatsReset) -> HistoricLayerInfo { + let layer_file_name = self.layer_desc().filename().file_name(); + let lsn_start = self.layer_desc().image_layer_lsn(); HistoricLayerInfo::Image { layer_file_name, layer_file_size: self.desc.file_size, - lsn_start: lsn_range.start, + lsn_start, remote: false, access_stats: self.access_stats.as_api_model(reset), } } - fn access_stats(&self) -> &LayerAccessStats { + pub(crate) fn access_stats(&self) -> &LayerAccessStats { &self.access_stats } -} -impl ImageLayer { fn path_for( path_or_conf: &PathOrConf, timeline_id: TimelineId, @@ -338,7 +349,8 @@ impl ImageLayer { PathOrConf::Path(_) => None, }; - let loaded = ImageLayerInner::load(&path, self.desc.image_layer_lsn(), expected_summary)?; + let loaded = + ImageLayerInner::load(&path, self.desc.image_layer_lsn(), expected_summary).await?; if let PathOrConf::Path(ref path) = self.path_or_conf { // not production code @@ -371,7 +383,6 @@ impl ImageLayer { timeline_id, filename.key_range.clone(), filename.lsn, - false, file_size, ), // Now we assume image layer ALWAYS covers the full range. This may change in the future. lsn: filename.lsn, @@ -398,7 +409,6 @@ impl ImageLayer { summary.timeline_id, summary.key_range, summary.lsn, - false, metadata.len(), ), // Now we assume image layer ALWAYS covers the full range. This may change in the future. lsn: summary.lsn, @@ -423,7 +433,7 @@ impl ImageLayer { } impl ImageLayerInner { - pub(super) fn load( + pub(super) async fn load( path: &std::path::Path, lsn: Lsn, summary: Option, @@ -431,7 +441,7 @@ impl ImageLayerInner { let file = VirtualFile::open(path) .with_context(|| format!("Failed to open file '{}'", path.display()))?; let file = FileBlockReader::new(file); - let summary_blk = file.read_blk(0)?; + let summary_blk = file.read_blk(0).await?; let actual_summary = Summary::des_prefix(summary_blk.as_ref())?; if let Some(mut expected_summary) = summary { @@ -500,7 +510,6 @@ struct ImageLayerWriterInner { tenant_id: TenantId, key_range: Range, lsn: Lsn, - is_incremental: bool, blob_writer: WriteBlobWriter, tree: DiskBtreeBuilder, @@ -516,7 +525,6 @@ impl ImageLayerWriterInner { tenant_id: TenantId, key_range: &Range, lsn: Lsn, - is_incremental: bool, ) -> anyhow::Result { // Create the file initially with a temporary filename. // We'll atomically rename it to the final name when we're done. @@ -551,7 +559,6 @@ impl ImageLayerWriterInner { lsn, tree: tree_builder, blob_writer, - is_incremental, }; Ok(writer) @@ -612,7 +619,6 @@ impl ImageLayerWriterInner { self.timeline_id, self.key_range.clone(), self.lsn, - self.is_incremental, // for now, image layer ALWAYS covers the full range metadata.len(), ); @@ -687,7 +693,6 @@ impl ImageLayerWriter { tenant_id: TenantId, key_range: &Range, lsn: Lsn, - is_incremental: bool, ) -> anyhow::Result { Ok(Self { inner: Some(ImageLayerWriterInner::new( @@ -696,7 +701,6 @@ impl ImageLayerWriter { tenant_id, key_range, lsn, - is_incremental, )?), }) } diff --git a/pageserver/src/tenant/storage_layer/inmemory_layer.rs b/pageserver/src/tenant/storage_layer/inmemory_layer.rs index d3ec78887d..35a77a7331 100644 --- a/pageserver/src/tenant/storage_layer/inmemory_layer.rs +++ b/pageserver/src/tenant/storage_layer/inmemory_layer.rs @@ -7,14 +7,12 @@ use crate::config::PageServerConf; use crate::context::RequestContext; use crate::repository::{Key, Value}; -use crate::tenant::blob_io::BlobWriter; use crate::tenant::block_io::BlockReader; use crate::tenant::ephemeral_file::EphemeralFile; use crate::tenant::storage_layer::{ValueReconstructResult, ValueReconstructState}; use crate::walrecord; use anyhow::{ensure, Result}; use pageserver_api::models::InMemoryLayerInfo; -use std::cell::RefCell; use std::collections::HashMap; use std::sync::OnceLock; use tracing::*; @@ -32,12 +30,6 @@ use tokio::sync::RwLock; use super::{DeltaLayer, DeltaLayerWriter, Layer}; -thread_local! { - /// A buffer for serializing object during [`InMemoryLayer::put_value`]. - /// This buffer is reused for each serialization to avoid additional malloc calls. - static SER_BUFFER: RefCell> = RefCell::new(Vec::new()); -} - pub struct InMemoryLayer { conf: &'static PageServerConf, tenant_id: TenantId, @@ -85,11 +77,11 @@ impl std::fmt::Debug for InMemoryLayerInner { } impl InMemoryLayer { - pub fn get_timeline_id(&self) -> TimelineId { + pub(crate) fn get_timeline_id(&self) -> TimelineId { self.timeline_id } - pub fn info(&self) -> InMemoryLayerInfo { + pub(crate) fn info(&self) -> InMemoryLayerInfo { let lsn_start = self.start_lsn; if let Some(&lsn_end) = self.end_lsn.get() { @@ -99,32 +91,22 @@ impl InMemoryLayer { } } - fn assert_writable(&self) { + pub(crate) fn assert_writable(&self) { assert!(self.end_lsn.get().is_none()); } - fn end_lsn_or_max(&self) -> Lsn { + pub(crate) fn end_lsn_or_max(&self) -> Lsn { self.end_lsn.get().copied().unwrap_or(Lsn::MAX) } -} -#[async_trait::async_trait] -impl Layer for InMemoryLayer { - fn get_key_range(&self) -> Range { - Key::MIN..Key::MAX - } - - fn get_lsn_range(&self) -> Range { + pub(crate) fn get_lsn_range(&self) -> Range { self.start_lsn..self.end_lsn_or_max() } - fn is_incremental(&self) -> bool { - // in-memory layer is always considered incremental. - true - } - /// debugging function to print out the contents of the layer - async fn dump(&self, verbose: bool, _ctx: &RequestContext) -> Result<()> { + /// + /// this is likely completly unused + pub async fn dump(&self, verbose: bool, _ctx: &RequestContext) -> Result<()> { let inner = self.inner.read().await; let end_str = self.end_lsn_or_max(); @@ -171,7 +153,7 @@ impl Layer for InMemoryLayer { } /// Look up given value in the layer. - async fn get_value_reconstruct_data( + pub(crate) async fn get_value_reconstruct_data( &self, key: Key, lsn_range: Range, @@ -221,6 +203,20 @@ impl Layer for InMemoryLayer { } } +#[async_trait::async_trait] +impl Layer for InMemoryLayer { + async fn get_value_reconstruct_data( + &self, + key: Key, + lsn_range: Range, + reconstruct_data: &mut ValueReconstructState, + ctx: &RequestContext, + ) -> Result { + self.get_value_reconstruct_data(key, lsn_range, reconstruct_data, ctx) + .await + } +} + impl std::fmt::Display for InMemoryLayer { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { let end_lsn = self.end_lsn_or_max(); @@ -234,7 +230,7 @@ impl InMemoryLayer { /// pub async fn size(&self) -> Result { let inner = self.inner.read().await; - Ok(inner.file.size()) + Ok(inner.file.len()) } /// @@ -269,17 +265,17 @@ impl InMemoryLayer { /// Adds the page version to the in-memory tree pub async fn put_value(&self, key: Key, lsn: Lsn, val: &Value) -> Result<()> { trace!("put_value key {} at {}/{}", key, self.timeline_id, lsn); - let mut inner = self.inner.write().await; + let inner: &mut _ = &mut *self.inner.write().await; self.assert_writable(); let off = { - SER_BUFFER.with(|x| -> Result<_> { - let mut buf = x.borrow_mut(); - buf.clear(); - val.ser_into(&mut (*buf))?; - let off = inner.file.write_blob(&buf)?; - Ok(off) - })? + // Avoid doing allocations for "small" values. + // In the regression test suite, the limit of 256 avoided allocations in 95% of cases: + // https://github.com/neondatabase/neon/pull/5056#discussion_r1301975061 + let mut buf = smallvec::SmallVec::<[u8; 256]>::new(); + buf.clear(); + val.ser_into(&mut buf)?; + inner.file.write_blob(&buf).await? }; let vec_map = inner.index.entry(key).or_default(); @@ -317,7 +313,7 @@ impl InMemoryLayer { /// Write this frozen in-memory layer to disk. /// /// Returns a new delta layer with all the same data as this in-memory layer - pub async fn write_to_disk(&self) -> Result { + pub(crate) async fn write_to_disk(&self) -> Result { // Grab the lock in read-mode. We hold it over the I/O, but because this // layer is not writeable anymore, no one should be trying to acquire the // write lock on it, so we shouldn't block anyone. There's one exception diff --git a/pageserver/src/tenant/storage_layer/layer_desc.rs b/pageserver/src/tenant/storage_layer/layer_desc.rs index 42c3925b73..6d6ab801d8 100644 --- a/pageserver/src/tenant/storage_layer/layer_desc.rs +++ b/pageserver/src/tenant/storage_layer/layer_desc.rs @@ -19,16 +19,17 @@ use serde::{Deserialize, Serialize}; pub struct PersistentLayerDesc { pub tenant_id: TenantId, pub timeline_id: TimelineId, + /// Range of keys that this layer covers pub key_range: Range, - /// For image layer, this is `[lsn, lsn+1)`. + /// Inclusive start, exclusive end of the LSN range that this layer holds. + /// + /// - For an open in-memory layer, the end bound is MAX_LSN + /// - For a frozen in-memory layer or a delta layer, the end bound is a valid lsn after the + /// range start + /// - An image layer represents snapshot at one LSN, so end_lsn is always the snapshot LSN + 1 pub lsn_range: Range, - /// Whether this is a delta layer. + /// Whether this is a delta layer, and also, is this incremental. pub is_delta: bool, - /// Whether this layer only contains page images for part of the keys in the range. In the current implementation, this should - /// always be equal to `is_delta`. If we land the partial image layer PR someday, image layer could also be - /// incremental. - pub is_incremental: bool, - /// File size pub file_size: u64, } @@ -61,7 +62,6 @@ impl PersistentLayerDesc { key_range, lsn_range: Lsn(0)..Lsn(1), is_delta: false, - is_incremental: false, file_size: 0, } } @@ -71,7 +71,6 @@ impl PersistentLayerDesc { timeline_id: TimelineId, key_range: Range, lsn: Lsn, - is_incremental: bool, file_size: u64, ) -> Self { Self { @@ -80,7 +79,6 @@ impl PersistentLayerDesc { key_range, lsn_range: Self::image_layer_lsn_range(lsn), is_delta: false, - is_incremental, file_size, } } @@ -98,7 +96,6 @@ impl PersistentLayerDesc { key_range, lsn_range, is_delta: true, - is_incremental: true, file_size, } } @@ -164,8 +161,12 @@ impl PersistentLayerDesc { self.tenant_id } + /// Does this layer only contain some data for the key-range (incremental), + /// or does it contain a version of every page? This is important to know + /// for garbage collecting old layers: an incremental layer depends on + /// the previous non-incremental layer. pub fn is_incremental(&self) -> bool { - self.is_incremental + self.is_delta } pub fn is_delta(&self) -> bool { @@ -182,7 +183,7 @@ impl PersistentLayerDesc { self.lsn_range.start, self.lsn_range.end, self.is_delta, - self.is_incremental, + self.is_incremental(), self.file_size, ); diff --git a/pageserver/src/tenant/storage_layer/remote_layer.rs b/pageserver/src/tenant/storage_layer/remote_layer.rs index 36a6593779..3f8d700863 100644 --- a/pageserver/src/tenant/storage_layer/remote_layer.rs +++ b/pageserver/src/tenant/storage_layer/remote_layer.rs @@ -60,7 +60,7 @@ impl std::fmt::Debug for RemoteLayer { f.debug_struct("RemoteLayer") .field("file_name", &self.desc.filename()) .field("layer_metadata", &self.layer_metadata) - .field("is_incremental", &self.desc.is_incremental) + .field("is_incremental", &self.desc.is_incremental()) .finish() } } @@ -76,39 +76,6 @@ impl Layer for RemoteLayer { ) -> Result { bail!("layer {self} needs to be downloaded"); } - - /// debugging function to print out the contents of the layer - async fn dump(&self, _verbose: bool, _ctx: &RequestContext) -> Result<()> { - println!( - "----- remote layer for ten {} tli {} keys {}-{} lsn {}-{} is_delta {} is_incremental {} size {} ----", - self.desc.tenant_id, - self.desc.timeline_id, - self.desc.key_range.start, - self.desc.key_range.end, - self.desc.lsn_range.start, - self.desc.lsn_range.end, - self.desc.is_delta, - self.desc.is_incremental, - self.desc.file_size, - ); - - Ok(()) - } - - /// Boilerplate to implement the Layer trait, always use layer_desc for persistent layers. - fn get_key_range(&self) -> Range { - self.layer_desc().key_range.clone() - } - - /// Boilerplate to implement the Layer trait, always use layer_desc for persistent layers. - fn get_lsn_range(&self) -> Range { - self.layer_desc().lsn_range.clone() - } - - /// Boilerplate to implement the Layer trait, always use layer_desc for persistent layers. - fn is_incremental(&self) -> bool { - self.layer_desc().is_incremental - } } /// Boilerplate to implement the Layer trait, always use layer_desc for persistent layers. @@ -142,8 +109,8 @@ impl PersistentLayer for RemoteLayer { } fn info(&self, reset: LayerAccessStatsReset) -> HistoricLayerInfo { - let layer_file_name = self.filename().file_name(); - let lsn_range = self.get_lsn_range(); + let layer_file_name = self.layer_desc().filename().file_name(); + let lsn_range = self.layer_desc().lsn_range.clone(); if self.desc.is_delta { HistoricLayerInfo::Delta { @@ -184,7 +151,6 @@ impl RemoteLayer { timelineid, fname.key_range.clone(), fname.lsn, - false, layer_metadata.file_size(), ), layer_metadata: layer_metadata.clone(), @@ -217,9 +183,9 @@ impl RemoteLayer { } /// Create a Layer struct representing this layer, after it has been downloaded. - pub fn create_downloaded_layer( + pub(crate) fn create_downloaded_layer( &self, - layer_map_lock_held_witness: &LayerManager, + _layer_map_lock_held_witness: &LayerManager, conf: &'static PageServerConf, file_size: u64, ) -> Arc { @@ -231,10 +197,8 @@ impl RemoteLayer { self.desc.tenant_id, &fname, file_size, - self.access_stats.clone_for_residence_change( - layer_map_lock_held_witness, - LayerResidenceStatus::Resident, - ), + self.access_stats + .clone_for_residence_change(LayerResidenceStatus::Resident), )) } else { let fname = self.desc.image_file_name(); @@ -244,10 +208,8 @@ impl RemoteLayer { self.desc.tenant_id, &fname, file_size, - self.access_stats.clone_for_residence_change( - layer_map_lock_held_witness, - LayerResidenceStatus::Resident, - ), + self.access_stats + .clone_for_residence_change(LayerResidenceStatus::Resident), )) } } diff --git a/pageserver/src/tenant/timeline.rs b/pageserver/src/tenant/timeline.rs index 7d5ec66053..ff4f11db14 100644 --- a/pageserver/src/tenant/timeline.rs +++ b/pageserver/src/tenant/timeline.rs @@ -1,5 +1,6 @@ pub mod delete; mod eviction_task; +mod init; pub mod layer_manager; mod logical_size; pub mod span; @@ -27,7 +28,6 @@ use utils::id::TenantTimelineId; use std::cmp::{max, min, Ordering}; use std::collections::{BinaryHeap, HashMap, HashSet}; -use std::fs; use std::ops::{Deref, Range}; use std::path::{Path, PathBuf}; use std::pin::pin; @@ -39,15 +39,13 @@ use crate::context::{ AccessStatsBehavior, DownloadBehavior, RequestContext, RequestContextBuilder, }; use crate::deletion_queue::DeletionQueueClient; -use crate::tenant::remote_timeline_client::{self, index::LayerFileMetadata}; +use crate::tenant::remote_timeline_client::index::LayerFileMetadata; use crate::tenant::storage_layer::delta_layer::DeltaEntry; use crate::tenant::storage_layer::{ - DeltaFileName, DeltaLayerWriter, ImageFileName, ImageLayerWriter, InMemoryLayer, - LayerAccessStats, LayerFileName, RemoteLayer, + DeltaLayerWriter, ImageLayerWriter, InMemoryLayer, LayerAccessStats, LayerFileName, RemoteLayer, }; use crate::tenant::timeline::logical_size::CurrentLogicalSize; use crate::tenant::{ - ephemeral_file::is_ephemeral_file, layer_map::{LayerMap, SearchResult}, metadata::{save_metadata, TimelineMetadata}, par_fsync, @@ -79,11 +77,10 @@ use utils::{ use crate::page_cache; use crate::repository::GcResult; use crate::repository::{Key, Value}; +use crate::task_mgr; use crate::task_mgr::TaskKind; use crate::walredo::WalRedoManager; -use crate::METADATA_FILE_NAME; use crate::ZERO_PAGE; -use crate::{is_temporary, task_mgr}; use self::delete::DeleteTimelineFlow; pub(super) use self::eviction_task::EvictionTaskTenantState; @@ -96,7 +93,7 @@ use super::config::TenantConf; use super::remote_timeline_client::index::IndexPart; use super::remote_timeline_client::RemoteTimelineClient; use super::storage_layer::{ - AsLayerDesc, DeltaLayer, ImageLayer, Layer, LayerAccessStatsReset, PersistentLayerDesc, + AsLayerDesc, DeltaLayer, ImageLayer, LayerAccessStatsReset, PersistentLayerDesc, }; #[derive(Debug, PartialEq, Eq, Clone, Copy)] @@ -473,7 +470,7 @@ impl Timeline { // The cached image can be returned directly if there is no WAL between the cached image // and requested LSN. The cached image can also be used to reduce the amount of WAL needed // for redo. - let cached_page_img = match self.lookup_cached_page(&key, lsn) { + let cached_page_img = match self.lookup_cached_page(&key, lsn).await { Some((cached_lsn, cached_img)) => { match cached_lsn.cmp(&lsn) { Ordering::Less => {} // there might be WAL between cached_lsn and lsn, we need to check @@ -502,6 +499,7 @@ impl Timeline { RECONSTRUCT_TIME .observe_closure_duration(|| self.reconstruct_value(key, lsn, reconstruct_state)) + .await } /// Get last or prev record separately. Same as get_last_record_rlsn().last/prev. @@ -1216,7 +1214,7 @@ impl Timeline { &layer_metadata, local_layer .access_stats() - .clone_for_residence_change(layer_mgr, LayerResidenceStatus::Evicted), + .clone_for_residence_change(LayerResidenceStatus::Evicted), ), LayerFileName::Delta(delta_name) => RemoteLayer::new_delta( self.tenant_id, @@ -1225,7 +1223,7 @@ impl Timeline { &layer_metadata, local_layer .access_stats() - .clone_for_residence_change(layer_mgr, LayerResidenceStatus::Evicted), + .clone_for_residence_change(LayerResidenceStatus::Evicted), ), }); @@ -1536,7 +1534,7 @@ impl Timeline { let layer_flush_start_rx = self.layer_flush_start_tx.subscribe(); let self_clone = Arc::clone(self); - info!("spawning flush loop"); + debug!("spawning flush loop"); *flush_loop_state = FlushLoopState::Running { #[cfg(test)] expect_initdb_optimization: false, @@ -1607,9 +1605,7 @@ impl Timeline { )); } - /// /// Initialize with an empty layer map. Used when creating a new timeline. - /// pub(super) fn init_empty_layer_map(&self, start_lsn: Lsn) { let mut layers = self.layers.try_write().expect( "in the context where we call this function, no other task has access to the object", @@ -1617,10 +1613,16 @@ impl Timeline { layers.initialize_empty(Lsn(start_lsn.0)); } - /// - /// Scan the timeline directory to populate the layer map. - /// - pub(super) async fn load_layer_map(&self, disk_consistent_lsn: Lsn) -> anyhow::Result<()> { + /// Scan the timeline directory, cleanup, populate the layer map, and schedule uploads for local-only + /// files. + pub(super) async fn load_layer_map( + &self, + disk_consistent_lsn: Lsn, + index_part: Option, + ) -> anyhow::Result<()> { + use init::{Decision::*, Discovered, FutureLayer}; + use LayerFileName::*; + let mut guard = self.layers.write().await; let timer = self.metrics.load_layer_map_histo.start_timer(); @@ -1628,102 +1630,157 @@ impl Timeline { // Scan timeline directory and create ImageFileName and DeltaFilename // structs representing all files on disk let timeline_path = self.conf.timeline_path(&self.tenant_id, &self.timeline_id); - // total size of layer files in the current timeline directory - let mut total_physical_size = 0; + let (conf, tenant_id, timeline_id) = (self.conf, self.tenant_id, self.timeline_id); + let span = tracing::Span::current(); - let mut loaded_layers = Vec::>::new(); + let (loaded_layers, to_sync, total_physical_size) = tokio::task::spawn_blocking({ + move || { + let _g = span.entered(); + let discovered = init::scan_timeline_dir(&timeline_path)?; + let mut discovered_layers = Vec::with_capacity(discovered.len()); + let mut unrecognized_files = Vec::new(); - for direntry in fs::read_dir(timeline_path)? { - let direntry = direntry?; - let direntry_path = direntry.path(); - let fname = direntry.file_name(); - let fname = fname.to_string_lossy(); + let mut path = timeline_path; - if let Some(filename) = ImageFileName::parse_str(&fname) { - // create an ImageLayer struct for each image file. - if filename.lsn > disk_consistent_lsn { - info!( - "found future image layer {} on timeline {} disk_consistent_lsn is {}", - filename, self.timeline_id, disk_consistent_lsn - ); - - rename_to_backup(&direntry_path)?; - continue; + for discovered in discovered { + let (name, kind) = match discovered { + Discovered::Layer(file_name, file_size) => { + discovered_layers.push((file_name, file_size)); + continue; + } + Discovered::Metadata | Discovered::IgnoredBackup => { + continue; + } + Discovered::Unknown(file_name) => { + // we will later error if there are any + unrecognized_files.push(file_name); + continue; + } + Discovered::Ephemeral(name) => (name, "old ephemeral file"), + Discovered::Temporary(name) => (name, "temporary timeline file"), + Discovered::TemporaryDownload(name) => (name, "temporary download"), + }; + path.push(name); + init::cleanup(&path, kind)?; + path.pop(); } - let file_size = direntry_path.metadata()?.len(); - let stats = - LayerAccessStats::for_loading_layer(&guard, LayerResidenceStatus::Resident); - - let layer = ImageLayer::new( - self.conf, - self.timeline_id, - self.tenant_id, - &filename, - file_size, - stats, - ); - - total_physical_size += file_size; - loaded_layers.push(Arc::new(layer)); - } else if let Some(filename) = DeltaFileName::parse_str(&fname) { - // Create a DeltaLayer struct for each delta file. - // The end-LSN is exclusive, while disk_consistent_lsn is - // inclusive. For example, if disk_consistent_lsn is 100, it is - // OK for a delta layer to have end LSN 101, but if the end LSN - // is 102, then it might not have been fully flushed to disk - // before crash. - if filename.lsn_range.end > disk_consistent_lsn + 1 { - info!( - "found future delta layer {} on timeline {} disk_consistent_lsn is {}", - filename, self.timeline_id, disk_consistent_lsn + if !unrecognized_files.is_empty() { + // assume that if there are any there are many many. + let n = unrecognized_files.len(); + let first = &unrecognized_files[..n.min(10)]; + anyhow::bail!( + "unrecognized files in timeline dir (total {n}), first 10: {first:?}" ); - - rename_to_backup(&direntry_path)?; - continue; } - let file_size = direntry_path.metadata()?.len(); - let stats = - LayerAccessStats::for_loading_layer(&guard, LayerResidenceStatus::Resident); + let decided = + init::reconcile(discovered_layers, index_part.as_ref(), disk_consistent_lsn); - let layer = DeltaLayer::new( - self.conf, - self.timeline_id, - self.tenant_id, - &filename, - file_size, - stats, - ); + let mut loaded_layers = Vec::new(); + let mut needs_upload = Vec::new(); + let mut needs_cleanup = Vec::new(); + let mut total_physical_size = 0; - total_physical_size += file_size; - loaded_layers.push(Arc::new(layer)); - } else if fname == METADATA_FILE_NAME || fname.ends_with(".old") { - // ignore these - } else if remote_timeline_client::is_temp_download_file(&direntry_path) { - info!( - "skipping temp download file, reconcile_with_remote will resume / clean up: {}", - fname - ); - } else if is_ephemeral_file(&fname) { - // Delete any old ephemeral files - trace!("deleting old ephemeral file in timeline dir: {}", fname); - fs::remove_file(&direntry_path)?; - } else if is_temporary(&direntry_path) { - info!("removing temp timeline file at {}", direntry_path.display()); - fs::remove_file(&direntry_path).with_context(|| { - format!( - "failed to remove temp download file at {}", - direntry_path.display() - ) - })?; - } else { - warn!("unrecognized filename in timeline dir: {}", fname); + for (name, decision) in decided { + let decision = match decision { + Ok(UseRemote { local, remote }) => { + path.push(name.file_name()); + init::cleanup_local_file_for_remote(&path, &local, &remote)?; + path.pop(); + + UseRemote { local, remote } + } + Ok(decision) => decision, + Err(FutureLayer { local }) => { + if local.is_some() { + path.push(name.file_name()); + init::cleanup_future_layer(&path, &name, disk_consistent_lsn)?; + path.pop(); + } + needs_cleanup.push(name); + continue; + } + }; + + match &name { + Delta(d) => assert!(d.lsn_range.end <= disk_consistent_lsn + 1), + Image(i) => assert!(i.lsn <= disk_consistent_lsn), + } + + let status = match &decision { + UseLocal(_) | NeedsUpload(_) => LayerResidenceStatus::Resident, + Evicted(_) | UseRemote { .. } => LayerResidenceStatus::Evicted, + }; + + let stats = LayerAccessStats::for_loading_layer(status); + + let layer: Arc = match (name, &decision) { + (Delta(d), UseLocal(m) | NeedsUpload(m)) => { + total_physical_size += m.file_size(); + Arc::new(DeltaLayer::new( + conf, + timeline_id, + tenant_id, + &d, + m.file_size(), + stats, + )) + } + (Image(i), UseLocal(m) | NeedsUpload(m)) => { + total_physical_size += m.file_size(); + Arc::new(ImageLayer::new( + conf, + timeline_id, + tenant_id, + &i, + m.file_size(), + stats, + )) + } + (Delta(d), Evicted(remote) | UseRemote { remote, .. }) => Arc::new( + RemoteLayer::new_delta(tenant_id, timeline_id, &d, remote, stats), + ), + (Image(i), Evicted(remote) | UseRemote { remote, .. }) => Arc::new( + RemoteLayer::new_img(tenant_id, timeline_id, &i, remote, stats), + ), + }; + + if let NeedsUpload(m) = decision { + needs_upload.push((layer.clone(), m)); + } + + loaded_layers.push(layer); + } + Ok(( + loaded_layers, + (needs_upload, needs_cleanup), + total_physical_size, + )) } - } + }) + .await + .map_err(anyhow::Error::new) + .and_then(|x| x)?; let num_layers = loaded_layers.len(); - guard.initialize_local_layers(loaded_layers, Lsn(disk_consistent_lsn.0) + 1); + + guard.initialize_local_layers(loaded_layers, disk_consistent_lsn + 1); + + if let Some(rtc) = self.remote_client.as_ref() { + // Deletion queue client is always Some if remote_client is Some + let deletion_queue_client = self.deletion_queue_client.as_ref().unwrap(); + + let (needs_upload, needs_cleanup) = to_sync; + for (layer, m) in needs_upload { + rtc.schedule_layer_file_upload(&layer.layer_desc().filename(), &m)?; + } + rtc.schedule_layer_file_deletion(&needs_cleanup, &deletion_queue_client) + .await?; + rtc.schedule_index_upload_for_file_changes()?; + // Tenant::create_timeline will wait for these uploads to happen before returning, or + // on retry. + } info!( "loaded layer map with {} layers at {}, total physical size: {}", @@ -1734,236 +1791,6 @@ impl Timeline { .set(total_physical_size); timer.stop_and_record(); - - Ok(()) - } - - async fn create_remote_layers( - &self, - index_part: &IndexPart, - local_layers: HashMap>, - up_to_date_disk_consistent_lsn: Lsn, - ) -> anyhow::Result>> { - // Are we missing some files that are present in remote storage? - // Create RemoteLayer instances for them. - let mut local_only_layers = local_layers; - - // We're holding a layer map lock for a while but this - // method is only called during init so it's fine. - let mut guard = self.layers.write().await; - - let mut corrupted_local_layers = Vec::new(); - let mut added_remote_layers = Vec::new(); - for remote_layer_name in index_part.layer_metadata.keys() { - let local_layer = local_only_layers.remove(remote_layer_name); - - let remote_layer_metadata = index_part - .layer_metadata - .get(remote_layer_name) - .map(LayerFileMetadata::from) - .with_context(|| { - format!( - "No remote layer metadata found for layer {}", - remote_layer_name.file_name() - ) - })?; - - // Is the local layer's size different from the size stored in the - // remote index file? - // If so, rename_to_backup those files & replace their local layer with - // a RemoteLayer in the layer map so that we re-download them on-demand. - if let Some(local_layer) = local_layer { - let local_layer_path = local_layer - .local_path() - .expect("caller must ensure that local_layers only contains local layers"); - ensure!( - local_layer_path.exists(), - "every layer from local_layers must exist on disk: {}", - local_layer_path.display() - ); - - let remote_size = remote_layer_metadata.file_size(); - let metadata = local_layer_path.metadata().with_context(|| { - format!( - "get file size of local layer {}", - local_layer_path.display() - ) - })?; - let local_size = metadata.len(); - if local_size != remote_size { - warn!("removing local file {local_layer_path:?} because it has unexpected length {local_size}; length in remote index is {remote_size}"); - if let Err(err) = rename_to_backup(&local_layer_path) { - assert!(local_layer_path.exists(), "we would leave the local_layer without a file if this does not hold: {}", local_layer_path.display()); - anyhow::bail!("could not rename file {local_layer_path:?}: {err:?}"); - } else { - self.metrics.resident_physical_size_gauge.sub(local_size); - corrupted_local_layers.push(local_layer); - // fall-through to adding the remote layer - } - } else { - debug!( - "layer is present locally and file size matches remote, using it: {}", - local_layer_path.display() - ); - continue; - } - } - - info!( - "remote layer does not exist locally, creating remote layer: {}", - remote_layer_name.file_name() - ); - - match remote_layer_name { - LayerFileName::Image(imgfilename) => { - if imgfilename.lsn > up_to_date_disk_consistent_lsn { - info!( - "found future image layer {} on timeline {} remote_consistent_lsn is {}", - imgfilename, self.timeline_id, up_to_date_disk_consistent_lsn - ); - continue; - } - let stats = - LayerAccessStats::for_loading_layer(&guard, LayerResidenceStatus::Evicted); - - let remote_layer = RemoteLayer::new_img( - self.tenant_id, - self.timeline_id, - imgfilename, - &remote_layer_metadata, - stats, - ); - let remote_layer = Arc::new(remote_layer); - added_remote_layers.push(remote_layer); - } - LayerFileName::Delta(deltafilename) => { - // Create a RemoteLayer for the delta file. - // The end-LSN is exclusive, while disk_consistent_lsn is - // inclusive. For example, if disk_consistent_lsn is 100, it is - // OK for a delta layer to have end LSN 101, but if the end LSN - // is 102, then it might not have been fully flushed to disk - // before crash. - if deltafilename.lsn_range.end > up_to_date_disk_consistent_lsn + 1 { - info!( - "found future delta layer {} on timeline {} remote_consistent_lsn is {}", - deltafilename, self.timeline_id, up_to_date_disk_consistent_lsn - ); - continue; - } - let stats = - LayerAccessStats::for_loading_layer(&guard, LayerResidenceStatus::Evicted); - - let remote_layer = RemoteLayer::new_delta( - self.tenant_id, - self.timeline_id, - deltafilename, - &remote_layer_metadata, - stats, - ); - let remote_layer = Arc::new(remote_layer); - added_remote_layers.push(remote_layer); - } - } - } - guard.initialize_remote_layers(corrupted_local_layers, added_remote_layers); - Ok(local_only_layers) - } - - /// This function will synchronize local state with what we have in remote storage. - /// - /// Steps taken: - /// 1. Initialize upload queue based on `index_part`. - /// 2. Create `RemoteLayer` instances for layers that exist only on the remote. - /// The list of layers on the remote comes from `index_part`. - /// The list of local layers is given by the layer map's `iter_historic_layers()`. - /// So, the layer map must have been loaded already. - /// 3. Schedule upload of local-only layer files (which will then also update the remote - /// IndexPart to include the new layer files). - /// - /// Refer to the [`remote_timeline_client`] module comment for more context. - /// - /// # TODO - /// May be a bit cleaner to do things based on populated remote client, - /// and then do things based on its upload_queue.latest_files. - #[instrument(skip(self, index_part, up_to_date_metadata))] - pub async fn reconcile_with_remote( - &self, - up_to_date_metadata: &TimelineMetadata, - index_part: Option<&IndexPart>, - ) -> anyhow::Result<()> { - info!("starting"); - let remote_client = self - .remote_client - .as_ref() - .ok_or_else(|| anyhow!("cannot download without remote storage"))?; - - let disk_consistent_lsn = up_to_date_metadata.disk_consistent_lsn(); - - let local_layers = { - let guard = self.layers.read().await; - let layers = guard.layer_map(); - layers - .iter_historic_layers() - .map(|l| (l.filename(), guard.get_from_desc(&l))) - .collect::>() - }; - - // If no writes happen, new branches do not have any layers, only the metadata file. - let has_local_layers = !local_layers.is_empty(); - let local_only_layers = match index_part { - Some(index_part) => { - info!( - "initializing upload queue from remote index with {} layer files", - index_part.layer_metadata.len() - ); - remote_client.init_upload_queue(index_part)?; - self.create_remote_layers(index_part, local_layers, disk_consistent_lsn) - .await? - } - None => { - info!("initializing upload queue as empty"); - remote_client.init_upload_queue_for_empty_remote(up_to_date_metadata)?; - local_layers - } - }; - - if has_local_layers { - // Are there local files that don't exist remotely? Schedule uploads for them. - // Local timeline metadata will get uploaded to remove along witht he layers. - for (layer_name, layer) in &local_only_layers { - // XXX solve this in the type system - let layer_path = layer - .local_path() - .expect("local_only_layers only contains local layers"); - let layer_size = layer_path - .metadata() - .with_context(|| format!("failed to get file {layer_path:?} metadata"))? - .len(); - info!("scheduling {layer_path:?} for upload"); - remote_client - .schedule_layer_file_upload(layer_name, &LayerFileMetadata::new(layer_size))?; - } - remote_client.schedule_index_upload_for_file_changes()?; - } else if index_part.is_none() { - // No data on the remote storage, no local layers, local metadata file. - // - // TODO https://github.com/neondatabase/neon/issues/3865 - // Currently, console does not wait for the timeline data upload to the remote storage - // and considers the timeline created, expecting other pageserver nodes to work with it. - // Branch metadata upload could get interrupted (e.g pageserver got killed), - // hence any locally existing branch metadata with no remote counterpart should be uploaded, - // otherwise any other pageserver won't see the branch on `attach`. - // - // After the issue gets implemented, pageserver should rather remove the branch, - // since absence on S3 means we did not acknowledge the branch creation and console will have to retry, - // no need to keep the old files. - remote_client.schedule_index_upload_for_metadata_update(up_to_date_metadata)?; - } else { - // Local timeline has a metadata file, remote one too, both have no layers to sync. - } - - info!("Done"); - Ok(()) } @@ -2460,7 +2287,15 @@ impl Timeline { ))); } } - ancestor.wait_lsn(timeline.ancestor_lsn, ctx).await?; + ancestor + .wait_lsn(timeline.ancestor_lsn, ctx) + .await + .with_context(|| { + format!( + "wait for lsn {} on ancestor timeline_id={}", + timeline.ancestor_lsn, ancestor.timeline_id + ) + })?; timeline_owned = ancestor; timeline = &*timeline_owned; @@ -2639,13 +2474,14 @@ impl Timeline { } } - fn lookup_cached_page(&self, key: &Key, lsn: Lsn) -> Option<(Lsn, Bytes)> { + async fn lookup_cached_page(&self, key: &Key, lsn: Lsn) -> Option<(Lsn, Bytes)> { let cache = page_cache::get(); // FIXME: It's pointless to check the cache for things that are not 8kB pages. // We should look at the key to determine if it's a cacheable object - let (lsn, read_guard) = - cache.lookup_materialized_page(self.tenant_id, self.timeline_id, key, lsn)?; + let (lsn, read_guard) = cache + .lookup_materialized_page(self.tenant_id, self.timeline_id, key, lsn) + .await?; let img = Bytes::from(read_guard.to_vec()); Some((lsn, img)) } @@ -2870,7 +2706,6 @@ impl Timeline { if let Some(ref l) = delta_layer_to_add { // TODO: move access stats, metrics update, etc. into layer manager. l.access_stats().record_residence_event( - &guard, LayerResidenceStatus::Resident, LayerResidenceEventReason::LayerCreate, ); @@ -3167,7 +3002,6 @@ impl Timeline { self.tenant_id, &img_range, lsn, - false, // image layer always covers the full range )?; fail_point!("image-layer-writer-fail-before-finish", |_| { @@ -3260,7 +3094,6 @@ impl Timeline { .add(metadata.len()); let l = Arc::new(l); l.access_stats().record_residence_event( - &guard, LayerResidenceStatus::Resident, LayerResidenceEventReason::LayerCreate, ); @@ -3428,8 +3261,8 @@ impl Timeline { /// start of level0 files compaction, the on-demand download should be revisited as well. /// /// [`compact_inner`]: Self::compact_inner - fn compact_level0_phase1( - self: Arc, + async fn compact_level0_phase1( + self: &Arc, _layer_removal_cs: Arc>, guard: tokio::sync::OwnedRwLockReadGuard, mut stats: CompactLevel0Phase1StatsBuilder, @@ -3580,7 +3413,7 @@ impl Timeline { .collect(); for dl in downcast_deltas.iter() { // TODO: replace this with an await once we fully go async - all_keys.extend(Handle::current().block_on(DeltaLayer::load_keys(dl, ctx))?); + all_keys.extend(DeltaLayer::load_keys(dl, ctx).await?); } // The current stdlib sorting implementation is designed in a way where it is @@ -3694,107 +3527,103 @@ impl Timeline { let mut dup_start_lsn: Lsn = Lsn::INVALID; // start LSN of layer containing values of the single key let mut dup_end_lsn: Lsn = Lsn::INVALID; // end LSN of layer containing values of the single key - // TODO remove this block_on wrapper once we fully go async - Handle::current().block_on(async { - for &DeltaEntry { - key, lsn, ref val, .. - } in all_values_iter - { - let value = val.load().await?; - let same_key = prev_key.map_or(false, |prev_key| prev_key == key); - // We need to check key boundaries once we reach next key or end of layer with the same key - if !same_key || lsn == dup_end_lsn { - let mut next_key_size = 0u64; - let is_dup_layer = dup_end_lsn.is_valid(); - dup_start_lsn = Lsn::INVALID; - if !same_key { - dup_end_lsn = Lsn::INVALID; - } - // Determine size occupied by this key. We stop at next key or when size becomes larger than target_file_size - for (next_key, next_lsn, next_size) in all_keys_iter.by_ref() { - next_key_size = next_size; - if key != next_key { - if dup_end_lsn.is_valid() { - // We are writting segment with duplicates: - // place all remaining values of this key in separate segment - dup_start_lsn = dup_end_lsn; // new segments starts where old stops - dup_end_lsn = lsn_range.end; // there are no more values of this key till end of LSN range - } - break; - } - key_values_total_size += next_size; - // Check if it is time to split segment: if total keys size is larger than target file size. - // We need to avoid generation of empty segments if next_size > target_file_size. - if key_values_total_size > target_file_size && lsn != next_lsn { - // Split key between multiple layers: such layer can contain only single key - dup_start_lsn = if dup_end_lsn.is_valid() { - dup_end_lsn // new segment with duplicates starts where old one stops - } else { - lsn // start with the first LSN for this key - }; - dup_end_lsn = next_lsn; // upper LSN boundary is exclusive - break; - } - } - // handle case when loop reaches last key: in this case dup_end is non-zero but dup_start is not set. - if dup_end_lsn.is_valid() && !dup_start_lsn.is_valid() { - dup_start_lsn = dup_end_lsn; - dup_end_lsn = lsn_range.end; - } - if writer.is_some() { - let written_size = writer.as_mut().unwrap().size(); - let contains_hole = - next_hole < holes.len() && key >= holes[next_hole].key_range.end; - // check if key cause layer overflow or contains hole... - if is_dup_layer - || dup_end_lsn.is_valid() - || written_size + key_values_total_size > target_file_size - || contains_hole - { - // ... if so, flush previous layer and prepare to write new one - new_layers.push(Arc::new( - writer.take().unwrap().finish(prev_key.unwrap().next())?, - )); - writer = None; - - if contains_hole { - // skip hole - next_hole += 1; - } - } - } - // Remember size of key value because at next iteration we will access next item - key_values_total_size = next_key_size; + for &DeltaEntry { + key, lsn, ref val, .. + } in all_values_iter + { + let value = val.load().await?; + let same_key = prev_key.map_or(false, |prev_key| prev_key == key); + // We need to check key boundaries once we reach next key or end of layer with the same key + if !same_key || lsn == dup_end_lsn { + let mut next_key_size = 0u64; + let is_dup_layer = dup_end_lsn.is_valid(); + dup_start_lsn = Lsn::INVALID; + if !same_key { + dup_end_lsn = Lsn::INVALID; } - if writer.is_none() { - // Create writer if not initiaized yet - writer = Some(DeltaLayerWriter::new( - self.conf, - self.timeline_id, - self.tenant_id, - key, + // Determine size occupied by this key. We stop at next key or when size becomes larger than target_file_size + for (next_key, next_lsn, next_size) in all_keys_iter.by_ref() { + next_key_size = next_size; + if key != next_key { if dup_end_lsn.is_valid() { - // this is a layer containing slice of values of the same key - debug!("Create new dup layer {}..{}", dup_start_lsn, dup_end_lsn); - dup_start_lsn..dup_end_lsn + // We are writting segment with duplicates: + // place all remaining values of this key in separate segment + dup_start_lsn = dup_end_lsn; // new segments starts where old stops + dup_end_lsn = lsn_range.end; // there are no more values of this key till end of LSN range + } + break; + } + key_values_total_size += next_size; + // Check if it is time to split segment: if total keys size is larger than target file size. + // We need to avoid generation of empty segments if next_size > target_file_size. + if key_values_total_size > target_file_size && lsn != next_lsn { + // Split key between multiple layers: such layer can contain only single key + dup_start_lsn = if dup_end_lsn.is_valid() { + dup_end_lsn // new segment with duplicates starts where old one stops } else { - debug!("Create new layer {}..{}", lsn_range.start, lsn_range.end); - lsn_range.clone() - }, - )?); + lsn // start with the first LSN for this key + }; + dup_end_lsn = next_lsn; // upper LSN boundary is exclusive + break; + } } + // handle case when loop reaches last key: in this case dup_end is non-zero but dup_start is not set. + if dup_end_lsn.is_valid() && !dup_start_lsn.is_valid() { + dup_start_lsn = dup_end_lsn; + dup_end_lsn = lsn_range.end; + } + if writer.is_some() { + let written_size = writer.as_mut().unwrap().size(); + let contains_hole = + next_hole < holes.len() && key >= holes[next_hole].key_range.end; + // check if key cause layer overflow or contains hole... + if is_dup_layer + || dup_end_lsn.is_valid() + || written_size + key_values_total_size > target_file_size + || contains_hole + { + // ... if so, flush previous layer and prepare to write new one + new_layers.push(Arc::new( + writer.take().unwrap().finish(prev_key.unwrap().next())?, + )); + writer = None; - fail_point!("delta-layer-writer-fail-before-finish", |_| { - Result::<_>::Err(anyhow::anyhow!( - "failpoint delta-layer-writer-fail-before-finish" - )) - }); - - writer.as_mut().unwrap().put_value(key, lsn, value)?; - prev_key = Some(key); + if contains_hole { + // skip hole + next_hole += 1; + } + } + } + // Remember size of key value because at next iteration we will access next item + key_values_total_size = next_key_size; } - Ok(()) - })?; + if writer.is_none() { + // Create writer if not initiaized yet + writer = Some(DeltaLayerWriter::new( + self.conf, + self.timeline_id, + self.tenant_id, + key, + if dup_end_lsn.is_valid() { + // this is a layer containing slice of values of the same key + debug!("Create new dup layer {}..{}", dup_start_lsn, dup_end_lsn); + dup_start_lsn..dup_end_lsn + } else { + debug!("Create new layer {}..{}", lsn_range.start, lsn_range.end); + lsn_range.clone() + }, + )?); + } + + fail_point!("delta-layer-writer-fail-before-finish", |_| { + Err(CompactionError::Other(anyhow::anyhow!( + "failpoint delta-layer-writer-fail-before-finish" + ))) + }); + + writer.as_mut().unwrap().put_value(key, lsn, value)?; + prev_key = Some(key); + } if let Some(writer) = writer { new_layers.push(Arc::new(writer.finish(prev_key.unwrap().next())?)); } @@ -3807,10 +3636,10 @@ impl Timeline { // we still might easily hit the limit otherwise. let warn_limit = target_file_size * 2 + page_cache::PAGE_SZ as u64 * 2; for layer in new_layers.iter() { - if layer.desc.file_size > warn_limit { + if layer.layer_desc().file_size > warn_limit { warn!( %layer, - "created delta file of size {} larger than double of target of {target_file_size}", layer.desc.file_size + "created delta file of size {} larger than double of target of {target_file_size}", layer.layer_desc().file_size ); } } @@ -3828,7 +3657,7 @@ impl Timeline { stats.write_layer_files_micros = stats.read_lock_drop_micros.till_now(); stats.new_deltas_count = Some(new_layers.len()); - stats.new_deltas_size = Some(new_layers.iter().map(|l| l.desc.file_size).sum()); + stats.new_deltas_size = Some(new_layers.iter().map(|l| l.layer_desc().file_size).sum()); match TryInto::::try_into(stats) .and_then(|stats| serde_json::to_string(&stats).context("serde_json::to_string")) @@ -3868,8 +3697,7 @@ impl Timeline { deltas_to_compact, } = { let phase1_span = info_span!("compact_level0_phase1"); - let myself = Arc::clone(self); - let ctx = ctx.attached_child(); // technically, the spawn_blocking can outlive this future + let ctx = ctx.attached_child(); let mut stats = CompactLevel0Phase1StatsBuilder { version: Some(2), tenant_id: Some(self.tenant_id), @@ -3883,18 +3711,15 @@ impl Timeline { stats.read_lock_acquisition_micros = DurationRecorder::Recorded(RecordedDuration(now - begin), now); let layer_removal_cs = layer_removal_cs.clone(); - tokio::task::spawn_blocking(move || { - let _entered = phase1_span.enter(); - myself.compact_level0_phase1( - layer_removal_cs, - phase1_layers_locked, - stats, - target_file_size, - &ctx, - ) - }) - .await - .context("spawn_blocking")?? + self.compact_level0_phase1( + layer_removal_cs, + phase1_layers_locked, + stats, + target_file_size, + &ctx, + ) + .instrument(phase1_span) + .await? }; if new_layers.is_empty() && deltas_to_compact.is_empty() { @@ -3948,7 +3773,6 @@ impl Timeline { new_layer_paths.insert(new_delta_path, LayerFileMetadata::new(metadata.len())); l.access_stats().record_residence_event( - &guard, LayerResidenceStatus::Resident, LayerResidenceEventReason::LayerCreate, ); @@ -4353,7 +4177,7 @@ impl Timeline { /// /// Reconstruct a value, using the given base image and WAL records in 'data'. /// - fn reconstruct_value( + async fn reconstruct_value( &self, key: Key, request_lsn: Lsn, @@ -4422,6 +4246,7 @@ impl Timeline { last_rec_lsn, &img, ) + .await .context("Materialized page memoization failed") { return Err(PageReconstructError::from(e)); @@ -4881,7 +4706,8 @@ fn rename_to_backup(path: &Path) -> anyhow::Result<()> { for i in 0u32.. { new_path.set_file_name(format!("{filename}.{i}.old")); if !new_path.exists() { - std::fs::rename(path, &new_path)?; + std::fs::rename(path, &new_path) + .with_context(|| format!("rename {path:?} to {new_path:?}"))?; return Ok(()); } } diff --git a/pageserver/src/tenant/timeline/init.rs b/pageserver/src/tenant/timeline/init.rs new file mode 100644 index 0000000000..a270d96677 --- /dev/null +++ b/pageserver/src/tenant/timeline/init.rs @@ -0,0 +1,199 @@ +use crate::{ + is_temporary, + tenant::{ + ephemeral_file::is_ephemeral_file, + remote_timeline_client::{ + self, + index::{IndexPart, LayerFileMetadata}, + }, + storage_layer::LayerFileName, + }, + METADATA_FILE_NAME, +}; +use anyhow::Context; +use std::{collections::HashMap, ffi::OsString, path::Path, str::FromStr}; +use utils::lsn::Lsn; + +/// Identified files in the timeline directory. +pub(super) enum Discovered { + /// The only one we care about + Layer(LayerFileName, u64), + /// Old ephmeral files from previous launches, should be removed + Ephemeral(OsString), + /// Old temporary timeline files, unsure what these really are, should be removed + Temporary(OsString), + /// Temporary on-demand download files, should be removed + TemporaryDownload(OsString), + /// "metadata" file we persist locally and include in `index_part.json` + Metadata, + /// Backup file from previously future layers + IgnoredBackup, + /// Unrecognized, warn about these + Unknown(OsString), +} + +/// Scans the timeline directory for interesting files. +pub(super) fn scan_timeline_dir(path: &Path) -> anyhow::Result> { + let mut ret = Vec::new(); + + for direntry in std::fs::read_dir(path)? { + let direntry = direntry?; + let direntry_path = direntry.path(); + let file_name = direntry.file_name(); + + let fname = file_name.to_string_lossy(); + + let discovered = match LayerFileName::from_str(&fname) { + Ok(file_name) => { + let file_size = direntry.metadata()?.len(); + Discovered::Layer(file_name, file_size) + } + Err(_) => { + if fname == METADATA_FILE_NAME { + Discovered::Metadata + } else if fname.ends_with(".old") { + // ignore these + Discovered::IgnoredBackup + } else if remote_timeline_client::is_temp_download_file(&direntry_path) { + Discovered::TemporaryDownload(file_name) + } else if is_ephemeral_file(&fname) { + Discovered::Ephemeral(file_name) + } else if is_temporary(&direntry_path) { + Discovered::Temporary(file_name) + } else { + Discovered::Unknown(file_name) + } + } + }; + + ret.push(discovered); + } + + Ok(ret) +} + +/// Decision on what to do with a layer file after considering its local and remote metadata. +#[derive(Clone)] +pub(super) enum Decision { + /// The layer is not present locally. + Evicted(LayerFileMetadata), + /// The layer is present locally, but local metadata does not match remote; we must + /// delete it and treat it as evicted. + UseRemote { + local: LayerFileMetadata, + remote: LayerFileMetadata, + }, + /// The layer is present locally, and metadata matches. + UseLocal(LayerFileMetadata), + /// The layer is only known locally, it needs to be uploaded. + NeedsUpload(LayerFileMetadata), +} + +/// The related layer is is in future compared to disk_consistent_lsn, it must not be loaded. +#[derive(Debug)] +pub(super) struct FutureLayer { + /// The local metadata. `None` if the layer is only known through [`IndexPart`]. + pub(super) local: Option, +} + +/// Merges local discoveries and remote [`IndexPart`] to a collection of decisions. +/// +/// This function should not gain additional reasons to fail than [`FutureLayer`], consider adding +/// the checks earlier to [`scan_timeline_dir`]. +pub(super) fn reconcile( + discovered: Vec<(LayerFileName, u64)>, + index_part: Option<&IndexPart>, + disk_consistent_lsn: Lsn, +) -> Vec<(LayerFileName, Result)> { + use Decision::*; + + // name => (local, remote) + type Collected = HashMap, Option)>; + + let mut discovered = discovered + .into_iter() + .map(|(name, file_size)| (name, (Some(LayerFileMetadata::new(file_size)), None))) + .collect::(); + + // merge any index_part information, when available + index_part + .as_ref() + .map(|ip| ip.layer_metadata.iter()) + .into_iter() + .flatten() + .map(|(name, metadata)| (name, LayerFileMetadata::from(metadata))) + .for_each(|(name, metadata)| { + if let Some(existing) = discovered.get_mut(name) { + existing.1 = Some(metadata); + } else { + discovered.insert(name.to_owned(), (None, Some(metadata))); + } + }); + + discovered + .into_iter() + .map(|(name, (local, remote))| { + let decision = if name.is_in_future(disk_consistent_lsn) { + Err(FutureLayer { local }) + } else { + Ok(match (local, remote) { + (Some(local), Some(remote)) if local != remote => UseRemote { local, remote }, + (Some(x), Some(_)) => UseLocal(x), + (None, Some(x)) => Evicted(x), + (Some(x), None) => NeedsUpload(x), + (None, None) => { + unreachable!("there must not be any non-local non-remote files") + } + }) + }; + + (name, decision) + }) + .collect::>() +} + +pub(super) fn cleanup(path: &Path, kind: &str) -> anyhow::Result<()> { + let file_name = path.file_name().expect("must be file path"); + tracing::debug!(kind, ?file_name, "cleaning up"); + std::fs::remove_file(path) + .with_context(|| format!("failed to remove {kind} at {}", path.display())) +} + +pub(super) fn cleanup_local_file_for_remote( + path: &Path, + local: &LayerFileMetadata, + remote: &LayerFileMetadata, +) -> anyhow::Result<()> { + let local_size = local.file_size(); + let remote_size = remote.file_size(); + + let file_name = path.file_name().expect("must be file path"); + tracing::warn!("removing local file {file_name:?} because it has unexpected length {local_size}; length in remote index is {remote_size}"); + if let Err(err) = crate::tenant::timeline::rename_to_backup(path) { + assert!( + path.exists(), + "we would leave the local_layer without a file if this does not hold: {}", + path.display() + ); + Err(err) + } else { + Ok(()) + } +} + +pub(super) fn cleanup_future_layer( + path: &Path, + name: &LayerFileName, + disk_consistent_lsn: Lsn, +) -> anyhow::Result<()> { + use LayerFileName::*; + let kind = match name { + Delta(_) => "delta", + Image(_) => "image", + }; + // future image layers are allowed to be produced always for not yet flushed to disk + // lsns stored in InMemoryLayer. + tracing::info!("found future {kind} layer {name} disk_consistent_lsn is {disk_consistent_lsn}"); + crate::tenant::timeline::rename_to_backup(path)?; + Ok(()) +} diff --git a/pageserver/src/tenant/timeline/layer_manager.rs b/pageserver/src/tenant/timeline/layer_manager.rs index 40212265cf..5522ea1788 100644 --- a/pageserver/src/tenant/timeline/layer_manager.rs +++ b/pageserver/src/tenant/timeline/layer_manager.rs @@ -12,38 +12,38 @@ use crate::{ tenant::{ layer_map::{BatchedUpdates, LayerMap}, storage_layer::{ - AsLayerDesc, DeltaLayer, ImageLayer, InMemoryLayer, Layer, PersistentLayer, - PersistentLayerDesc, PersistentLayerKey, RemoteLayer, + AsLayerDesc, DeltaLayer, ImageLayer, InMemoryLayer, PersistentLayer, + PersistentLayerDesc, PersistentLayerKey, }, timeline::compare_arced_layers, }, }; /// Provides semantic APIs to manipulate the layer map. -pub struct LayerManager { +pub(crate) struct LayerManager { layer_map: LayerMap, layer_fmgr: LayerFileManager, } /// After GC, the layer map changes will not be applied immediately. Users should manually apply the changes after /// scheduling deletes in remote client. -pub struct ApplyGcResultGuard<'a>(BatchedUpdates<'a>); +pub(crate) struct ApplyGcResultGuard<'a>(BatchedUpdates<'a>); impl ApplyGcResultGuard<'_> { - pub fn flush(self) { + pub(crate) fn flush(self) { self.0.flush(); } } impl LayerManager { - pub fn create() -> Self { + pub(crate) fn create() -> Self { Self { layer_map: LayerMap::default(), layer_fmgr: LayerFileManager::new(), } } - pub fn get_from_desc(&self, desc: &PersistentLayerDesc) -> Arc { + pub(crate) fn get_from_desc(&self, desc: &PersistentLayerDesc) -> Arc { self.layer_fmgr.get_from_desc(desc) } @@ -51,18 +51,12 @@ impl LayerManager { /// /// We expect users only to be able to get an immutable layer map. If users want to make modifications, /// they should use the below semantic APIs. This design makes us step closer to immutable storage state. - pub fn layer_map(&self) -> &LayerMap { + pub(crate) fn layer_map(&self) -> &LayerMap { &self.layer_map } - /// Get a mutable reference to the layer map. This function will be removed once `flush_frozen_layer` - /// gets a refactor. - pub fn layer_map_mut(&mut self) -> &mut LayerMap { - &mut self.layer_map - } - /// Replace layers in the layer file manager, used in evictions and layer downloads. - pub fn replace_and_verify( + pub(crate) fn replace_and_verify( &mut self, expected: Arc, new: Arc, @@ -73,7 +67,7 @@ impl LayerManager { /// Called from `load_layer_map`. Initialize the layer manager with: /// 1. all on-disk layers /// 2. next open layer (with disk disk_consistent_lsn LSN) - pub fn initialize_local_layers( + pub(crate) fn initialize_local_layers( &mut self, on_disk_layers: Vec>, next_open_layer_at: Lsn, @@ -87,28 +81,13 @@ impl LayerManager { } /// Initialize when creating a new timeline, called in `init_empty_layer_map`. - pub fn initialize_empty(&mut self, next_open_layer_at: Lsn) { + pub(crate) fn initialize_empty(&mut self, next_open_layer_at: Lsn) { self.layer_map.next_open_layer_at = Some(next_open_layer_at); } - pub fn initialize_remote_layers( - &mut self, - corrupted_local_layers: Vec>, - remote_layers: Vec>, - ) { - let mut updates = self.layer_map.batch_update(); - for layer in corrupted_local_layers { - Self::remove_historic_layer(layer, &mut updates, &mut self.layer_fmgr); - } - for layer in remote_layers { - Self::insert_historic_layer(layer, &mut updates, &mut self.layer_fmgr); - } - updates.flush(); - } - /// Open a new writable layer to append data if there is no open layer, otherwise return the current open layer, /// called within `get_layer_for_write`. - pub fn get_layer_for_write( + pub(crate) fn get_layer_for_write( &mut self, lsn: Lsn, last_record_lsn: Lsn, @@ -163,7 +142,7 @@ impl LayerManager { } /// Called from `freeze_inmem_layer`, returns true if successfully frozen. - pub async fn try_freeze_in_memory_layer( + pub(crate) async fn try_freeze_in_memory_layer( &mut self, Lsn(last_record_lsn): Lsn, last_freeze_at: &AtomicLsn, @@ -185,7 +164,7 @@ impl LayerManager { } /// Add image layers to the layer map, called from `create_image_layers`. - pub fn track_new_image_layers(&mut self, image_layers: Vec) { + pub(crate) fn track_new_image_layers(&mut self, image_layers: Vec) { let mut updates = self.layer_map.batch_update(); for layer in image_layers { Self::insert_historic_layer(Arc::new(layer), &mut updates, &mut self.layer_fmgr); @@ -194,7 +173,7 @@ impl LayerManager { } /// Flush a frozen layer and add the written delta layer to the layer map. - pub fn finish_flush_l0_layer( + pub(crate) fn finish_flush_l0_layer( &mut self, delta_layer: Option, frozen_layer_for_check: &Arc, @@ -214,7 +193,7 @@ impl LayerManager { } /// Called when compaction is completed. - pub fn finish_compact_l0( + pub(crate) fn finish_compact_l0( &mut self, layer_removal_cs: Arc>, compact_from: Vec>, @@ -242,7 +221,7 @@ impl LayerManager { } /// Called when garbage collect the timeline. Returns a guard that will apply the updates to the layer map. - pub fn finish_gc_timeline( + pub(crate) fn finish_gc_timeline( &mut self, layer_removal_cs: Arc>, gc_layers: Vec>, @@ -271,16 +250,6 @@ impl LayerManager { mapping.insert(layer); } - /// Helper function to remove a layer into the layer map and file manager - fn remove_historic_layer( - layer: Arc, - updates: &mut BatchedUpdates<'_>, - mapping: &mut LayerFileManager, - ) { - updates.remove_historic(layer.layer_desc()); - mapping.remove(layer); - } - /// Removes the layer from local FS (if present) and from memory. /// Remote storage is not affected by this operation. fn delete_historic_layer( @@ -313,7 +282,7 @@ impl LayerManager { } } -pub struct LayerFileManager( +pub(crate) struct LayerFileManager( HashMap>, ); diff --git a/pageserver/src/tenant/timeline/walreceiver/connection_manager.rs b/pageserver/src/tenant/timeline/walreceiver/connection_manager.rs index d52b29f175..36960c631c 100644 --- a/pageserver/src/tenant/timeline/walreceiver/connection_manager.rs +++ b/pageserver/src/tenant/timeline/walreceiver/connection_manager.rs @@ -17,7 +17,7 @@ use crate::metrics::{ WALRECEIVER_ACTIVE_MANAGERS, WALRECEIVER_BROKER_UPDATES, WALRECEIVER_CANDIDATES_ADDED, WALRECEIVER_CANDIDATES_REMOVED, WALRECEIVER_SWITCHES, }; -use crate::task_mgr::TaskKind; +use crate::task_mgr::{shutdown_token, TaskKind}; use crate::tenant::{debug_assert_current_span_has_tenant_and_timeline_id, Timeline}; use anyhow::Context; use chrono::{NaiveDateTime, Utc}; @@ -31,10 +31,11 @@ use storage_broker::Streaming; use tokio::select; use tracing::*; -use postgres_connection::{parse_host_port, PgConnectionConfig}; +use postgres_connection::PgConnectionConfig; use utils::backoff::{ exponential_backoff, DEFAULT_BASE_BACKOFF_SECONDS, DEFAULT_MAX_BACKOFF_SECONDS, }; +use utils::postgres_client::wal_stream_connection_config; use utils::{ id::{NodeId, TenantTimelineId}, lsn::Lsn, @@ -211,11 +212,14 @@ async fn subscribe_for_timeline_updates( id: TenantTimelineId, ) -> Streaming { let mut attempt = 0; + let cancel = shutdown_token(); + loop { exponential_backoff( attempt, DEFAULT_BASE_BACKOFF_SECONDS, DEFAULT_MAX_BACKOFF_SECONDS, + &cancel, ) .await; attempt += 1; @@ -876,33 +880,6 @@ impl ReconnectReason { } } -fn wal_stream_connection_config( - TenantTimelineId { - tenant_id, - timeline_id, - }: TenantTimelineId, - listen_pg_addr_str: &str, - auth_token: Option<&str>, - availability_zone: Option<&str>, -) -> anyhow::Result { - let (host, port) = - parse_host_port(listen_pg_addr_str).context("Unable to parse listen_pg_addr_str")?; - let port = port.unwrap_or(5432); - let mut connstr = PgConnectionConfig::new_host_port(host, port) - .extend_options([ - "-c".to_owned(), - format!("timeline_id={}", timeline_id), - format!("tenant_id={}", tenant_id), - ]) - .set_password(auth_token.map(|s| s.to_owned())); - - if let Some(availability_zone) = availability_zone { - connstr = connstr.extend_options([format!("availability_zone={}", availability_zone)]); - } - - Ok(connstr) -} - #[cfg(test)] mod tests { use super::*; @@ -918,6 +895,7 @@ mod tests { timeline: SafekeeperTimelineInfo { safekeeper_id: 0, tenant_timeline_id: None, + term: 0, last_log_term: 0, flush_lsn: 0, commit_lsn, @@ -926,6 +904,7 @@ mod tests { peer_horizon_lsn: 0, local_start_lsn: 0, safekeeper_connstr: safekeeper_connstr.to_owned(), + http_connstr: safekeeper_connstr.to_owned(), availability_zone: None, }, latest_update, diff --git a/pageserver/src/tenant/upload_queue.rs b/pageserver/src/tenant/upload_queue.rs index 4880b613bf..64ee7d8114 100644 --- a/pageserver/src/tenant/upload_queue.rs +++ b/pageserver/src/tenant/upload_queue.rs @@ -144,17 +144,16 @@ impl UploadQueue { ); } - let index_part_metadata = index_part.parse_metadata()?; info!( "initializing upload queue with remote index_part.disk_consistent_lsn: {}", - index_part_metadata.disk_consistent_lsn() + index_part.metadata.disk_consistent_lsn() ); let state = UploadQueueInitialized { latest_files: files, latest_files_changes_since_metadata_upload_scheduled: 0, - latest_metadata: index_part_metadata.clone(), - last_uploaded_consistent_lsn: index_part_metadata.disk_consistent_lsn(), + latest_metadata: index_part.metadata.clone(), + last_uploaded_consistent_lsn: index_part.metadata.disk_consistent_lsn(), // what follows are boring default initializations task_counter: 0, num_inprogress_layer_uploads: 0, diff --git a/proxy/src/http/conn_pool.rs b/proxy/src/http/conn_pool.rs index c02ec61945..e771e5d7ed 100644 --- a/proxy/src/http/conn_pool.rs +++ b/proxy/src/http/conn_pool.rs @@ -408,9 +408,9 @@ async fn connect_to_compute_once( let (tx, mut rx) = tokio::sync::watch::channel(session); let conn_id = uuid::Uuid::new_v4(); - let span = info_span!(parent: None, "connection", %conn_info, %conn_id); + let span = info_span!(parent: None, "connection", %conn_id); span.in_scope(|| { - info!(%session, "new connection"); + info!(%conn_info, %session, "new connection"); }); tokio::spawn( @@ -420,26 +420,28 @@ async fn connect_to_compute_once( info!(%session, "changed session"); } - let message = ready!(connection.poll_message(cx)); + loop { + let message = ready!(connection.poll_message(cx)); - match message { - Some(Ok(AsyncMessage::Notice(notice))) => { - info!(%session, "notice: {}", notice); - Poll::Pending + match message { + Some(Ok(AsyncMessage::Notice(notice))) => { + info!(%session, "notice: {}", notice); + } + Some(Ok(AsyncMessage::Notification(notif))) => { + warn!(%session, pid = notif.process_id(), channel = notif.channel(), "notification received"); + } + Some(Ok(_)) => { + warn!(%session, "unknown message"); + } + Some(Err(e)) => { + error!(%session, "connection error: {}", e); + return Poll::Ready(()) + } + None => { + info!("connection closed"); + return Poll::Ready(()) + } } - Some(Ok(AsyncMessage::Notification(notif))) => { - warn!(%session, pid = notif.process_id(), channel = notif.channel(), "notification received"); - Poll::Pending - } - Some(Ok(_)) => { - warn!(%session, "unknown message"); - Poll::Pending - } - Some(Err(e)) => { - error!(%session, "connection error: {}", e); - Poll::Ready(()) - } - None => Poll::Ready(()), } }) .instrument(span) diff --git a/safekeeper/src/bin/safekeeper.rs b/safekeeper/src/bin/safekeeper.rs index 8d2201b0eb..848b1d7644 100644 --- a/safekeeper/src/bin/safekeeper.rs +++ b/safekeeper/src/bin/safekeeper.rs @@ -341,21 +341,35 @@ async fn start_safekeeper(conf: SafeKeeperConf) -> Result<()> { let (wal_backup_launcher_tx, wal_backup_launcher_rx) = mpsc::channel(100); - // Load all timelines from disk to memory. - GlobalTimelines::init(conf.clone(), wal_backup_launcher_tx)?; - // Keep handles to main tasks to die if any of them disappears. let mut tasks_handles: FuturesUnordered> = FuturesUnordered::new(); + // Start wal backup launcher before loading timelines as we'll notify it + // through the channel about timelines which need offloading, not draining + // the channel would cause deadlock. + let current_thread_rt = conf + .current_thread_runtime + .then(|| Handle::try_current().expect("no runtime in main")); + let conf_ = conf.clone(); + let wal_backup_handle = current_thread_rt + .as_ref() + .unwrap_or_else(|| WAL_BACKUP_RUNTIME.handle()) + .spawn(wal_backup::wal_backup_launcher_task_main( + conf_, + wal_backup_launcher_rx, + )) + .map(|res| ("WAL backup launcher".to_owned(), res)); + tasks_handles.push(Box::pin(wal_backup_handle)); + + // Load all timelines from disk to memory. + GlobalTimelines::init(conf.clone(), wal_backup_launcher_tx).await?; + let conf_ = conf.clone(); // Run everything in current thread rt, if asked. if conf.current_thread_runtime { info!("running in current thread runtime"); } - let current_thread_rt = conf - .current_thread_runtime - .then(|| Handle::try_current().expect("no runtime in main")); let wal_service_handle = current_thread_rt .as_ref() @@ -408,17 +422,6 @@ async fn start_safekeeper(conf: SafeKeeperConf) -> Result<()> { .map(|res| ("WAL remover".to_owned(), res)); tasks_handles.push(Box::pin(wal_remover_handle)); - let conf_ = conf.clone(); - let wal_backup_handle = current_thread_rt - .as_ref() - .unwrap_or_else(|| WAL_BACKUP_RUNTIME.handle()) - .spawn(wal_backup::wal_backup_launcher_task_main( - conf_, - wal_backup_launcher_rx, - )) - .map(|res| ("WAL backup launcher".to_owned(), res)); - tasks_handles.push(Box::pin(wal_backup_handle)); - set_build_info_metric(GIT_VERSION); // TODO: update tokio-stream, convert to real async Stream with diff --git a/safekeeper/src/control_file_upgrade.rs b/safekeeper/src/control_file_upgrade.rs index 95cb96fae9..a7f17df797 100644 --- a/safekeeper/src/control_file_upgrade.rs +++ b/safekeeper/src/control_file_upgrade.rs @@ -1,7 +1,6 @@ //! Code to deal with safekeeper control file upgrades use crate::safekeeper::{ - AcceptorState, PersistedPeers, PgUuid, SafeKeeperState, ServerInfo, Term, TermHistory, - TermSwitchEntry, + AcceptorState, PersistedPeers, PgUuid, SafeKeeperState, ServerInfo, Term, TermHistory, TermLsn, }; use anyhow::{bail, Result}; use pq_proto::SystemId; @@ -145,7 +144,7 @@ pub fn upgrade_control_file(buf: &[u8], version: u32) -> Result let oldstate = SafeKeeperStateV1::des(&buf[..buf.len()])?; let ac = AcceptorState { term: oldstate.acceptor_state.term, - term_history: TermHistory(vec![TermSwitchEntry { + term_history: TermHistory(vec![TermLsn { term: oldstate.acceptor_state.epoch, lsn: Lsn(0), }]), diff --git a/safekeeper/src/http/routes.rs b/safekeeper/src/http/routes.rs index 411bfa295c..6104b54f44 100644 --- a/safekeeper/src/http/routes.rs +++ b/safekeeper/src/http/routes.rs @@ -15,8 +15,11 @@ use tokio::fs::File; use tokio::io::AsyncReadExt; use utils::http::endpoint::request_span; +use crate::receive_wal::WalReceiverState; use crate::safekeeper::ServerInfo; use crate::safekeeper::Term; +use crate::send_wal::WalSenderState; +use crate::timeline::PeerInfo; use crate::{debug_dump, pull_timeline}; use crate::timelines_global_map::TimelineDeleteForceResult; @@ -99,6 +102,9 @@ pub struct TimelineStatus { pub peer_horizon_lsn: Lsn, #[serde_as(as = "DisplayFromStr")] pub remote_consistent_lsn: Lsn, + pub peers: Vec, + pub walsenders: Vec, + pub walreceivers: Vec, } fn check_permission(request: &Request, tenant_id: Option) -> Result<(), ApiError> { @@ -136,6 +142,7 @@ async fn timeline_status_handler(request: Request) -> Result) -> Result) -> Result, term: Term, lsn: Lsn) -> any let history = tli.get_state().await.1.acceptor_state.term_history; let history = history.up_to(lsn.checked_sub(1u64).unwrap()); let mut history_entries = history.0; - history_entries.push(TermSwitchEntry { term, lsn }); + history_entries.push(TermLsn { term, lsn }); let history = TermHistory(history_entries); let proposer_elected_request = ProposerAcceptorMessage::Elected(ProposerElected { diff --git a/safekeeper/src/lib.rs b/safekeeper/src/lib.rs index f8adb86250..d785d0e53a 100644 --- a/safekeeper/src/lib.rs +++ b/safekeeper/src/lib.rs @@ -19,6 +19,7 @@ pub mod json_ctrl; pub mod metrics; pub mod pull_timeline; pub mod receive_wal; +pub mod recovery; pub mod remove_wal; pub mod safekeeper; pub mod send_wal; diff --git a/safekeeper/src/pull_timeline.rs b/safekeeper/src/pull_timeline.rs index 8d8ef6192c..a2ed4c0cf4 100644 --- a/safekeeper/src/pull_timeline.rs +++ b/safekeeper/src/pull_timeline.rs @@ -227,7 +227,9 @@ async fn pull_timeline(status: TimelineStatus, host: String) -> Result tokio::fs::create_dir_all(conf.tenant_dir(&ttid.tenant_id)).await?; tokio::fs::rename(tli_dir_path, &timeline_path).await?; - let tli = GlobalTimelines::load_timeline(ttid).context("Failed to load timeline after copy")?; + let tli = GlobalTimelines::load_timeline(ttid) + .await + .context("Failed to load timeline after copy")?; info!( "Loaded timeline {}, flush_lsn={}", diff --git a/safekeeper/src/receive_wal.rs b/safekeeper/src/receive_wal.rs index a5e99c5f0a..cf63eace21 100644 --- a/safekeeper/src/receive_wal.rs +++ b/safekeeper/src/receive_wal.rs @@ -11,11 +11,16 @@ use crate::wal_service::ConnectionId; use crate::GlobalTimelines; use anyhow::{anyhow, Context}; use bytes::BytesMut; +use parking_lot::MappedMutexGuard; +use parking_lot::Mutex; +use parking_lot::MutexGuard; use postgres_backend::CopyStreamHandlerEnd; use postgres_backend::PostgresBackend; use postgres_backend::PostgresBackendReader; use postgres_backend::QueryError; use pq_proto::BeMessage; +use serde::Deserialize; +use serde::Serialize; use std::net::SocketAddr; use std::sync::Arc; use tokio::io::AsyncRead; @@ -32,6 +37,105 @@ use tracing::*; use utils::id::TenantTimelineId; use utils::lsn::Lsn; +/// Registry of WalReceivers (compute connections). Timeline holds it (wrapped +/// in Arc). +pub struct WalReceivers { + mutex: Mutex, +} + +/// Id under which walreceiver is registered in shmem. +type WalReceiverId = usize; + +impl WalReceivers { + pub fn new() -> Arc { + Arc::new(WalReceivers { + mutex: Mutex::new(WalReceiversShared { slots: Vec::new() }), + }) + } + + /// Register new walreceiver. Returned guard provides access to the slot and + /// automatically deregisters in Drop. + pub fn register(self: &Arc) -> WalReceiverGuard { + let slots = &mut self.mutex.lock().slots; + let walreceiver = WalReceiverState::Voting; + // find empty slot or create new one + let pos = if let Some(pos) = slots.iter().position(|s| s.is_none()) { + slots[pos] = Some(walreceiver); + pos + } else { + let pos = slots.len(); + slots.push(Some(walreceiver)); + pos + }; + WalReceiverGuard { + id: pos, + walreceivers: self.clone(), + } + } + + /// Get reference to locked slot contents. Slot must exist (registered + /// earlier). + fn get_slot<'a>( + self: &'a Arc, + id: WalReceiverId, + ) -> MappedMutexGuard<'a, WalReceiverState> { + MutexGuard::map(self.mutex.lock(), |locked| { + locked.slots[id] + .as_mut() + .expect("walreceiver doesn't exist") + }) + } + + /// Get number of walreceivers (compute connections). + pub fn get_num(self: &Arc) -> usize { + self.mutex.lock().slots.iter().flatten().count() + } + + /// Get state of all walreceivers. + pub fn get_all(self: &Arc) -> Vec { + self.mutex.lock().slots.iter().flatten().cloned().collect() + } + + /// Unregister walsender. + fn unregister(self: &Arc, id: WalReceiverId) { + let mut shared = self.mutex.lock(); + shared.slots[id] = None; + } +} + +/// Only a few connections are expected (normally one), so store in Vec. +struct WalReceiversShared { + slots: Vec>, +} + +/// Walreceiver status. Currently only whether it passed voting stage and +/// started receiving the stream, but it is easy to add more if needed. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub enum WalReceiverState { + Voting, + Streaming, +} + +/// Scope guard to access slot in WalSenders registry and unregister from it in +/// Drop. +pub struct WalReceiverGuard { + id: WalReceiverId, + walreceivers: Arc, +} + +impl WalReceiverGuard { + /// Get reference to locked shared state contents. + fn get(&self) -> MappedMutexGuard { + self.walreceivers.get_slot(self.id) + } +} + +impl Drop for WalReceiverGuard { + fn drop(&mut self) { + self.walreceivers.unregister(self.id); + } +} + const MSG_QUEUE_SIZE: usize = 256; const REPLY_QUEUE_SIZE: usize = 16; @@ -246,10 +350,13 @@ impl WalAcceptor { /// it must mean that network thread terminated. async fn run(&mut self) -> anyhow::Result<()> { // Register the connection and defer unregister. - self.tli.on_compute_connect().await?; - let _guard = ComputeConnectionGuard { + // Order of the next two lines is important: we want first to remove our entry and then + // update status which depends on registered connections. + let _compute_conn_guard = ComputeConnectionGuard { timeline: Arc::clone(&self.tli), }; + let walreceiver_guard = self.tli.get_walreceivers().register(); + self.tli.update_status_notify().await?; // After this timestamp we will stop processing AppendRequests and send a response // to the walproposer. walproposer sends at least one AppendRequest per second, @@ -263,6 +370,11 @@ impl WalAcceptor { } let mut next_msg = opt_msg.unwrap(); + // Update walreceiver state in shmem for reporting. + if let ProposerAcceptorMessage::Elected(_) = &next_msg { + *walreceiver_guard.get() = WalReceiverState::Streaming; + } + let reply_msg = if matches!(next_msg, ProposerAcceptorMessage::AppendRequest(_)) { // loop through AppendRequest's while it's readily available to // write as many WAL as possible without fsyncing @@ -311,6 +423,7 @@ impl WalAcceptor { } } +/// Calls update_status_notify in drop to update timeline status. struct ComputeConnectionGuard { timeline: Arc, } @@ -318,11 +431,9 @@ struct ComputeConnectionGuard { impl Drop for ComputeConnectionGuard { fn drop(&mut self) { let tli = self.timeline.clone(); - // tokio forbids to call blocking_send inside the runtime, and see - // comments in on_compute_disconnect why we call blocking_send. tokio::spawn(async move { - if let Err(e) = tli.on_compute_disconnect().await { - error!("failed to unregister compute connection: {}", e); + if let Err(e) = tli.update_status_notify().await { + error!("failed to update timeline status: {}", e); } }); } diff --git a/safekeeper/src/recovery.rs b/safekeeper/src/recovery.rs new file mode 100644 index 0000000000..90ba3d2e16 --- /dev/null +++ b/safekeeper/src/recovery.rs @@ -0,0 +1,40 @@ +//! This module implements pulling WAL from peer safekeepers if compute can't +//! provide it, i.e. safekeeper lags too much. + +use std::sync::Arc; + +use tokio::{select, time::sleep, time::Duration}; +use tracing::{info, instrument}; + +use crate::{timeline::Timeline, SafeKeeperConf}; + +/// Entrypoint for per timeline task which always runs, checking whether +/// recovery for this safekeeper is needed and starting it if so. +#[instrument(name = "recovery task", skip_all, fields(ttid = %tli.ttid))] +pub async fn recovery_main(tli: Arc, _conf: SafeKeeperConf) { + info!("started"); + let mut cancellation_rx = match tli.get_cancellation_rx() { + Ok(rx) => rx, + Err(_) => { + info!("timeline canceled during task start"); + return; + } + }; + + select! { + _ = recovery_main_loop(tli) => { unreachable!() } + _ = cancellation_rx.changed() => { + info!("stopped"); + } + } +} + +const CHECK_INTERVAL_MS: u64 = 2000; + +/// Check regularly whether we need to start recovery. +async fn recovery_main_loop(_tli: Arc) { + let check_duration = Duration::from_millis(CHECK_INTERVAL_MS); + loop { + sleep(check_duration).await; + } +} diff --git a/safekeeper/src/safekeeper.rs b/safekeeper/src/safekeeper.rs index d0b14a1282..b9fcd2c0b2 100644 --- a/safekeeper/src/safekeeper.rs +++ b/safekeeper/src/safekeeper.rs @@ -34,22 +34,33 @@ pub const UNKNOWN_SERVER_VERSION: u32 = 0; /// Consensus logical timestamp. pub type Term = u64; -const INVALID_TERM: Term = 0; +pub const INVALID_TERM: Term = 0; -#[derive(Debug, Clone, Copy, Serialize, Deserialize)] -pub struct TermSwitchEntry { +#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)] +pub struct TermLsn { pub term: Term, pub lsn: Lsn, } + +// Creation from tuple provides less typing (e.g. for unit tests). +impl From<(Term, Lsn)> for TermLsn { + fn from(pair: (Term, Lsn)) -> TermLsn { + TermLsn { + term: pair.0, + lsn: pair.1, + } + } +} + #[derive(Clone, Serialize, Deserialize)] -pub struct TermHistory(pub Vec); +pub struct TermHistory(pub Vec); impl TermHistory { pub fn empty() -> TermHistory { TermHistory(Vec::new()) } - // Parse TermHistory as n_entries followed by TermSwitchEntry pairs + // Parse TermHistory as n_entries followed by TermLsn pairs pub fn from_bytes(bytes: &mut Bytes) -> Result { if bytes.remaining() < 4 { bail!("TermHistory misses len"); @@ -60,7 +71,7 @@ impl TermHistory { if bytes.remaining() < 16 { bail!("TermHistory is incomplete"); } - res.push(TermSwitchEntry { + res.push(TermLsn { term: bytes.get_u64_le(), lsn: bytes.get_u64_le().into(), }) @@ -557,12 +568,17 @@ where .up_to(self.flush_lsn()) } + /// Get current term. + pub fn get_term(&self) -> Term { + self.state.acceptor_state.term + } + pub fn get_epoch(&self) -> Term { self.state.acceptor_state.get_epoch(self.flush_lsn()) } /// wal_store wrapper avoiding commit_lsn <= flush_lsn violation when we don't have WAL yet. - fn flush_lsn(&self) -> Lsn { + pub fn flush_lsn(&self) -> Lsn { max(self.wal_store.flush_lsn(), self.state.timeline_start_lsn) } @@ -1138,7 +1154,7 @@ mod tests { let pem = ProposerElected { term: 1, start_streaming_at: Lsn(1), - term_history: TermHistory(vec![TermSwitchEntry { + term_history: TermHistory(vec![TermLsn { term: 1, lsn: Lsn(3), }]), diff --git a/safekeeper/src/send_wal.rs b/safekeeper/src/send_wal.rs index 808177d2c4..b684083446 100644 --- a/safekeeper/src/send_wal.rs +++ b/safekeeper/src/send_wal.rs @@ -2,12 +2,12 @@ //! with the "START_REPLICATION" message, and registry of walsenders. use crate::handler::SafekeeperPostgresHandler; -use crate::safekeeper::Term; +use crate::safekeeper::{Term, TermLsn}; use crate::timeline::Timeline; use crate::wal_service::ConnectionId; use crate::wal_storage::WalReader; use crate::GlobalTimelines; -use anyhow::Context as AnyhowContext; +use anyhow::{bail, Context as AnyhowContext}; use bytes::Bytes; use parking_lot::Mutex; use postgres_backend::PostgresBackend; @@ -390,26 +390,25 @@ impl SafekeeperPostgresHandler { self.appname.clone(), )); - let commit_lsn_watch_rx = tli.get_commit_lsn_watch_rx(); - - // Walproposer gets special handling: safekeeper must give proposer all - // local WAL till the end, whether committed or not (walproposer will - // hang otherwise). That's because walproposer runs the consensus and - // synchronizes safekeepers on the most advanced one. + // Walsender can operate in one of two modes which we select by + // application_name: give only committed WAL (used by pageserver) or all + // existing WAL (up to flush_lsn, used by walproposer or peer recovery). + // The second case is always driven by a consensus leader which term + // must generally be also supplied. However we're sloppy to do this in + // walproposer recovery which will be removed soon. So TODO is to make + // it not Option'al then. // - // There is a small risk of this WAL getting concurrently garbaged if - // another compute rises which collects majority and starts fixing log - // on this safekeeper itself. That's ok as (old) proposer will never be - // able to commit such WAL. - let stop_pos: Option = if self.is_walproposer_recovery() { - let wal_end = tli.get_flush_lsn().await; - Some(wal_end) + // Fetching WAL without term in recovery creates a small risk of this + // WAL getting concurrently garbaged if another compute rises which + // collects majority and starts fixing log on this safekeeper itself. + // That's ok as (old) proposer will never be able to commit such WAL. + let end_watch = if self.is_walproposer_recovery() { + EndWatch::Flush(tli.get_term_flush_lsn_watch_rx()) } else { - None + EndWatch::Commit(tli.get_commit_lsn_watch_rx()) }; - - // take the latest commit_lsn if don't have stop_pos - let end_pos = stop_pos.unwrap_or(*commit_lsn_watch_rx.borrow()); + // we don't check term here; it will be checked on first waiting/WAL reading anyway. + let end_pos = end_watch.get(); if end_pos < start_pos { warn!( @@ -419,8 +418,10 @@ impl SafekeeperPostgresHandler { } info!( - "starting streaming from {:?} till {:?}, available WAL ends at {}", - start_pos, stop_pos, end_pos + "starting streaming from {:?}, available WAL ends at {}, recovery={}", + start_pos, + end_pos, + matches!(end_watch, EndWatch::Flush(_)) ); // switch to copy @@ -445,9 +446,8 @@ impl SafekeeperPostgresHandler { appname, start_pos, end_pos, - stop_pos, term, - commit_lsn_watch_rx, + end_watch, ws_guard: ws_guard.clone(), wal_reader, send_buf: [0; MAX_SEND_SIZE], @@ -466,6 +466,32 @@ impl SafekeeperPostgresHandler { } } +/// Walsender streams either up to commit_lsn (normally) or flush_lsn in the +/// given term (recovery by walproposer or peer safekeeper). +enum EndWatch { + Commit(Receiver), + Flush(Receiver), +} + +impl EndWatch { + /// Get current end of WAL. + fn get(&self) -> Lsn { + match self { + EndWatch::Commit(r) => *r.borrow(), + EndWatch::Flush(r) => r.borrow().lsn, + } + } + + /// Wait for the update. + async fn changed(&mut self) -> anyhow::Result<()> { + match self { + EndWatch::Commit(r) => r.changed().await?, + EndWatch::Flush(r) => r.changed().await?, + } + Ok(()) + } +} + /// A half driving sending WAL. struct WalSender<'a, IO> { pgb: &'a mut PostgresBackend, @@ -480,14 +506,12 @@ struct WalSender<'a, IO> { // We send this LSN to the receiver as wal_end, so that it knows how much // WAL this safekeeper has. This LSN should be as fresh as possible. end_pos: Lsn, - // If present, terminate after reaching this position; used by walproposer - // in recovery. - stop_pos: Option, /// When streaming uncommitted part, the term the client acts as the leader /// in. Streaming is stopped if local term changes to a different (higher) /// value. term: Option, - commit_lsn_watch_rx: Receiver, + /// Watch channel receiver to learn end of available WAL (and wait for its advancement). + end_watch: EndWatch, ws_guard: Arc, wal_reader: WalReader, // buffer for readling WAL into to send it @@ -497,29 +521,20 @@ struct WalSender<'a, IO> { impl WalSender<'_, IO> { /// Send WAL until /// - an error occurs - /// - if we are streaming to walproposer, we've streamed until stop_pos - /// (recovery finished) - /// - receiver is caughtup and there is no computes + /// - receiver is caughtup and there is no computes (if streaming up to commit_lsn) /// /// Err(CopyStreamHandlerEnd) is always returned; Result is used only for ? /// convenience. async fn run(&mut self) -> Result<(), CopyStreamHandlerEnd> { loop { - // If we are streaming to walproposer, check it is time to stop. - if let Some(stop_pos) = self.stop_pos { - if self.start_pos >= stop_pos { - // recovery finished - return Err(CopyStreamHandlerEnd::ServerInitiated(format!( - "ending streaming to walproposer at {}, recovery finished", - self.start_pos - ))); - } - } else { - // Wait for the next portion if it is not there yet, or just - // update our end of WAL available for sending value, we - // communicate it to the receiver. - self.wait_wal().await?; - } + // Wait for the next portion if it is not there yet, or just + // update our end of WAL available for sending value, we + // communicate it to the receiver. + self.wait_wal().await?; + assert!( + self.end_pos > self.start_pos, + "nothing to send after waiting for WAL" + ); // try to send as much as available, capped by MAX_SEND_SIZE let mut send_size = self @@ -567,7 +582,7 @@ impl WalSender<'_, IO> { /// exit in the meanwhile async fn wait_wal(&mut self) -> Result<(), CopyStreamHandlerEnd> { loop { - self.end_pos = *self.commit_lsn_watch_rx.borrow(); + self.end_pos = self.end_watch.get(); if self.end_pos > self.start_pos { // We have something to send. trace!("got end_pos {:?}, streaming", self.end_pos); @@ -575,27 +590,31 @@ impl WalSender<'_, IO> { } // Wait for WAL to appear, now self.end_pos == self.start_pos. - if let Some(lsn) = wait_for_lsn(&mut self.commit_lsn_watch_rx, self.start_pos).await? { + if let Some(lsn) = wait_for_lsn(&mut self.end_watch, self.term, self.start_pos).await? { self.end_pos = lsn; trace!("got end_pos {:?}, streaming", self.end_pos); return Ok(()); } - // Timed out waiting for WAL, check for termination and send KA - if let Some(remote_consistent_lsn) = self - .ws_guard - .walsenders - .get_ws_remote_consistent_lsn(self.ws_guard.id) - { - if self.tli.should_walsender_stop(remote_consistent_lsn).await { - // Terminate if there is nothing more to send. - // Note that "ending streaming" part of the string is used by - // pageserver to identify WalReceiverError::SuccessfulCompletion, - // do not change this string without updating pageserver. - return Err(CopyStreamHandlerEnd::ServerInitiated(format!( + // Timed out waiting for WAL, check for termination and send KA. + // Check for termination only if we are streaming up to commit_lsn + // (to pageserver). + if let EndWatch::Commit(_) = self.end_watch { + if let Some(remote_consistent_lsn) = self + .ws_guard + .walsenders + .get_ws_remote_consistent_lsn(self.ws_guard.id) + { + if self.tli.should_walsender_stop(remote_consistent_lsn).await { + // Terminate if there is nothing more to send. + // Note that "ending streaming" part of the string is used by + // pageserver to identify WalReceiverError::SuccessfulCompletion, + // do not change this string without updating pageserver. + return Err(CopyStreamHandlerEnd::ServerInitiated(format!( "ending streaming to {:?} at {}, receiver is caughtup and there is no computes", self.appname, self.start_pos, ))); + } } } @@ -663,22 +682,32 @@ impl ReplyReader { const POLL_STATE_TIMEOUT: Duration = Duration::from_secs(1); -/// Wait until we have commit_lsn > lsn or timeout expires. Returns -/// - Ok(Some(commit_lsn)) if needed lsn is successfully observed; +/// Wait until we have available WAL > start_pos or timeout expires. Returns +/// - Ok(Some(end_pos)) if needed lsn is successfully observed; /// - Ok(None) if timeout expired; -/// - Err in case of error (if watch channel is in trouble, shouldn't happen). -async fn wait_for_lsn(rx: &mut Receiver, lsn: Lsn) -> anyhow::Result> { +/// - Err in case of error -- only if 1) term changed while fetching in recovery +/// mode 2) watch channel closed, which must never happen. +async fn wait_for_lsn( + rx: &mut EndWatch, + client_term: Option, + start_pos: Lsn, +) -> anyhow::Result> { let res = timeout(POLL_STATE_TIMEOUT, async move { - let mut commit_lsn; loop { - rx.changed().await?; - commit_lsn = *rx.borrow(); - if commit_lsn > lsn { - break; + let end_pos = rx.get(); + if end_pos > start_pos { + return Ok(end_pos); } + if let EndWatch::Flush(rx) = rx { + let curr_term = rx.borrow().term; + if let Some(client_term) = client_term { + if curr_term != client_term { + bail!("term changed: requested {}, now {}", client_term, curr_term); + } + } + } + rx.changed().await?; } - - Ok(commit_lsn) }) .await; diff --git a/safekeeper/src/timeline.rs b/safekeeper/src/timeline.rs index df51c45aff..3e066de34f 100644 --- a/safekeeper/src/timeline.rs +++ b/safekeeper/src/timeline.rs @@ -3,8 +3,11 @@ use anyhow::{anyhow, bail, Result}; use postgres_ffi::XLogSegNo; +use serde::{Deserialize, Serialize}; +use serde_with::serde_as; use tokio::fs; +use serde_with::DisplayFromStr; use std::cmp::max; use std::path::PathBuf; use std::sync::Arc; @@ -23,9 +26,11 @@ use utils::{ use storage_broker::proto::SafekeeperTimelineInfo; use storage_broker::proto::TenantTimelineId as ProtoTenantTimelineId; +use crate::receive_wal::WalReceivers; +use crate::recovery::recovery_main; use crate::safekeeper::{ AcceptorProposerMessage, ProposerAcceptorMessage, SafeKeeper, SafeKeeperState, - SafekeeperMemState, ServerInfo, Term, + SafekeeperMemState, ServerInfo, Term, TermLsn, INVALID_TERM, }; use crate::send_wal::WalSenders; use crate::{control_file, safekeeper::UNKNOWN_SERVER_VERSION}; @@ -36,18 +41,25 @@ use crate::SafeKeeperConf; use crate::{debug_dump, wal_storage}; /// Things safekeeper should know about timeline state on peers. -#[derive(Debug, Clone)] +#[serde_as] +#[derive(Debug, Clone, Serialize, Deserialize)] pub struct PeerInfo { pub sk_id: NodeId, /// Term of the last entry. _last_log_term: Term, /// LSN of the last record. + #[serde_as(as = "DisplayFromStr")] _flush_lsn: Lsn, + #[serde_as(as = "DisplayFromStr")] pub commit_lsn: Lsn, /// Since which LSN safekeeper has WAL. TODO: remove this once we fill new /// sk since backup_lsn. + #[serde_as(as = "DisplayFromStr")] pub local_start_lsn: Lsn, - /// When info was received. + /// When info was received. Serde annotations are not very useful but make + /// the code compile -- we don't rely on this field externally. + #[serde(skip)] + #[serde(default = "Instant::now")] ts: Instant, } @@ -164,8 +176,8 @@ impl SharedState { }) } - fn is_active(&self, remote_consistent_lsn: Lsn) -> bool { - self.is_wal_backup_required() + fn is_active(&self, num_computes: usize, remote_consistent_lsn: Lsn) -> bool { + self.is_wal_backup_required(num_computes) // FIXME: add tracking of relevant pageservers and check them here individually, // otherwise migration won't work (we suspend too early). || remote_consistent_lsn < self.sk.inmem.commit_lsn @@ -173,29 +185,34 @@ impl SharedState { /// Mark timeline active/inactive and return whether s3 offloading requires /// start/stop action. - fn update_status(&mut self, remote_consistent_lsn: Lsn, ttid: TenantTimelineId) -> bool { - let is_active = self.is_active(remote_consistent_lsn); + fn update_status( + &mut self, + num_computes: usize, + remote_consistent_lsn: Lsn, + ttid: TenantTimelineId, + ) -> bool { + let is_active = self.is_active(num_computes, remote_consistent_lsn); if self.active != is_active { info!("timeline {} active={} now", ttid, is_active); } self.active = is_active; - self.is_wal_backup_action_pending() + self.is_wal_backup_action_pending(num_computes) } /// Should we run s3 offloading in current state? - fn is_wal_backup_required(&self) -> bool { + fn is_wal_backup_required(&self, num_computes: usize) -> bool { let seg_size = self.get_wal_seg_size(); - self.num_computes > 0 || + num_computes > 0 || // Currently only the whole segment is offloaded, so compare segment numbers. - (self.sk.inmem.commit_lsn.segment_number(seg_size) > - self.sk.inmem.backup_lsn.segment_number(seg_size)) + (self.sk.inmem.commit_lsn.segment_number(seg_size) > + self.sk.inmem.backup_lsn.segment_number(seg_size)) } /// Is current state of s3 offloading is not what it ought to be? - fn is_wal_backup_action_pending(&self) -> bool { - let res = self.wal_backup_active != self.is_wal_backup_required(); + fn is_wal_backup_action_pending(&self, num_computes: usize) -> bool { + let res = self.wal_backup_active != self.is_wal_backup_required(num_computes); if res { - let action_pending = if self.is_wal_backup_required() { + let action_pending = if self.is_wal_backup_required(num_computes) { "start" } else { "stop" @@ -210,8 +227,8 @@ impl SharedState { /// Returns whether s3 offloading is required and sets current status as /// matching. - fn wal_backup_attend(&mut self) -> bool { - self.wal_backup_active = self.is_wal_backup_required(); + fn wal_backup_attend(&mut self, num_computes: usize) -> bool { + self.wal_backup_active = self.is_wal_backup_required(num_computes); self.wal_backup_active } @@ -231,8 +248,9 @@ impl SharedState { tenant_id: ttid.tenant_id.as_ref().to_owned(), timeline_id: ttid.timeline_id.as_ref().to_owned(), }), + term: self.sk.state.acceptor_state.term, last_log_term: self.sk.get_epoch(), - flush_lsn: self.sk.wal_store.flush_lsn().0, + flush_lsn: self.sk.flush_lsn().0, // note: this value is not flushed to control file yet and can be lost commit_lsn: self.sk.inmem.commit_lsn.0, remote_consistent_lsn: remote_consistent_lsn.0, @@ -241,6 +259,7 @@ impl SharedState { .advertise_pg_addr .to_owned() .unwrap_or(conf.listen_pg_addr.clone()), + http_connstr: conf.listen_http_addr.to_owned(), backup_lsn: self.sk.inmem.backup_lsn.0, local_start_lsn: self.sk.state.local_start_lsn.0, availability_zone: conf.availability_zone.clone(), @@ -290,11 +309,19 @@ pub struct Timeline { commit_lsn_watch_tx: watch::Sender, commit_lsn_watch_rx: watch::Receiver, + /// Broadcasts (current term, flush_lsn) updates, walsender is interested in + /// them when sending in recovery mode (to walproposer or peers). Note: this + /// is just a notification, WAL reading should always done with lock held as + /// term can change otherwise. + term_flush_lsn_watch_tx: watch::Sender, + term_flush_lsn_watch_rx: watch::Receiver, + /// Safekeeper and other state, that should remain consistent and /// synchronized with the disk. This is tokio mutex as we write WAL to disk /// while holding it, ensuring that consensus checks are in order. mutex: Mutex, walsenders: Arc, + walreceivers: Arc, /// Cancellation channel. Delete/cancel will send `true` here as a cancellation signal. cancellation_tx: watch::Sender, @@ -310,16 +337,20 @@ pub struct Timeline { impl Timeline { /// Load existing timeline from disk. pub fn load_timeline( - conf: SafeKeeperConf, + conf: &SafeKeeperConf, ttid: TenantTimelineId, wal_backup_launcher_tx: Sender, ) -> Result { let _enter = info_span!("load_timeline", timeline = %ttid.timeline_id).entered(); - let shared_state = SharedState::restore(&conf, &ttid)?; + let shared_state = SharedState::restore(conf, &ttid)?; let rcl = shared_state.sk.state.remote_consistent_lsn; let (commit_lsn_watch_tx, commit_lsn_watch_rx) = watch::channel(shared_state.sk.state.commit_lsn); + let (term_flush_lsn_watch_tx, term_flush_lsn_watch_rx) = watch::channel(TermLsn::from(( + shared_state.sk.get_term(), + shared_state.sk.flush_lsn(), + ))); let (cancellation_tx, cancellation_rx) = watch::channel(false); Ok(Timeline { @@ -327,8 +358,11 @@ impl Timeline { wal_backup_launcher_tx, commit_lsn_watch_tx, commit_lsn_watch_rx, + term_flush_lsn_watch_tx, + term_flush_lsn_watch_rx, mutex: Mutex::new(shared_state), walsenders: WalSenders::new(rcl), + walreceivers: WalReceivers::new(), cancellation_rx, cancellation_tx, timeline_dir: conf.timeline_dir(&ttid), @@ -337,7 +371,7 @@ impl Timeline { /// Create a new timeline, which is not yet persisted to disk. pub fn create_empty( - conf: SafeKeeperConf, + conf: &SafeKeeperConf, ttid: TenantTimelineId, wal_backup_launcher_tx: Sender, server_info: ServerInfo, @@ -345,6 +379,8 @@ impl Timeline { local_start_lsn: Lsn, ) -> Result { let (commit_lsn_watch_tx, commit_lsn_watch_rx) = watch::channel(Lsn::INVALID); + let (term_flush_lsn_watch_tx, term_flush_lsn_watch_rx) = + watch::channel(TermLsn::from((INVALID_TERM, Lsn::INVALID))); let (cancellation_tx, cancellation_rx) = watch::channel(false); let state = SafeKeeperState::new(&ttid, server_info, vec![], commit_lsn, local_start_lsn); @@ -353,20 +389,27 @@ impl Timeline { wal_backup_launcher_tx, commit_lsn_watch_tx, commit_lsn_watch_rx, - mutex: Mutex::new(SharedState::create_new(&conf, &ttid, state)?), + term_flush_lsn_watch_tx, + term_flush_lsn_watch_rx, + mutex: Mutex::new(SharedState::create_new(conf, &ttid, state)?), walsenders: WalSenders::new(Lsn(0)), + walreceivers: WalReceivers::new(), cancellation_rx, cancellation_tx, timeline_dir: conf.timeline_dir(&ttid), }) } - /// Initialize fresh timeline on disk and start background tasks. If bootstrap + /// Initialize fresh timeline on disk and start background tasks. If init /// fails, timeline is cancelled and cannot be used anymore. /// - /// Bootstrap is transactional, so if it fails, created files will be deleted, + /// Init is transactional, so if it fails, created files will be deleted, /// and state on disk should remain unchanged. - pub async fn bootstrap(&self, shared_state: &mut MutexGuard<'_, SharedState>) -> Result<()> { + pub async fn init_new( + self: &Arc, + shared_state: &mut MutexGuard<'_, SharedState>, + conf: &SafeKeeperConf, + ) -> Result<()> { match fs::metadata(&self.timeline_dir).await { Ok(_) => { // Timeline directory exists on disk, we should leave state unchanged @@ -382,7 +425,7 @@ impl Timeline { // Create timeline directory. fs::create_dir_all(&self.timeline_dir).await?; - // Write timeline to disk and TODO: start background tasks. + // Write timeline to disk and start background tasks. if let Err(e) = shared_state.sk.persist().await { // Bootstrap failed, cancel timeline and remove timeline directory. self.cancel(shared_state); @@ -396,12 +439,16 @@ impl Timeline { return Err(e); } - - // TODO: add more initialization steps here - self.update_status(shared_state); + self.bootstrap(conf); Ok(()) } + /// Bootstrap new or existing timeline starting background stasks. + pub fn bootstrap(self: &Arc, conf: &SafeKeeperConf) { + // Start recovery task which always runs on the timeline. + tokio::spawn(recovery_main(self.clone(), conf.clone())); + } + /// Delete timeline from disk completely, by removing timeline directory. Background /// timeline activities will stop eventually. pub async fn delete_from_disk( @@ -435,46 +482,38 @@ impl Timeline { *self.cancellation_rx.borrow() } + /// Returns watch channel which gets value when timeline is cancelled. It is + /// guaranteed to have not cancelled value observed (errors otherwise). + pub fn get_cancellation_rx(&self) -> Result> { + let rx = self.cancellation_rx.clone(); + if *rx.borrow() { + bail!(TimelineError::Cancelled(self.ttid)); + } + Ok(rx) + } + /// Take a writing mutual exclusive lock on timeline shared_state. pub async fn write_shared_state(&self) -> MutexGuard { self.mutex.lock().await } fn update_status(&self, shared_state: &mut SharedState) -> bool { - shared_state.update_status(self.get_walsenders().get_remote_consistent_lsn(), self.ttid) + shared_state.update_status( + self.walreceivers.get_num(), + self.get_walsenders().get_remote_consistent_lsn(), + self.ttid, + ) } - /// Register compute connection, starting timeline-related activity if it is - /// not running yet. - pub async fn on_compute_connect(&self) -> Result<()> { + /// Update timeline status and kick wal backup launcher to stop/start offloading if needed. + pub async fn update_status_notify(&self) -> Result<()> { if self.is_cancelled() { bail!(TimelineError::Cancelled(self.ttid)); } - - let is_wal_backup_action_pending: bool; - { + let is_wal_backup_action_pending: bool = { let mut shared_state = self.write_shared_state().await; - shared_state.num_computes += 1; - is_wal_backup_action_pending = self.update_status(&mut shared_state); - } - // Wake up wal backup launcher, if offloading not started yet. - if is_wal_backup_action_pending { - // Can fail only if channel to a static thread got closed, which is not normal at all. - self.wal_backup_launcher_tx.send(self.ttid).await?; - } - Ok(()) - } - - /// De-register compute connection, shutting down timeline activity if - /// pageserver doesn't need catchup. - pub async fn on_compute_disconnect(&self) -> Result<()> { - let is_wal_backup_action_pending: bool; - { - let mut shared_state = self.write_shared_state().await; - shared_state.num_computes -= 1; - is_wal_backup_action_pending = self.update_status(&mut shared_state); - } - // Wake up wal backup launcher, if it is time to stop the offloading. + self.update_status(&mut shared_state) + }; if is_wal_backup_action_pending { // Can fail only if channel to a static thread got closed, which is not normal at all. self.wal_backup_launcher_tx.send(self.ttid).await?; @@ -519,7 +558,9 @@ impl Timeline { return false; } - self.write_shared_state().await.wal_backup_attend() + self.write_shared_state() + .await + .wal_backup_attend(self.walreceivers.get_num()) } /// Returns commit_lsn watch channel. @@ -527,6 +568,11 @@ impl Timeline { self.commit_lsn_watch_rx.clone() } + /// Returns term_flush_lsn watch channel. + pub fn get_term_flush_lsn_watch_rx(&self) -> watch::Receiver { + self.term_flush_lsn_watch_rx.clone() + } + /// Pass arrived message to the safekeeper. pub async fn process_msg( &self, @@ -538,6 +584,7 @@ impl Timeline { let mut rmsg: Option; let commit_lsn: Lsn; + let term_flush_lsn: TermLsn; { let mut shared_state = self.write_shared_state().await; rmsg = shared_state.sk.process_msg(msg).await?; @@ -551,8 +598,11 @@ impl Timeline { } commit_lsn = shared_state.sk.inmem.commit_lsn; + term_flush_lsn = + TermLsn::from((shared_state.sk.get_term(), shared_state.sk.flush_lsn())); } self.commit_lsn_watch_tx.send(commit_lsn)?; + self.term_flush_lsn_watch_tx.send(term_flush_lsn)?; Ok(rmsg) } @@ -650,6 +700,10 @@ impl Timeline { &self.walsenders } + pub fn get_walreceivers(&self) -> &Arc { + &self.walreceivers + } + /// Returns flush_lsn. pub async fn get_flush_lsn(&self) -> Lsn { self.write_shared_state().await.sk.wal_store.flush_lsn() diff --git a/safekeeper/src/timelines_global_map.rs b/safekeeper/src/timelines_global_map.rs index eda5b9044e..2f591655a9 100644 --- a/safekeeper/src/timelines_global_map.rs +++ b/safekeeper/src/timelines_global_map.rs @@ -11,7 +11,7 @@ use serde::Serialize; use std::collections::HashMap; use std::path::PathBuf; use std::str::FromStr; -use std::sync::{Arc, Mutex, MutexGuard}; +use std::sync::{Arc, Mutex}; use tokio::sync::mpsc::Sender; use tracing::*; use utils::id::{TenantId, TenantTimelineId, TimelineId}; @@ -71,19 +71,23 @@ pub struct GlobalTimelines; impl GlobalTimelines { /// Inject dependencies needed for the timeline constructors and load all timelines to memory. - pub fn init( + pub async fn init( conf: SafeKeeperConf, wal_backup_launcher_tx: Sender, ) -> Result<()> { - let mut state = TIMELINES_STATE.lock().unwrap(); - assert!(state.wal_backup_launcher_tx.is_none()); - state.wal_backup_launcher_tx = Some(wal_backup_launcher_tx); - state.conf = Some(conf); + // clippy isn't smart enough to understand that drop(state) releases the + // lock, so use explicit block + let tenants_dir = { + let mut state = TIMELINES_STATE.lock().unwrap(); + assert!(state.wal_backup_launcher_tx.is_none()); + state.wal_backup_launcher_tx = Some(wal_backup_launcher_tx); + state.conf = Some(conf); - // Iterate through all directories and load tenants for all directories - // named as a valid tenant_id. + // Iterate through all directories and load tenants for all directories + // named as a valid tenant_id. + state.get_conf().workdir.clone() + }; let mut tenant_count = 0; - let tenants_dir = state.get_conf().workdir.clone(); for tenants_dir_entry in std::fs::read_dir(&tenants_dir) .with_context(|| format!("failed to list tenants dir {}", tenants_dir.display()))? { @@ -93,7 +97,7 @@ impl GlobalTimelines { TenantId::from_str(tenants_dir_entry.file_name().to_str().unwrap_or("")) { tenant_count += 1; - GlobalTimelines::load_tenant_timelines(&mut state, tenant_id)?; + GlobalTimelines::load_tenant_timelines(tenant_id).await?; } } Err(e) => error!( @@ -108,7 +112,7 @@ impl GlobalTimelines { info!( "found {} tenants directories, successfully loaded {} timelines", tenant_count, - state.timelines.len() + TIMELINES_STATE.lock().unwrap().timelines.len() ); Ok(()) } @@ -116,17 +120,21 @@ impl GlobalTimelines { /// Loads all timelines for the given tenant to memory. Returns fs::read_dir /// errors if any. /// - /// Note: This function (and all reading/loading below) is sync because - /// timelines are loaded while holding GlobalTimelinesState lock. Which is - /// fine as this is called only from single threaded main runtime on boot, - /// but clippy complains anyway, and suppressing that isn't trivial as async - /// is the keyword, ha. That only other user is pull_timeline.rs for which - /// being blocked is not that bad, and we can do spawn_blocking. - fn load_tenant_timelines( - state: &mut MutexGuard<'_, GlobalTimelinesState>, - tenant_id: TenantId, - ) -> Result<()> { - let timelines_dir = state.get_conf().tenant_dir(&tenant_id); + /// It is async for update_status_notify sake. Since TIMELINES_STATE lock is + /// sync and there is no important reason to make it async (it is always + /// held for a short while) we just lock and unlock it for each timeline -- + /// this function is called during init when nothing else is running, so + /// this is fine. + async fn load_tenant_timelines(tenant_id: TenantId) -> Result<()> { + let (conf, wal_backup_launcher_tx) = { + let state = TIMELINES_STATE.lock().unwrap(); + ( + state.get_conf().clone(), + state.wal_backup_launcher_tx.as_ref().unwrap().clone(), + ) + }; + + let timelines_dir = conf.tenant_dir(&tenant_id); for timelines_dir_entry in std::fs::read_dir(&timelines_dir) .with_context(|| format!("failed to list timelines dir {}", timelines_dir.display()))? { @@ -136,13 +144,16 @@ impl GlobalTimelines { TimelineId::from_str(timeline_dir_entry.file_name().to_str().unwrap_or("")) { let ttid = TenantTimelineId::new(tenant_id, timeline_id); - match Timeline::load_timeline( - state.get_conf().clone(), - ttid, - state.wal_backup_launcher_tx.as_ref().unwrap().clone(), - ) { + match Timeline::load_timeline(&conf, ttid, wal_backup_launcher_tx.clone()) { Ok(timeline) => { - state.timelines.insert(ttid, Arc::new(timeline)); + let tli = Arc::new(timeline); + TIMELINES_STATE + .lock() + .unwrap() + .timelines + .insert(ttid, tli.clone()); + tli.bootstrap(&conf); + tli.update_status_notify().await.unwrap(); } // If we can't load a timeline, it's most likely because of a corrupted // directory. We will log an error and won't allow to delete/recreate @@ -168,18 +179,22 @@ impl GlobalTimelines { } /// Load timeline from disk to the memory. - pub fn load_timeline(ttid: TenantTimelineId) -> Result> { + pub async fn load_timeline(ttid: TenantTimelineId) -> Result> { let (conf, wal_backup_launcher_tx) = TIMELINES_STATE.lock().unwrap().get_dependencies(); - match Timeline::load_timeline(conf, ttid, wal_backup_launcher_tx) { + match Timeline::load_timeline(&conf, ttid, wal_backup_launcher_tx) { Ok(timeline) => { let tli = Arc::new(timeline); + // TODO: prevent concurrent timeline creation/loading TIMELINES_STATE .lock() .unwrap() .timelines .insert(ttid, tli.clone()); + + tli.bootstrap(&conf); + Ok(tli) } // If we can't load a timeline, it's bad. Caller will figure it out. @@ -217,7 +232,7 @@ impl GlobalTimelines { info!("creating new timeline {}", ttid); let timeline = Arc::new(Timeline::create_empty( - conf, + &conf, ttid, wal_backup_launcher_tx, server_info, @@ -240,23 +255,24 @@ impl GlobalTimelines { // Write the new timeline to the disk and start background workers. // Bootstrap is transactional, so if it fails, the timeline will be deleted, // and the state on disk should remain unchanged. - if let Err(e) = timeline.bootstrap(&mut shared_state).await { - // Note: the most likely reason for bootstrap failure is that the timeline + if let Err(e) = timeline.init_new(&mut shared_state, &conf).await { + // Note: the most likely reason for init failure is that the timeline // directory already exists on disk. This happens when timeline is corrupted // and wasn't loaded from disk on startup because of that. We want to preserve // the timeline directory in this case, for further inspection. // TODO: this is an unusual error, perhaps we should send it to sentry // TODO: compute will try to create timeline every second, we should add backoff - error!("failed to bootstrap timeline {}: {}", ttid, e); + error!("failed to init new timeline {}: {}", ttid, e); - // Timeline failed to bootstrap, it cannot be used. Remove it from the map. + // Timeline failed to init, it cannot be used. Remove it from the map. TIMELINES_STATE.lock().unwrap().timelines.remove(&ttid); return Err(e); } // We are done with bootstrap, release the lock, return the timeline. // {} block forces release before .await } + timeline.update_status_notify().await?; timeline.wal_backup_launcher_tx.send(timeline.ttid).await?; Ok(timeline) } diff --git a/scripts/combine_control_files.py b/scripts/combine_control_files.py deleted file mode 100644 index b1fe73881e..0000000000 --- a/scripts/combine_control_files.py +++ /dev/null @@ -1,76 +0,0 @@ -#! /usr/bin/env python3 -# Script to generate ext_index.json metadata file -# that stores content of the control files and location of extension archives -# for all extensions in extensions subdir. -import argparse -import json -import subprocess -from pathlib import Path - -""" -# ext_index.json example: -{ - "public_extensions": [ - "anon" - ], - "library_index": { - "anon": "anon", - // for more complex extensions like postgis - // we might have something like: - // address_standardizer: postgis - // postgis_tiger: postgis - }, - "extension_data": { - "anon": { - "control_data": { - "anon.control": "# PostgreSQL Anonymizer (anon) extension \ncomment = 'Data anonymization tools' \ndefault_version = '1.1.0' \ndirectory='extension/anon' \nrelocatable = false \nrequires = 'pgcrypto' \nsuperuser = false \nmodule_pathname = '$libdir/anon' \ntrusted = true \n" - }, - "archive_path": "5648391853/v15/extensions/anon.tar.zst" - } - } -} -""" - -if __name__ == "__main__": - parser = argparse.ArgumentParser(description="generate ext_index.json") - parser.add_argument("pg_version", type=str, choices=["v14", "v15"], help="pg_version") - parser.add_argument("BUILD_TAG", type=str, help="BUILD_TAG for this compute image") - parser.add_argument("--public_extensions", type=str, help="list of public extensions") - args = parser.parse_args() - pg_version = args.pg_version - BUILD_TAG = args.BUILD_TAG - public_ext_list = args.public_extensions.split(",") - - ext_index = {} - library_index = {} - EXT_PATH = Path("extensions") - for extension in EXT_PATH.iterdir(): - if extension.is_dir(): - control_data = {} - for control_file in extension.glob("*.control"): - if control_file.suffix != ".control": - continue - with open(control_file, "r") as f: - control_data[control_file.name] = f.read() - ext_index[extension.name] = { - "control_data": control_data, - "archive_path": f"{BUILD_TAG}/{pg_version}/extensions/{extension.name}.tar.zst", - } - elif extension.suffix == ".zst": - file_list = ( - str(subprocess.check_output(["tar", "tf", str(extension)]), "utf-8") - .strip() - .split("\n") - ) - for file in file_list: - if file.endswith(".so") and file.startswith("lib/"): - lib_name = file[4:-3] - library_index[lib_name] = extension.name.replace(".tar.zst", "") - - all_data = { - "public_extensions": public_ext_list, - "library_index": library_index, - "extension_data": ext_index, - } - with open("ext_index.json", "w") as f: - json.dump(all_data, f) diff --git a/scripts/flaky_tests.py b/scripts/flaky_tests.py index a3b29909e5..5ef2f76798 100755 --- a/scripts/flaky_tests.py +++ b/scripts/flaky_tests.py @@ -12,25 +12,26 @@ import psycopg2.extras # We call the test "flaky" if it failed at least once on the main branch in the last N=10 days. FLAKY_TESTS_QUERY = """ SELECT - DISTINCT parent_suite, suite, test + DISTINCT parent_suite, suite, REGEXP_REPLACE(test, '(release|debug)-pg(\\d+)-?', '') as deparametrized_test FROM ( SELECT - revision, - jsonb_array_elements(data -> 'children') -> 'name' as parent_suite, - jsonb_array_elements(jsonb_array_elements(data -> 'children') -> 'children') -> 'name' as suite, - jsonb_array_elements(jsonb_array_elements(jsonb_array_elements(data -> 'children') -> 'children') -> 'children') -> 'name' as test, - jsonb_array_elements(jsonb_array_elements(jsonb_array_elements(data -> 'children') -> 'children') -> 'children') -> 'status' as status, - jsonb_array_elements(jsonb_array_elements(jsonb_array_elements(data -> 'children') -> 'children') -> 'children') -> 'retriesStatusChange' as retries_status_change, - to_timestamp((jsonb_array_elements(jsonb_array_elements(jsonb_array_elements(data -> 'children') -> 'children') -> 'children') -> 'time' -> 'start')::bigint / 1000)::date as timestamp + reference, + jsonb_array_elements(data -> 'children') ->> 'name' as parent_suite, + jsonb_array_elements(jsonb_array_elements(data -> 'children') -> 'children') ->> 'name' as suite, + jsonb_array_elements(jsonb_array_elements(jsonb_array_elements(data -> 'children') -> 'children') -> 'children') ->> 'name' as test, + jsonb_array_elements(jsonb_array_elements(jsonb_array_elements(data -> 'children') -> 'children') -> 'children') ->> 'status' as status, + jsonb_array_elements(jsonb_array_elements(jsonb_array_elements(data -> 'children') -> 'children') -> 'children') ->> 'retriesStatusChange' as retries_status_change, + to_timestamp((jsonb_array_elements(jsonb_array_elements(jsonb_array_elements(data -> 'children') -> 'children') -> 'children') -> 'time' ->> 'start')::bigint / 1000)::date as timestamp FROM regress_test_results - WHERE - reference = 'refs/heads/main' ) data WHERE timestamp > CURRENT_DATE - INTERVAL '%s' day - AND (status::text IN ('"failed"', '"broken"') OR retries_status_change::boolean) + AND ( + (status IN ('failed', 'broken') AND reference = 'refs/heads/main') + OR retries_status_change::boolean + ) ; """ @@ -40,6 +41,9 @@ def main(args: argparse.Namespace): interval_days = args.days output = args.output + build_type = args.build_type + pg_version = args.pg_version + res: DefaultDict[str, DefaultDict[str, Dict[str, bool]]] res = defaultdict(lambda: defaultdict(dict)) @@ -55,8 +59,21 @@ def main(args: argparse.Namespace): rows = [] for row in rows: - logging.info(f"\t{row['parent_suite'].replace('.', '/')}/{row['suite']}.py::{row['test']}") - res[row["parent_suite"]][row["suite"]][row["test"]] = True + # We don't want to automatically rerun tests in a performance suite + if row["parent_suite"] != "test_runner.regress": + continue + + deparametrized_test = row["deparametrized_test"] + dash_if_needed = "" if deparametrized_test.endswith("[]") else "-" + parametrized_test = deparametrized_test.replace( + "[", + f"[{build_type}-pg{pg_version}{dash_if_needed}", + ) + res[row["parent_suite"]][row["suite"]][parametrized_test] = True + + logging.info( + f"\t{row['parent_suite'].replace('.', '/')}/{row['suite']}.py::{parametrized_test}" + ) logging.info(f"saving results to {output.name}") json.dump(res, output, indent=2) @@ -77,6 +94,18 @@ if __name__ == "__main__": type=int, help="how many days to look back for flaky tests (default: 10)", ) + parser.add_argument( + "--build-type", + required=True, + type=str, + help="for which build type to create list of flaky tests (debug or release)", + ) + parser.add_argument( + "--pg-version", + required=True, + type=int, + help="for which Postgres version to create list of flaky tests (14, 15, etc.)", + ) parser.add_argument( "connstr", help="connection string to the test results database", diff --git a/storage_broker/benches/rps.rs b/storage_broker/benches/rps.rs index 6563fec8b6..a0c8e1f749 100644 --- a/storage_broker/benches/rps.rs +++ b/storage_broker/benches/rps.rs @@ -125,6 +125,7 @@ async fn publish(client: Option, n_keys: u64) { tenant_id: vec![0xFF; 16], timeline_id: tli_from_u64(counter % n_keys), }), + term: 0, last_log_term: 0, flush_lsn: counter, commit_lsn: 2, @@ -132,6 +133,7 @@ async fn publish(client: Option, n_keys: u64) { remote_consistent_lsn: 4, peer_horizon_lsn: 5, safekeeper_connstr: "zenith-1-sk-1.local:7676".to_owned(), + http_connstr: "zenith-1-sk-1.local:7677".to_owned(), local_start_lsn: 0, availability_zone: None, }; diff --git a/storage_broker/proto/broker.proto b/storage_broker/proto/broker.proto index 4b2de1a8e5..aa9d62a29f 100644 --- a/storage_broker/proto/broker.proto +++ b/storage_broker/proto/broker.proto @@ -22,6 +22,8 @@ message SubscribeSafekeeperInfoRequest { message SafekeeperTimelineInfo { uint64 safekeeper_id = 1; TenantTimelineId tenant_timeline_id = 2; + // Safekeeper term + uint64 term = 12; // Term of the last entry. uint64 last_log_term = 3; // LSN of the last record. @@ -36,6 +38,8 @@ message SafekeeperTimelineInfo { uint64 local_start_lsn = 9; // A connection string to use for WAL receiving. string safekeeper_connstr = 10; + // HTTP endpoint connection string + string http_connstr = 13; // Availability zone of a safekeeper. optional string availability_zone = 11; } diff --git a/storage_broker/src/bin/storage_broker.rs b/storage_broker/src/bin/storage_broker.rs index 597d9860d8..23d1bee354 100644 --- a/storage_broker/src/bin/storage_broker.rs +++ b/storage_broker/src/bin/storage_broker.rs @@ -519,6 +519,7 @@ mod tests { tenant_id: vec![0x00; 16], timeline_id, }), + term: 0, last_log_term: 0, flush_lsn: 1, commit_lsn: 2, @@ -526,6 +527,7 @@ mod tests { remote_consistent_lsn: 4, peer_horizon_lsn: 5, safekeeper_connstr: "neon-1-sk-1.local:7676".to_owned(), + http_connstr: "neon-1-sk-1.local:7677".to_owned(), local_start_lsn: 0, availability_zone: None, } diff --git a/test_runner/fixtures/metrics.py b/test_runner/fixtures/metrics.py index 52d32df43e..a6a25da332 100644 --- a/test_runner/fixtures/metrics.py +++ b/test_runner/fixtures/metrics.py @@ -70,6 +70,7 @@ PAGESERVER_GLOBAL_METRICS: Tuple[str, ...] = ( "pageserver_getpage_reconstruct_seconds_count", "pageserver_getpage_reconstruct_seconds_sum", *[f"pageserver_basebackup_query_seconds_{x}" for x in ["bucket", "count", "sum"]], + *histogram("pageserver_smgr_query_seconds_global"), *histogram("pageserver_read_num_fs_layers"), *histogram("pageserver_getpage_get_reconstruct_data_seconds"), *histogram("pageserver_wait_lsn_seconds"), diff --git a/test_runner/fixtures/pageserver/utils.py b/test_runner/fixtures/pageserver/utils.py index 9767d5b547..708e90d9b2 100644 --- a/test_runner/fixtures/pageserver/utils.py +++ b/test_runner/fixtures/pageserver/utils.py @@ -233,10 +233,19 @@ if TYPE_CHECKING: def assert_prefix_empty(neon_env_builder: "NeonEnvBuilder", prefix: Optional[str] = None): response = list_prefix(neon_env_builder, prefix) - objects = response.get("Contents") - assert ( - response["KeyCount"] == 0 - ), f"remote dir with prefix {prefix} is not empty after deletion: {objects}" + keys = response["KeyCount"] + objects = response.get("Contents", []) + + if keys != 0 and len(objects) == 0: + # this has been seen in one case with mock_s3: + # https://neon-github-public-dev.s3.amazonaws.com/reports/pr-4938/6000769714/index.html#suites/3556ed71f2d69272a7014df6dcb02317/ca01e4f4d8d9a11f + # looking at moto impl, it might be there's a race with common prefix (sub directory) not going away with deletes + common_prefixes = response.get("CommonPrefixes", []) + log.warn( + f"contradicting ListObjectsV2 response with KeyCount={keys} and Contents={objects}, CommonPrefixes={common_prefixes}" + ) + + assert keys == 0, f"remote dir with prefix {prefix} is not empty after deletion: {objects}" def assert_prefix_not_empty(neon_env_builder: "NeonEnvBuilder", prefix: Optional[str] = None): diff --git a/test_runner/regress/test_ondemand_download.py b/test_runner/regress/test_ondemand_download.py index 0640e65e57..a4e86e0519 100644 --- a/test_runner/regress/test_ondemand_download.py +++ b/test_runner/regress/test_ondemand_download.py @@ -369,7 +369,7 @@ def test_download_remote_layers_api( filled_current_physical = get_api_current_physical_size() log.info(filled_current_physical) filled_size = get_resident_physical_size() - log.info(filled_size) + log.info(f"filled_size: {filled_size}") assert filled_current_physical == filled_size, "we don't yet do layer eviction" env.pageserver.stop() @@ -377,7 +377,7 @@ def test_download_remote_layers_api( # remove all the layer files # XXX only delete some of the layer files, to show that it really just downloads all the layers for layer in (Path(env.repo_dir) / "tenants").glob("*/timelines/*/*-*_*"): - log.info(f"unlinking layer {layer}") + log.info(f"unlinking layer {layer.name}") layer.unlink() # Shut down safekeepers before starting the pageserver. @@ -403,7 +403,7 @@ def test_download_remote_layers_api( filled_current_physical == get_api_current_physical_size() ), "current_physical_size is sum of loaded layer sizes, independent of whether local or remote" post_unlink_size = get_resident_physical_size() - log.info(post_unlink_size) + log.info(f"post_unlink_size: {post_unlink_size}") assert ( post_unlink_size < filled_size ), "we just deleted layers and didn't cause anything to re-download them yet" diff --git a/vm-cgconfig.conf b/vm-cgconfig.conf deleted file mode 100644 index a2e201708e..0000000000 --- a/vm-cgconfig.conf +++ /dev/null @@ -1,12 +0,0 @@ -# Configuration for cgroups in VM compute nodes -group neon-postgres { - perm { - admin { - uid = vm-informant; - } - task { - gid = users; - } - } - memory {} -} diff --git a/workspace_hack/Cargo.toml b/workspace_hack/Cargo.toml index d79c7a4104..831cc6f6b1 100644 --- a/workspace_hack/Cargo.toml +++ b/workspace_hack/Cargo.toml @@ -14,11 +14,13 @@ publish = false ### BEGIN HAKARI SECTION [dependencies] anyhow = { version = "1", features = ["backtrace"] } +axum = { version = "0.6", features = ["ws"] } bytes = { version = "1", features = ["serde"] } chrono = { version = "0.4", default-features = false, features = ["clock", "serde"] } clap = { version = "4", features = ["derive", "string"] } clap_builder = { version = "4", default-features = false, features = ["color", "help", "std", "string", "suggestions", "usage"] } crossbeam-utils = { version = "0.8" } +digest = { version = "0.10", features = ["mac", "std"] } either = { version = "1" } fail = { version = "0.5", default-features = false, features = ["failpoints"] } futures = { version = "0.3" } @@ -27,6 +29,7 @@ futures-core = { version = "0.3" } futures-executor = { version = "0.3" } futures-sink = { version = "0.3" } futures-util = { version = "0.3", features = ["channel", "io", "sink"] } +hyper = { version = "0.14", features = ["full"] } itertools = { version = "0.10" } libc = { version = "0.2", features = ["extra_traits"] } log = { version = "0.4", default-features = false, features = ["std"] } @@ -45,6 +48,7 @@ rustls = { version = "0.20", features = ["dangerous_configuration"] } scopeguard = { version = "1" } serde = { version = "1", features = ["alloc", "derive"] } serde_json = { version = "1", features = ["raw_value"] } +smallvec = { version = "1", default-features = false, features = ["write"] } socket2 = { version = "0.4", default-features = false, features = ["all"] } tokio = { version = "1", features = ["fs", "io-std", "io-util", "macros", "net", "process", "rt-multi-thread", "signal", "test-util"] } tokio-rustls = { version = "0.23" } @@ -54,7 +58,6 @@ toml_edit = { version = "0.19", features = ["serde"] } tower = { version = "0.4", features = ["balance", "buffer", "limit", "retry", "timeout", "util"] } tracing = { version = "0.1", features = ["log"] } tracing-core = { version = "0.1" } -tracing-subscriber = { version = "0.3", default-features = false, features = ["env-filter", "fmt", "json", "smallvec", "tracing-log"] } url = { version = "2", features = ["serde"] } [build-dependencies]