diff --git a/.github/workflows/benchmarking.yml b/.github/workflows/benchmarking.yml index 08b74a2656..172b904331 100644 --- a/.github/workflows/benchmarking.yml +++ b/.github/workflows/benchmarking.yml @@ -180,7 +180,8 @@ jobs: image: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/rust:pinned options: --init - timeout-minutes: 360 # 6h + # Increase timeout to 8h, default timeout is 6h + timeout-minutes: 480 steps: - uses: actions/checkout@v3 @@ -321,8 +322,6 @@ jobs: image: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/rust:pinned options: --init - timeout-minutes: 360 # 6h - steps: - uses: actions/checkout@v3 @@ -414,8 +413,6 @@ jobs: image: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/rust:pinned options: --init - timeout-minutes: 360 # 6h - steps: - uses: actions/checkout@v3 @@ -501,8 +498,6 @@ jobs: image: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/rust:pinned options: --init - timeout-minutes: 360 # 6h - steps: - uses: actions/checkout@v3 diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index 336dea04eb..068e34420f 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -264,7 +264,7 @@ jobs: export REMOTE_STORAGE_S3_BUCKET=neon-github-public-dev export REMOTE_STORAGE_S3_REGION=eu-central-1 # Avoid `$CARGO_FEATURES` since there's no `testing` feature in the e2e tests now - ${cov_prefix} cargo test $CARGO_FLAGS --package remote_storage --test pagination_tests -- s3_pagination_should_work --exact + ${cov_prefix} cargo test $CARGO_FLAGS --package remote_storage --test test_real_s3 - name: Install rust binaries run: | @@ -492,19 +492,24 @@ jobs: env: COMMIT_URL: ${{ github.server_url }}/${{ github.repository }}/commit/${{ github.event.pull_request.head.sha || github.sha }} run: | - scripts/coverage \ - --dir=/tmp/coverage report \ + scripts/coverage --dir=/tmp/coverage \ + report \ --input-objects=/tmp/coverage/binaries.list \ --commit-url=${COMMIT_URL} \ --format=github + scripts/coverage --dir=/tmp/coverage \ + report \ + --input-objects=/tmp/coverage/binaries.list \ + --format=lcov + - name: Upload coverage report id: upload-coverage-report env: BUCKET: neon-github-public-dev COMMIT_SHA: ${{ github.event.pull_request.head.sha || github.sha }} run: | - aws s3 cp --only-show-errors --recursive /tmp/coverage/report s3://neon-github-public-dev/code-coverage/${COMMIT_SHA} + aws s3 cp --only-show-errors --recursive /tmp/coverage/report s3://${BUCKET}/code-coverage/${COMMIT_SHA} REPORT_URL=https://${BUCKET}.s3.amazonaws.com/code-coverage/${COMMIT_SHA}/index.html echo "report-url=${REPORT_URL}" >> $GITHUB_OUTPUT @@ -618,51 +623,6 @@ jobs: - name: Cleanup ECR folder run: rm -rf ~/.ecr - - neon-image-depot: - # For testing this will run side-by-side for a few merges. - # This action is not really optimized yet, but gets the job done - runs-on: [ self-hosted, gen3, large ] - needs: [ tag ] - container: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/base:pinned - permissions: - contents: read - id-token: write - - steps: - - name: Checkout - uses: actions/checkout@v3 - with: - submodules: true - fetch-depth: 0 - - - name: Setup go - uses: actions/setup-go@v3 - with: - go-version: '1.19' - - - name: Set up Depot CLI - uses: depot/setup-action@v1 - - - name: Install Crane & ECR helper - run: go install github.com/awslabs/amazon-ecr-credential-helper/ecr-login/cli/docker-credential-ecr-login@69c85dc22db6511932bbf119e1a0cc5c90c69a7f # v0.6.0 - - - name: Configure ECR login - run: | - mkdir /github/home/.docker/ - echo "{\"credsStore\":\"ecr-login\"}" > /github/home/.docker/config.json - - - name: Build and push - uses: depot/build-push-action@v1 - with: - # if no depot.json file is at the root of your repo, you must specify the project id - project: nrdv0s4kcs - push: true - tags: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/neon:depot-${{needs.tag.outputs.build-tag}} - build-args: | - GIT_VERSION=${{ github.sha }} - REPOSITORY=369495373322.dkr.ecr.eu-central-1.amazonaws.com - compute-tools-image: runs-on: [ self-hosted, gen3, large ] needs: [ tag ] @@ -699,6 +659,7 @@ jobs: --cache-repo 369495373322.dkr.ecr.eu-central-1.amazonaws.com/cache --context . --build-arg GIT_VERSION=${{ github.sha }} + --build-arg BUILD_TAG=${{needs.tag.outputs.build-tag}} --build-arg REPOSITORY=369495373322.dkr.ecr.eu-central-1.amazonaws.com --dockerfile Dockerfile.compute-tools --destination 369495373322.dkr.ecr.eu-central-1.amazonaws.com/compute-tools:${{needs.tag.outputs.build-tag}} @@ -756,10 +717,40 @@ jobs: --context . --build-arg GIT_VERSION=${{ github.sha }} --build-arg PG_VERSION=${{ matrix.version }} + --build-arg BUILD_TAG=${{needs.tag.outputs.build-tag}} --build-arg REPOSITORY=369495373322.dkr.ecr.eu-central-1.amazonaws.com --dockerfile Dockerfile.compute-node --destination 369495373322.dkr.ecr.eu-central-1.amazonaws.com/compute-node-${{ matrix.version }}:${{needs.tag.outputs.build-tag}} --destination neondatabase/compute-node-${{ matrix.version }}:${{needs.tag.outputs.build-tag}} + --cleanup + + # Due to a kaniko bug, we can't use cache for extensions image, thus it takes about the same amount of time as compute-node image to build (~10 min) + # During the transition period we need to have extensions in both places (in S3 and in compute-node image), + # so we won't build extension twice, but extract them from compute-node. + # + # For now we use extensions image only for new custom extensitons + - name: Kaniko build extensions only + run: | + # Kaniko is suposed to clean up after itself if --cleanup flag is set, but it doesn't. + # Despite some fixes were made in https://github.com/GoogleContainerTools/kaniko/pull/2504 (in kaniko v1.11.0), + # it still fails with error: + # error building image: could not save file: copying file: symlink postgres /kaniko/1/usr/local/pgsql/bin/postmaster: file exists + # + # Ref https://github.com/GoogleContainerTools/kaniko/issues/1406 + find /kaniko -maxdepth 1 -mindepth 1 -type d -regex "/kaniko/[0-9]*" -exec rm -rv {} \; + + /kaniko/executor --reproducible --snapshot-mode=redo --skip-unused-stages --cache=true \ + --cache-repo 369495373322.dkr.ecr.eu-central-1.amazonaws.com/cache \ + --context . \ + --build-arg GIT_VERSION=${{ github.sha }} \ + --build-arg PG_VERSION=${{ matrix.version }} \ + --build-arg BUILD_TAG=${{needs.tag.outputs.build-tag}} \ + --build-arg REPOSITORY=369495373322.dkr.ecr.eu-central-1.amazonaws.com \ + --dockerfile Dockerfile.compute-node \ + --destination 369495373322.dkr.ecr.eu-central-1.amazonaws.com/extensions-${{ matrix.version }}:${{needs.tag.outputs.build-tag}} \ + --destination neondatabase/extensions-${{ matrix.version }}:${{needs.tag.outputs.build-tag}} \ + --cleanup \ + --target postgres-extensions # Cleanup script fails otherwise - rm: cannot remove '/nvme/actions-runner/_work/_temp/_github_home/.ecr': Permission denied - name: Cleanup ECR folder @@ -776,7 +767,7 @@ jobs: run: shell: sh -eu {0} env: - VM_BUILDER_VERSION: v0.8.0 + VM_BUILDER_VERSION: v0.11.1 steps: - name: Checkout @@ -797,7 +788,7 @@ jobs: - name: Build vm image run: | - ./vm-builder -src=369495373322.dkr.ecr.eu-central-1.amazonaws.com/compute-node-${{ matrix.version }}:${{needs.tag.outputs.build-tag}} -dst=369495373322.dkr.ecr.eu-central-1.amazonaws.com/vm-compute-node-${{ matrix.version }}:${{needs.tag.outputs.build-tag}} + ./vm-builder -enable-file-cache -src=369495373322.dkr.ecr.eu-central-1.amazonaws.com/compute-node-${{ matrix.version }}:${{needs.tag.outputs.build-tag}} -dst=369495373322.dkr.ecr.eu-central-1.amazonaws.com/vm-compute-node-${{ matrix.version }}:${{needs.tag.outputs.build-tag}} - name: Pushing vm-compute-node image run: | @@ -878,8 +869,10 @@ jobs: crane tag 369495373322.dkr.ecr.eu-central-1.amazonaws.com/compute-tools:${{needs.tag.outputs.build-tag}} latest crane tag 369495373322.dkr.ecr.eu-central-1.amazonaws.com/compute-node-v14:${{needs.tag.outputs.build-tag}} latest crane tag 369495373322.dkr.ecr.eu-central-1.amazonaws.com/vm-compute-node-v14:${{needs.tag.outputs.build-tag}} latest + crane tag 369495373322.dkr.ecr.eu-central-1.amazonaws.com/extensions-v14:${{needs.tag.outputs.build-tag}} latest crane tag 369495373322.dkr.ecr.eu-central-1.amazonaws.com/compute-node-v15:${{needs.tag.outputs.build-tag}} latest crane tag 369495373322.dkr.ecr.eu-central-1.amazonaws.com/vm-compute-node-v15:${{needs.tag.outputs.build-tag}} latest + crane tag 369495373322.dkr.ecr.eu-central-1.amazonaws.com/extensions-v15:${{needs.tag.outputs.build-tag}} latest - name: Push images to production ECR if: | @@ -890,8 +883,10 @@ jobs: crane copy 369495373322.dkr.ecr.eu-central-1.amazonaws.com/compute-tools:${{needs.tag.outputs.build-tag}} 093970136003.dkr.ecr.eu-central-1.amazonaws.com/compute-tools:latest crane copy 369495373322.dkr.ecr.eu-central-1.amazonaws.com/compute-node-v14:${{needs.tag.outputs.build-tag}} 093970136003.dkr.ecr.eu-central-1.amazonaws.com/compute-node-v14:latest crane copy 369495373322.dkr.ecr.eu-central-1.amazonaws.com/vm-compute-node-v14:${{needs.tag.outputs.build-tag}} 093970136003.dkr.ecr.eu-central-1.amazonaws.com/vm-compute-node-v14:latest + crane copy 369495373322.dkr.ecr.eu-central-1.amazonaws.com/extensions-v14:${{needs.tag.outputs.build-tag}} 093970136003.dkr.ecr.eu-central-1.amazonaws.com/extensions-v14:latest crane copy 369495373322.dkr.ecr.eu-central-1.amazonaws.com/compute-node-v15:${{needs.tag.outputs.build-tag}} 093970136003.dkr.ecr.eu-central-1.amazonaws.com/compute-node-v15:latest crane copy 369495373322.dkr.ecr.eu-central-1.amazonaws.com/vm-compute-node-v15:${{needs.tag.outputs.build-tag}} 093970136003.dkr.ecr.eu-central-1.amazonaws.com/vm-compute-node-v15:latest + crane copy 369495373322.dkr.ecr.eu-central-1.amazonaws.com/extensions-v15:${{needs.tag.outputs.build-tag}} 093970136003.dkr.ecr.eu-central-1.amazonaws.com/extensions-v15:latest - name: Configure Docker Hub login run: | @@ -913,16 +908,89 @@ jobs: crane tag neondatabase/compute-tools:${{needs.tag.outputs.build-tag}} latest crane tag neondatabase/compute-node-v14:${{needs.tag.outputs.build-tag}} latest crane tag neondatabase/vm-compute-node-v14:${{needs.tag.outputs.build-tag}} latest + crane tag neondatabase/extensions-v14:${{needs.tag.outputs.build-tag}} latest crane tag neondatabase/compute-node-v15:${{needs.tag.outputs.build-tag}} latest crane tag neondatabase/vm-compute-node-v15:${{needs.tag.outputs.build-tag}} latest + crane tag neondatabase/extensions-v15:${{needs.tag.outputs.build-tag}} latest - name: Cleanup ECR folder run: rm -rf ~/.ecr + upload-postgres-extensions-to-s3: + if: | + (github.ref_name == 'main' || github.ref_name == 'release') && + github.event_name != 'workflow_dispatch' + runs-on: ${{ github.ref_name == 'release' && fromJSON('["self-hosted", "prod", "x64"]') || fromJSON('["self-hosted", "gen3", "small"]') }} + needs: [ tag, promote-images ] + strategy: + fail-fast: false + matrix: + version: [ v14, v15 ] + + env: + # While on transition period we extract public extensions from compute-node image and custom extensions from extensions image. + # Later all the extensions will be moved to extensions image. + EXTENSIONS_IMAGE: ${{ github.ref_name == 'release' && '093970136003' || '369495373322'}}.dkr.ecr.eu-central-1.amazonaws.com/extensions-${{ matrix.version }}:latest + COMPUTE_NODE_IMAGE: ${{ github.ref_name == 'release' && '093970136003' || '369495373322'}}.dkr.ecr.eu-central-1.amazonaws.com/compute-node-${{ matrix.version }}:latest + AWS_ACCESS_KEY_ID: ${{ github.ref_name == 'release' && secrets.AWS_ACCESS_KEY_PROD || secrets.AWS_ACCESS_KEY_DEV }} + AWS_SECRET_ACCESS_KEY: ${{ github.ref_name == 'release' && secrets.AWS_SECRET_KEY_PROD || secrets.AWS_SECRET_KEY_DEV }} + S3_BUCKETS: | + ${{ github.ref_name == 'release' && + 'neon-prod-extensions-ap-southeast-1 neon-prod-extensions-eu-central-1 neon-prod-extensions-us-east-1 neon-prod-extensions-us-east-2 neon-prod-extensions-us-west-2' || + 'neon-dev-extensions-eu-central-1 neon-dev-extensions-eu-west-1 neon-dev-extensions-us-east-2' }} + + steps: + - name: Pull postgres-extensions image + run: | + docker pull ${EXTENSIONS_IMAGE} + docker pull ${COMPUTE_NODE_IMAGE} + + - name: Create postgres-extensions container + id: create-container + run: | + EID=$(docker create ${EXTENSIONS_IMAGE} true) + echo "EID=${EID}" >> $GITHUB_OUTPUT + + CID=$(docker create ${COMPUTE_NODE_IMAGE} true) + echo "CID=${CID}" >> $GITHUB_OUTPUT + + - name: Extract postgres-extensions from container + run: | + rm -rf ./extensions-to-upload ./custom-extensions # Just in case + + # In compute image we have a bit different directory layout + mkdir -p extensions-to-upload/share + docker cp ${{ steps.create-container.outputs.CID }}:/usr/local/share/extension ./extensions-to-upload/share/extension + docker cp ${{ steps.create-container.outputs.CID }}:/usr/local/lib ./extensions-to-upload/lib + + # Delete Neon extensitons (they always present on compute-node image) + rm -rf ./extensions-to-upload/share/extension/neon* + rm -rf ./extensions-to-upload/lib/neon* + + docker cp ${{ steps.create-container.outputs.EID }}:/extensions ./custom-extensions + for EXT_NAME in $(ls ./custom-extensions); do + mkdir -p ./extensions-to-upload/${EXT_NAME}/share + + mv ./custom-extensions/${EXT_NAME}/share/extension ./extensions-to-upload/${EXT_NAME}/share/extension + mv ./custom-extensions/${EXT_NAME}/lib ./extensions-to-upload/${EXT_NAME}/lib + done + + - name: Upload postgres-extensions to S3 + run: | + for BUCKET in $(echo ${S3_BUCKETS}); do + aws s3 cp --recursive --only-show-errors ./extensions-to-upload s3://${BUCKET}/${{ needs.tag.outputs.build-tag }}/${{ matrix.version }} + done + + - name: Cleanup + if: ${{ always() && (steps.create-container.outputs.CID || steps.create-container.outputs.EID) }} + run: | + docker rm ${{ steps.create-container.outputs.CID }} || true + docker rm ${{ steps.create-container.outputs.EID }} || true + deploy: runs-on: [ self-hosted, gen3, small ] container: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/ansible:latest - needs: [ promote-images, tag, regress-tests ] + needs: [ upload-postgres-extensions-to-s3, promote-images, tag, regress-tests ] if: ( github.ref_name == 'main' || github.ref_name == 'release' ) && github.event_name != 'workflow_dispatch' steps: - name: Fix git ownership @@ -954,6 +1022,20 @@ jobs: exit 1 fi + - name: Create git tag + if: github.ref_name == 'release' + uses: actions/github-script@v6 + with: + # Retry script for 5XX server errors: https://github.com/actions/github-script#retries + retries: 5 + script: | + github.rest.git.createRef({ + owner: context.repo.owner, + repo: context.repo.repo, + ref: "refs/tags/${{ needs.tag.outputs.build-tag }}", + sha: context.sha, + }) + promote-compatibility-data: runs-on: [ self-hosted, gen3, small ] container: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 4bce9cdd1e..595ee05514 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -3,6 +3,7 @@ name: Create Release Branch on: schedule: - cron: '0 10 * * 2' + workflow_dispatch: jobs: create_release_branch: diff --git a/Cargo.lock b/Cargo.lock index d390df94e0..4be74614c2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2349,9 +2349,9 @@ checksum = "0ab1bc2a289d34bd04a330323ac98a1b4bc82c9d9fcb1e66b63caa84da26b575" [[package]] name = "openssl" -version = "0.10.52" +version = "0.10.55" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01b8574602df80f7b85fdfc5392fa884a4e3b3f4f35402c070ab34c3d3f78d56" +checksum = "345df152bc43501c5eb9e4654ff05f794effb78d4efe3d53abc158baddc0703d" dependencies = [ "bitflags", "cfg-if", @@ -2381,9 +2381,9 @@ checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" [[package]] name = "openssl-sys" -version = "0.9.87" +version = "0.9.90" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e17f59264b2809d77ae94f0e1ebabc434773f370d6ca667bd223ea10e06cc7e" +checksum = "374533b0e45f3a7ced10fcaeccca020e66656bc03dac384f852e4e5a7a8104a6" dependencies = [ "cc", "libc", @@ -2770,7 +2770,7 @@ dependencies = [ [[package]] name = "postgres" version = "0.19.4" -source = "git+https://github.com/neondatabase/rust-postgres.git?rev=2e9b5f1ddc481d1a98fa79f6b9378ac4f170b7c9#2e9b5f1ddc481d1a98fa79f6b9378ac4f170b7c9" +source = "git+https://github.com/neondatabase/rust-postgres.git?rev=1aaedab101b23f7612042850d8f2036810fa7c7f#1aaedab101b23f7612042850d8f2036810fa7c7f" dependencies = [ "bytes", "fallible-iterator", @@ -2783,7 +2783,7 @@ dependencies = [ [[package]] name = "postgres-native-tls" version = "0.5.0" -source = "git+https://github.com/neondatabase/rust-postgres.git?rev=2e9b5f1ddc481d1a98fa79f6b9378ac4f170b7c9#2e9b5f1ddc481d1a98fa79f6b9378ac4f170b7c9" +source = "git+https://github.com/neondatabase/rust-postgres.git?rev=1aaedab101b23f7612042850d8f2036810fa7c7f#1aaedab101b23f7612042850d8f2036810fa7c7f" dependencies = [ "native-tls", "tokio", @@ -2794,7 +2794,7 @@ dependencies = [ [[package]] name = "postgres-protocol" version = "0.6.4" -source = "git+https://github.com/neondatabase/rust-postgres.git?rev=2e9b5f1ddc481d1a98fa79f6b9378ac4f170b7c9#2e9b5f1ddc481d1a98fa79f6b9378ac4f170b7c9" +source = "git+https://github.com/neondatabase/rust-postgres.git?rev=1aaedab101b23f7612042850d8f2036810fa7c7f#1aaedab101b23f7612042850d8f2036810fa7c7f" dependencies = [ "base64 0.20.0", "byteorder", @@ -2812,7 +2812,7 @@ dependencies = [ [[package]] name = "postgres-types" version = "0.2.4" -source = "git+https://github.com/neondatabase/rust-postgres.git?rev=2e9b5f1ddc481d1a98fa79f6b9378ac4f170b7c9#2e9b5f1ddc481d1a98fa79f6b9378ac4f170b7c9" +source = "git+https://github.com/neondatabase/rust-postgres.git?rev=1aaedab101b23f7612042850d8f2036810fa7c7f#1aaedab101b23f7612042850d8f2036810fa7c7f" dependencies = [ "bytes", "fallible-iterator", @@ -2874,7 +2874,6 @@ dependencies = [ "serde", "thiserror", "utils", - "wal_craft", "workspace_hack", ] @@ -4273,7 +4272,7 @@ dependencies = [ [[package]] name = "tokio-postgres" version = "0.7.7" -source = "git+https://github.com/neondatabase/rust-postgres.git?rev=2e9b5f1ddc481d1a98fa79f6b9378ac4f170b7c9#2e9b5f1ddc481d1a98fa79f6b9378ac4f170b7c9" +source = "git+https://github.com/neondatabase/rust-postgres.git?rev=1aaedab101b23f7612042850d8f2036810fa7c7f#1aaedab101b23f7612042850d8f2036810fa7c7f" dependencies = [ "async-trait", "byteorder", @@ -4894,7 +4893,9 @@ dependencies = [ "once_cell", "postgres", "postgres_ffi", + "regex", "tempfile", + "utils", "workspace_hack", ] diff --git a/Cargo.toml b/Cargo.toml index 1cb8d65948..551a9dc783 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -9,7 +9,20 @@ members = [ "storage_broker", "workspace_hack", "trace", - "libs/*", + "libs/compute_api", + "libs/pageserver_api", + "libs/postgres_ffi", + "libs/safekeeper_api", + "libs/utils", + "libs/consumption_metrics", + "libs/postgres_backend", + "libs/pq_proto", + "libs/tenant_size_model", + "libs/metrics", + "libs/postgres_connection", + "libs/remote_storage", + "libs/tracing-utils", + "libs/postgres_ffi/wal_craft", ] [workspace.package] @@ -127,11 +140,11 @@ env_logger = "0.10" log = "0.4" ## Libraries from neondatabase/ git forks, ideally with changes to be upstreamed -postgres = { git = "https://github.com/neondatabase/rust-postgres.git", rev="2e9b5f1ddc481d1a98fa79f6b9378ac4f170b7c9" } -postgres-native-tls = { git = "https://github.com/neondatabase/rust-postgres.git", rev="2e9b5f1ddc481d1a98fa79f6b9378ac4f170b7c9" } -postgres-protocol = { git = "https://github.com/neondatabase/rust-postgres.git", rev="2e9b5f1ddc481d1a98fa79f6b9378ac4f170b7c9" } -postgres-types = { git = "https://github.com/neondatabase/rust-postgres.git", rev="2e9b5f1ddc481d1a98fa79f6b9378ac4f170b7c9" } -tokio-postgres = { git = "https://github.com/neondatabase/rust-postgres.git", rev="2e9b5f1ddc481d1a98fa79f6b9378ac4f170b7c9" } +postgres = { git = "https://github.com/neondatabase/rust-postgres.git", rev="1aaedab101b23f7612042850d8f2036810fa7c7f" } +postgres-native-tls = { git = "https://github.com/neondatabase/rust-postgres.git", rev="1aaedab101b23f7612042850d8f2036810fa7c7f" } +postgres-protocol = { git = "https://github.com/neondatabase/rust-postgres.git", rev="1aaedab101b23f7612042850d8f2036810fa7c7f" } +postgres-types = { git = "https://github.com/neondatabase/rust-postgres.git", rev="1aaedab101b23f7612042850d8f2036810fa7c7f" } +tokio-postgres = { git = "https://github.com/neondatabase/rust-postgres.git", rev="1aaedab101b23f7612042850d8f2036810fa7c7f" } tokio-tar = { git = "https://github.com/neondatabase/tokio-tar.git", rev="404df61437de0feef49ba2ccdbdd94eb8ad6e142" } ## Other git libraries @@ -167,7 +180,7 @@ tonic-build = "0.9" # This is only needed for proxy's tests. # TODO: we should probably fork `tokio-postgres-rustls` instead. -tokio-postgres = { git = "https://github.com/neondatabase/rust-postgres.git", rev="2e9b5f1ddc481d1a98fa79f6b9378ac4f170b7c9" } +tokio-postgres = { git = "https://github.com/neondatabase/rust-postgres.git", rev="1aaedab101b23f7612042850d8f2036810fa7c7f" } # Changes the MAX_THREADS limit from 4096 to 32768. # This is a temporary workaround for using tracing from many threads in safekeepers code, diff --git a/Dockerfile.compute-node b/Dockerfile.compute-node index de8a904c02..21877f6f24 100644 --- a/Dockerfile.compute-node +++ b/Dockerfile.compute-node @@ -2,6 +2,7 @@ ARG PG_VERSION ARG REPOSITORY=neondatabase ARG IMAGE=rust ARG TAG=pinned +ARG BUILD_TAG ######################################################################################### # @@ -67,7 +68,7 @@ RUN apt update && \ RUN wget https://gitlab.com/Oslandia/SFCGAL/-/archive/v1.3.10/SFCGAL-v1.3.10.tar.gz -O SFCGAL.tar.gz && \ echo "4e39b3b2adada6254a7bdba6d297bb28e1a9835a9f879b74f37e2dab70203232 SFCGAL.tar.gz" | sha256sum --check && \ mkdir sfcgal-src && cd sfcgal-src && tar xvzf ../SFCGAL.tar.gz --strip-components=1 -C . && \ - cmake . && make -j $(getconf _NPROCESSORS_ONLN) && \ + cmake -DCMAKE_BUILD_TYPE=Release . && make -j $(getconf _NPROCESSORS_ONLN) && \ DESTDIR=/sfcgal make install -j $(getconf _NPROCESSORS_ONLN) && \ make clean && cp -R /sfcgal/* / @@ -95,7 +96,7 @@ RUN wget https://github.com/pgRouting/pgrouting/archive/v3.4.2.tar.gz -O pgrouti mkdir pgrouting-src && cd pgrouting-src && tar xvzf ../pgrouting.tar.gz --strip-components=1 -C . && \ mkdir build && \ cd build && \ - cmake .. && \ + cmake -DCMAKE_BUILD_TYPE=Release .. && \ make -j $(getconf _NPROCESSORS_ONLN) && \ make -j $(getconf _NPROCESSORS_ONLN) install && \ echo 'trusted = true' >> /usr/local/pgsql/share/extension/pgrouting.control @@ -355,7 +356,7 @@ RUN apt-get update && \ wget https://github.com/timescale/timescaledb/archive/refs/tags/2.10.1.tar.gz -O timescaledb.tar.gz && \ echo "6fca72a6ed0f6d32d2b3523951ede73dc5f9b0077b38450a029a5f411fdb8c73 timescaledb.tar.gz" | sha256sum --check && \ mkdir timescaledb-src && cd timescaledb-src && tar xvzf ../timescaledb.tar.gz --strip-components=1 -C . && \ - ./bootstrap -DSEND_TELEMETRY_DEFAULT:BOOL=OFF -DUSE_TELEMETRY:BOOL=OFF -DAPACHE_ONLY:BOOL=ON && \ + ./bootstrap -DSEND_TELEMETRY_DEFAULT:BOOL=OFF -DUSE_TELEMETRY:BOOL=OFF -DAPACHE_ONLY:BOOL=ON -DCMAKE_BUILD_TYPE=Release && \ cd build && \ make -j $(getconf _NPROCESSORS_ONLN) && \ make install -j $(getconf _NPROCESSORS_ONLN) && \ @@ -410,7 +411,7 @@ RUN apt-get update && \ mkdir kq_imcx-src && cd kq_imcx-src && tar xvzf ../kq_imcx.tar.gz --strip-components=1 -C . && \ mkdir build && \ cd build && \ - cmake .. && \ + cmake -DCMAKE_BUILD_TYPE=Release .. && \ make -j $(getconf _NPROCESSORS_ONLN) && \ make -j $(getconf _NPROCESSORS_ONLN) install && \ echo 'trusted = true' >> /usr/local/pgsql/share/extension/kq_imcx.control @@ -432,6 +433,108 @@ RUN wget https://github.com/citusdata/pg_cron/archive/refs/tags/v1.5.2.tar.gz -O make -j $(getconf _NPROCESSORS_ONLN) install && \ echo 'trusted = true' >> /usr/local/pgsql/share/extension/pg_cron.control +######################################################################################### +# +# Layer "rdkit-pg-build" +# compile rdkit extension +# +######################################################################################### +FROM build-deps AS rdkit-pg-build +COPY --from=pg-build /usr/local/pgsql/ /usr/local/pgsql/ + +RUN apt-get update && \ + apt-get install -y \ + cmake \ + libboost-iostreams1.74-dev \ + libboost-regex1.74-dev \ + libboost-serialization1.74-dev \ + libboost-system1.74-dev \ + libeigen3-dev \ + libfreetype6-dev + +ENV PATH "/usr/local/pgsql/bin/:/usr/local/pgsql/:$PATH" +RUN wget https://github.com/rdkit/rdkit/archive/refs/tags/Release_2023_03_1.tar.gz -O rdkit.tar.gz && \ + echo "db346afbd0ba52c843926a2a62f8a38c7b774ffab37eaf382d789a824f21996c rdkit.tar.gz" | sha256sum --check && \ + mkdir rdkit-src && cd rdkit-src && tar xvzf ../rdkit.tar.gz --strip-components=1 -C . && \ + cmake \ + -D RDK_BUILD_CAIRO_SUPPORT=OFF \ + -D RDK_BUILD_INCHI_SUPPORT=ON \ + -D RDK_BUILD_AVALON_SUPPORT=ON \ + -D RDK_BUILD_PYTHON_WRAPPERS=OFF \ + -D RDK_BUILD_DESCRIPTORS3D=OFF \ + -D RDK_BUILD_FREESASA_SUPPORT=OFF \ + -D RDK_BUILD_COORDGEN_SUPPORT=ON \ + -D RDK_BUILD_MOLINTERCHANGE_SUPPORT=OFF \ + -D RDK_BUILD_YAEHMOP_SUPPORT=OFF \ + -D RDK_BUILD_STRUCTCHECKER_SUPPORT=OFF \ + -D RDK_USE_URF=OFF \ + -D RDK_BUILD_PGSQL=ON \ + -D RDK_PGSQL_STATIC=ON \ + -D PostgreSQL_CONFIG=pg_config \ + -D PostgreSQL_INCLUDE_DIR=`pg_config --includedir` \ + -D PostgreSQL_TYPE_INCLUDE_DIR=`pg_config --includedir-server` \ + -D PostgreSQL_LIBRARY_DIR=`pg_config --libdir` \ + -D RDK_INSTALL_INTREE=OFF \ + -D CMAKE_BUILD_TYPE=Release \ + . && \ + make -j $(getconf _NPROCESSORS_ONLN) && \ + make -j $(getconf _NPROCESSORS_ONLN) install && \ + echo 'trusted = true' >> /usr/local/pgsql/share/extension/rdkit.control + +######################################################################################### +# +# Layer "pg-uuidv7-pg-build" +# compile pg_uuidv7 extension +# +######################################################################################### +FROM build-deps AS pg-uuidv7-pg-build +COPY --from=pg-build /usr/local/pgsql/ /usr/local/pgsql/ + +ENV PATH "/usr/local/pgsql/bin/:$PATH" +RUN wget https://github.com/fboulnois/pg_uuidv7/archive/refs/tags/v1.0.1.tar.gz -O pg_uuidv7.tar.gz && \ + echo "0d0759ab01b7fb23851ecffb0bce27822e1868a4a5819bfd276101c716637a7a pg_uuidv7.tar.gz" | sha256sum --check && \ + mkdir pg_uuidv7-src && cd pg_uuidv7-src && tar xvzf ../pg_uuidv7.tar.gz --strip-components=1 -C . && \ + make -j $(getconf _NPROCESSORS_ONLN) && \ + make -j $(getconf _NPROCESSORS_ONLN) install && \ + echo 'trusted = true' >> /usr/local/pgsql/share/extension/pg_uuidv7.control + +######################################################################################### +# +# Layer "pg-roaringbitmap-pg-build" +# compile pg_roaringbitmap extension +# +######################################################################################### +FROM build-deps AS pg-roaringbitmap-pg-build +COPY --from=pg-build /usr/local/pgsql/ /usr/local/pgsql/ + +ENV PATH "/usr/local/pgsql/bin/:$PATH" +RUN wget https://github.com/ChenHuajun/pg_roaringbitmap/archive/refs/tags/v0.5.4.tar.gz -O pg_roaringbitmap.tar.gz && \ + echo "b75201efcb1c2d1b014ec4ae6a22769cc7a224e6e406a587f5784a37b6b5a2aa pg_roaringbitmap.tar.gz" | sha256sum --check && \ + mkdir pg_roaringbitmap-src && cd pg_roaringbitmap-src && tar xvzf ../pg_roaringbitmap.tar.gz --strip-components=1 -C . && \ + make -j $(getconf _NPROCESSORS_ONLN) && \ + make -j $(getconf _NPROCESSORS_ONLN) install && \ + echo 'trusted = true' >> /usr/local/pgsql/share/extension/roaringbitmap.control + +######################################################################################### +# +# Layer "pg-anon-pg-build" +# compile anon extension +# +######################################################################################### +FROM build-deps AS pg-anon-pg-build +COPY --from=pg-build /usr/local/pgsql/ /usr/local/pgsql/ + +# Kaniko doesn't allow to do `${from#/usr/local/pgsql/}`, so we use `${from:17}` instead +ENV PATH "/usr/local/pgsql/bin/:$PATH" +RUN wget https://gitlab.com/dalibo/postgresql_anonymizer/-/archive/1.1.0/postgresql_anonymizer-1.1.0.tar.gz -O pg_anon.tar.gz && \ + echo "08b09d2ff9b962f96c60db7e6f8e79cf7253eb8772516998fc35ece08633d3ad pg_anon.tar.gz" | sha256sum --check && \ + mkdir pg_anon-src && cd pg_anon-src && tar xvzf ../pg_anon.tar.gz --strip-components=1 -C . && \ + find /usr/local/pgsql -type f | sort > /before.txt && \ + make -j $(getconf _NPROCESSORS_ONLN) install PG_CONFIG=/usr/local/pgsql/bin/pg_config && \ + echo 'trusted = true' >> /usr/local/pgsql/share/extension/anon.control && \ + find /usr/local/pgsql -type f | sort > /after.txt && \ + /bin/bash -c 'for from in $(comm -13 /before.txt /after.txt); do to=/extensions/anon/${from:17} && mkdir -p $(dirname ${to}) && cp -a ${from} ${to}; done' + ######################################################################################### # # Layer "rust extensions" @@ -517,6 +620,22 @@ RUN wget https://github.com/kelvich/pg_tiktoken/archive/801f84f08c6881c8aa30f405 cargo pgx install --release && \ echo "trusted = true" >> /usr/local/pgsql/share/extension/pg_tiktoken.control +######################################################################################### +# +# Layer "pg-pgx-ulid-build" +# Compile "pgx_ulid" extension +# +######################################################################################### + +FROM rust-extensions-build AS pg-pgx-ulid-build + +RUN wget https://github.com/pksunkara/pgx_ulid/archive/refs/tags/v0.1.0.tar.gz -O pgx_ulid.tar.gz && \ + echo "908b7358e6f846e87db508ae5349fb56a88ee6305519074b12f3d5b0ff09f791 pgx_ulid.tar.gz" | sha256sum --check && \ + mkdir pgx_ulid-src && cd pgx_ulid-src && tar xvzf ../pgx_ulid.tar.gz --strip-components=1 -C . && \ + sed -i 's/pgx = "=0.7.3"/pgx = { version = "0.7.3", features = [ "unsafe-postgres" ] }/g' Cargo.toml && \ + cargo pgx install --release && \ + echo "trusted = true" >> /usr/local/pgsql/share/extension/ulid.control + ######################################################################################### # # Layer "neon-pg-ext-build" @@ -524,6 +643,7 @@ RUN wget https://github.com/kelvich/pg_tiktoken/archive/801f84f08c6881c8aa30f405 # ######################################################################################### FROM build-deps AS neon-pg-ext-build +# Public extensions COPY --from=postgis-build /usr/local/pgsql/ /usr/local/pgsql/ COPY --from=postgis-build /sfcgal/* / COPY --from=plv8-build /usr/local/pgsql/ /usr/local/pgsql/ @@ -547,6 +667,10 @@ COPY --from=timescaledb-pg-build /usr/local/pgsql/ /usr/local/pgsql/ COPY --from=pg-hint-plan-pg-build /usr/local/pgsql/ /usr/local/pgsql/ COPY --from=kq-imcx-pg-build /usr/local/pgsql/ /usr/local/pgsql/ COPY --from=pg-cron-pg-build /usr/local/pgsql/ /usr/local/pgsql/ +COPY --from=pg-pgx-ulid-build /usr/local/pgsql/ /usr/local/pgsql/ +COPY --from=rdkit-pg-build /usr/local/pgsql/ /usr/local/pgsql/ +COPY --from=pg-uuidv7-pg-build /usr/local/pgsql/ /usr/local/pgsql/ +COPY --from=pg-roaringbitmap-pg-build /usr/local/pgsql/ /usr/local/pgsql/ COPY pgxn/ pgxn/ RUN make -j $(getconf _NPROCESSORS_ONLN) \ @@ -556,6 +680,10 @@ RUN make -j $(getconf _NPROCESSORS_ONLN) \ make -j $(getconf _NPROCESSORS_ONLN) \ PG_CONFIG=/usr/local/pgsql/bin/pg_config \ -C pgxn/neon_utils \ + -s install && \ + make -j $(getconf _NPROCESSORS_ONLN) \ + PG_CONFIG=/usr/local/pgsql/bin/pg_config \ + -C pgxn/hnsw \ -s install ######################################################################################### @@ -564,6 +692,9 @@ RUN make -j $(getconf _NPROCESSORS_ONLN) \ # ######################################################################################### FROM $REPOSITORY/$IMAGE:$TAG AS compute-tools +ARG BUILD_TAG +ENV BUILD_TAG=$BUILD_TAG + USER nonroot # Copy entire project to get Cargo.* files with proper dependencies for the whole project COPY --chown=nonroot . . @@ -588,6 +719,22 @@ RUN rm -r /usr/local/pgsql/include # if they were to be used by other libraries. RUN rm /usr/local/pgsql/lib/lib*.a +######################################################################################### +# +# Extenstion only +# +######################################################################################### +FROM scratch AS postgres-extensions +# After the transition this layer will include all extensitons. +# As for now, it's only for new custom ones +# +# # Default extensions +# COPY --from=postgres-cleanup-layer /usr/local/pgsql/share/extension /usr/local/pgsql/share/extension +# COPY --from=postgres-cleanup-layer /usr/local/pgsql/lib /usr/local/pgsql/lib +# Custom extensions +COPY --from=pg-anon-pg-build /extensions/anon/lib/ /extensions/anon/lib +COPY --from=pg-anon-pg-build /extensions/anon/share/extension /extensions/anon/share/extension + ######################################################################################### # # Final layer @@ -616,14 +763,19 @@ COPY --from=compute-tools --chown=postgres /home/nonroot/target/release-line-deb # libgeos, libgdal, libsfcgal1, libproj and libprotobuf-c1 for PostGIS # libxml2, libxslt1.1 for xml2 # libzstd1 for zstd +# libboost*, libfreetype6, and zlib1g for rdkit RUN apt update && \ apt install --no-install-recommends -y \ gdb \ - locales \ libicu67 \ liblz4-1 \ libreadline8 \ + libboost-iostreams1.74.0 \ + libboost-regex1.74.0 \ + libboost-serialization1.74.0 \ + libboost-system1.74.0 \ libossp-uuid16 \ + libfreetype6 \ libgeos-c1v5 \ libgdal28 \ libproj19 \ @@ -633,7 +785,9 @@ RUN apt update && \ libxslt1.1 \ libzstd1 \ libcurl4-openssl-dev \ - procps && \ + locales \ + procps \ + zlib1g && \ rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* && \ localedef -i en_US -c -f UTF-8 -A /usr/share/locale/locale.alias en_US.UTF-8 diff --git a/Dockerfile.compute-tools b/Dockerfile.compute-tools index e86fb40ca4..3066e3f7ca 100644 --- a/Dockerfile.compute-tools +++ b/Dockerfile.compute-tools @@ -3,6 +3,7 @@ ARG REPOSITORY=neondatabase ARG IMAGE=rust ARG TAG=pinned +ARG BUILD_TAG FROM $REPOSITORY/$IMAGE:$TAG AS rust-build WORKDIR /home/nonroot @@ -16,6 +17,8 @@ ENV CACHEPOT_S3_KEY_PREFIX=cachepot ARG CACHEPOT_BUCKET=neon-github-dev #ARG AWS_ACCESS_KEY_ID #ARG AWS_SECRET_ACCESS_KEY +ARG BUILD_TAG +ENV BUILD_TAG=$BUILD_TAG COPY . . diff --git a/Makefile b/Makefile index 9d78c5d0fc..ae979b8b4c 100644 --- a/Makefile +++ b/Makefile @@ -138,6 +138,11 @@ neon-pg-ext-%: postgres-% $(MAKE) PG_CONFIG=$(POSTGRES_INSTALL_DIR)/$*/bin/pg_config CFLAGS='$(PG_CFLAGS) $(COPT)' \ -C $(POSTGRES_INSTALL_DIR)/build/neon-utils-$* \ -f $(ROOT_PROJECT_DIR)/pgxn/neon_utils/Makefile install + +@echo "Compiling hnsw $*" + mkdir -p $(POSTGRES_INSTALL_DIR)/build/hnsw-$* + $(MAKE) PG_CONFIG=$(POSTGRES_INSTALL_DIR)/$*/bin/pg_config CFLAGS='$(PG_CFLAGS) $(COPT)' \ + -C $(POSTGRES_INSTALL_DIR)/build/hnsw-$* \ + -f $(ROOT_PROJECT_DIR)/pgxn/hnsw/Makefile install .PHONY: neon-pg-ext-clean-% neon-pg-ext-clean-%: @@ -153,6 +158,9 @@ neon-pg-ext-clean-%: $(MAKE) PG_CONFIG=$(POSTGRES_INSTALL_DIR)/$*/bin/pg_config \ -C $(POSTGRES_INSTALL_DIR)/build/neon-utils-$* \ -f $(ROOT_PROJECT_DIR)/pgxn/neon_utils/Makefile clean + $(MAKE) PG_CONFIG=$(POSTGRES_INSTALL_DIR)/$*/bin/pg_config \ + -C $(POSTGRES_INSTALL_DIR)/build/hnsw-$* \ + -f $(ROOT_PROJECT_DIR)/pgxn/hnsw/Makefile clean .PHONY: neon-pg-ext neon-pg-ext: \ diff --git a/README.md b/README.md index 8e6f2cda81..efa714e5be 100644 --- a/README.md +++ b/README.md @@ -28,18 +28,19 @@ See developer documentation in [SUMMARY.md](/docs/SUMMARY.md) for more informati * On Ubuntu or Debian, this set of packages should be sufficient to build the code: ```bash apt install build-essential libtool libreadline-dev zlib1g-dev flex bison libseccomp-dev \ -libssl-dev clang pkg-config libpq-dev cmake postgresql-client protobuf-compiler +libssl-dev clang pkg-config libpq-dev cmake postgresql-client protobuf-compiler \ +libcurl4-openssl-dev ``` * On Fedora, these packages are needed: ```bash dnf install flex bison readline-devel zlib-devel openssl-devel \ libseccomp-devel perl clang cmake postgresql postgresql-contrib protobuf-compiler \ - protobuf-devel + protobuf-devel libcurl-devel ``` * On Arch based systems, these packages are needed: ```bash pacman -S base-devel readline zlib libseccomp openssl clang \ -postgresql-libs cmake postgresql protobuf +postgresql-libs cmake postgresql protobuf curl ``` Building Neon requires 3.15+ version of `protoc` (protobuf-compiler). If your distribution provides an older version, you can install a newer version from [here](https://github.com/protocolbuffers/protobuf/releases). diff --git a/compute_tools/src/bin/compute_ctl.rs b/compute_tools/src/bin/compute_ctl.rs index 2f515c9bf1..68f6bf3844 100644 --- a/compute_tools/src/bin/compute_ctl.rs +++ b/compute_tools/src/bin/compute_ctl.rs @@ -54,11 +54,20 @@ use compute_tools::monitor::launch_monitor; use compute_tools::params::*; use compute_tools::spec::*; +const BUILD_TAG_DEFAULT: &str = "local"; + fn main() -> Result<()> { init_tracing_and_logging(DEFAULT_LOG_LEVEL)?; + let build_tag = option_env!("BUILD_TAG").unwrap_or(BUILD_TAG_DEFAULT); + + info!("build_tag: {build_tag}"); + let matches = cli().get_matches(); + let http_port = *matches + .get_one::("http-port") + .expect("http-port is required"); let pgdata = matches .get_one::("pgdata") .expect("PGDATA path is required"); @@ -178,7 +187,8 @@ fn main() -> Result<()> { // Launch http service first, so we were able to serve control-plane // requests, while configuration is still in progress. - let _http_handle = launch_http_server(&compute).expect("cannot launch http endpoint thread"); + let _http_handle = + launch_http_server(http_port, &compute).expect("cannot launch http endpoint thread"); if !spec_set { // No spec provided, hang waiting for it. @@ -246,6 +256,16 @@ fn main() -> Result<()> { exit_code = ecode.code() } + // Maybe sync safekeepers again, to speed up next startup + let compute_state = compute.state.lock().unwrap().clone(); + let pspec = compute_state.pspec.as_ref().expect("spec must be set"); + if matches!(pspec.spec.mode, compute_api::spec::ComputeMode::Primary) { + info!("syncing safekeepers on shutdown"); + let storage_auth_token = pspec.storage_auth_token.clone(); + let lsn = compute.sync_safekeepers(storage_auth_token)?; + info!("synced safekeepers at lsn {lsn}"); + } + if let Err(err) = compute.check_for_core_dumps() { error!("error while checking for core dumps: {err:?}"); } @@ -286,6 +306,14 @@ fn cli() -> clap::Command { let version = option_env!("CARGO_PKG_VERSION").unwrap_or("unknown"); clap::Command::new("compute_ctl") .version(version) + .arg( + Arg::new("http-port") + .long("http-port") + .value_name("HTTP_PORT") + .default_value("3080") + .value_parser(clap::value_parser!(u16)) + .required(false), + ) .arg( Arg::new("connstr") .short('C') diff --git a/compute_tools/src/compute.rs b/compute_tools/src/compute.rs index a7746629a8..70d83a7b47 100644 --- a/compute_tools/src/compute.rs +++ b/compute_tools/src/compute.rs @@ -1,19 +1,3 @@ -// -// XXX: This starts to be scarry similar to the `PostgresNode` from `control_plane`, -// but there are several things that makes `PostgresNode` usage inconvenient in the -// cloud: -// - it inherits from `LocalEnv`, which contains **all-all** the information about -// a complete service running -// - it uses `PageServerNode` with information about http endpoint, which we do not -// need in the cloud again -// - many tiny pieces like, for example, we do not use `pg_ctl` in the cloud -// -// Thus, to use `PostgresNode` in the cloud, we need to 'mock' a bunch of required -// attributes (not required for the cloud). Yet, it is still tempting to unify these -// `PostgresNode` and `ComputeNode` and use one in both places. -// -// TODO: stabilize `ComputeNode` and think about using it in the `control_plane`. -// use std::fs; use std::os::unix::fs::PermissionsExt; use std::path::Path; @@ -106,26 +90,38 @@ pub struct ParsedSpec { impl TryFrom for ParsedSpec { type Error = String; fn try_from(spec: ComputeSpec) -> Result { + // Extract the options from the spec file that are needed to connect to + // the storage system. + // + // For backwards-compatibility, the top-level fields in the spec file + // may be empty. In that case, we need to dig them from the GUCs in the + // cluster.settings field. let pageserver_connstr = spec - .cluster - .settings - .find("neon.pageserver_connstring") + .pageserver_connstring + .clone() + .or_else(|| spec.cluster.settings.find("neon.pageserver_connstring")) .ok_or("pageserver connstr should be provided")?; let storage_auth_token = spec.storage_auth_token.clone(); - let tenant_id: TenantId = spec - .cluster - .settings - .find("neon.tenant_id") - .ok_or("tenant id should be provided") - .map(|s| TenantId::from_str(&s))? - .or(Err("invalid tenant id"))?; - let timeline_id: TimelineId = spec - .cluster - .settings - .find("neon.timeline_id") - .ok_or("timeline id should be provided") - .map(|s| TimelineId::from_str(&s))? - .or(Err("invalid timeline id"))?; + let tenant_id: TenantId = if let Some(tenant_id) = spec.tenant_id { + tenant_id + } else { + spec.cluster + .settings + .find("neon.tenant_id") + .ok_or("tenant id should be provided") + .map(|s| TenantId::from_str(&s))? + .or(Err("invalid tenant id"))? + }; + let timeline_id: TimelineId = if let Some(timeline_id) = spec.timeline_id { + timeline_id + } else { + spec.cluster + .settings + .find("neon.timeline_id") + .ok_or("timeline id should be provided") + .map(|s| TimelineId::from_str(&s))? + .or(Err("invalid timeline id"))? + }; Ok(ParsedSpec { spec, @@ -137,6 +133,84 @@ impl TryFrom for ParsedSpec { } } +/// Create special neon_superuser role, that's a slightly nerfed version of a real superuser +/// that we give to customers +fn create_neon_superuser(spec: &ComputeSpec, client: &mut Client) -> Result<()> { + let roles = spec + .cluster + .roles + .iter() + .map(|r| format!("'{}'", escape_literal(&r.name))) + .collect::>(); + + let dbs = spec + .cluster + .databases + .iter() + .map(|db| format!("'{}'", escape_literal(&db.name))) + .collect::>(); + + let roles_decl = if roles.is_empty() { + String::from("roles text[] := NULL;") + } else { + format!( + r#" + roles text[] := ARRAY(SELECT rolname + FROM pg_catalog.pg_roles + WHERE rolname IN ({}));"#, + roles.join(", ") + ) + }; + + let database_decl = if dbs.is_empty() { + String::from("dbs text[] := NULL;") + } else { + format!( + r#" + dbs text[] := ARRAY(SELECT datname + FROM pg_catalog.pg_database + WHERE datname IN ({}));"#, + dbs.join(", ") + ) + }; + + // ALL PRIVILEGES grants CREATE, CONNECT, and TEMPORARY on all databases + // (see https://www.postgresql.org/docs/current/ddl-priv.html) + let query = format!( + r#" + DO $$ + DECLARE + r text; + {} + {} + BEGIN + IF NOT EXISTS ( + SELECT FROM pg_catalog.pg_roles WHERE rolname = 'neon_superuser') + THEN + CREATE ROLE neon_superuser CREATEDB CREATEROLE NOLOGIN IN ROLE pg_read_all_data, pg_write_all_data; + IF array_length(roles, 1) IS NOT NULL THEN + EXECUTE format('GRANT neon_superuser TO %s', + array_to_string(ARRAY(SELECT quote_ident(x) FROM unnest(roles) as x), ', ')); + FOREACH r IN ARRAY roles LOOP + EXECUTE format('ALTER ROLE %s CREATEROLE CREATEDB', quote_ident(r)); + END LOOP; + END IF; + IF array_length(dbs, 1) IS NOT NULL THEN + EXECUTE format('GRANT ALL PRIVILEGES ON DATABASE %s TO neon_superuser', + array_to_string(ARRAY(SELECT quote_ident(x) FROM unnest(dbs) as x), ', ')); + END IF; + END IF; + END + $$;"#, + roles_decl, database_decl, + ); + info!("Neon superuser created:\n{}", &query); + client + .simple_query(&query) + .map_err(|e| anyhow::anyhow!(e).context(query))?; + Ok(()) +} + impl ComputeNode { pub fn set_status(&self, status: ComputeStatus) { let mut state = self.state.lock().unwrap(); @@ -161,7 +235,7 @@ impl ComputeNode { // Get basebackup from the libpq connection to pageserver using `connstr` and // unarchive it to `pgdata` directory overriding all its previous content. - #[instrument(skip(self, compute_state))] + #[instrument(skip_all, fields(%lsn))] fn get_basebackup(&self, compute_state: &ComputeState, lsn: Lsn) -> Result<()> { let spec = compute_state.pspec.as_ref().expect("spec must be set"); let start_time = Utc::now(); @@ -203,8 +277,8 @@ impl ComputeNode { // Run `postgres` in a special mode with `--sync-safekeepers` argument // and return the reported LSN back to the caller. - #[instrument(skip(self, storage_auth_token))] - fn sync_safekeepers(&self, storage_auth_token: Option) -> Result { + #[instrument(skip_all)] + pub fn sync_safekeepers(&self, storage_auth_token: Option) -> Result { let start_time = Utc::now(); let sync_handle = Command::new(&self.pgbin) @@ -248,7 +322,7 @@ impl ComputeNode { /// Do all the preparations like PGDATA directory creation, configuration, /// safekeepers sync, basebackup, etc. - #[instrument(skip(self, compute_state))] + #[instrument(skip_all)] pub fn prepare_pgdata(&self, compute_state: &ComputeState) -> Result<()> { let pspec = compute_state.pspec.as_ref().expect("spec must be set"); let spec = &pspec.spec; @@ -295,8 +369,8 @@ impl ComputeNode { update_pg_hba(pgdata_path)?; match spec.mode { - ComputeMode::Primary | ComputeMode::Static(..) => {} - ComputeMode::Replica => { + ComputeMode::Primary => {} + ComputeMode::Replica | ComputeMode::Static(..) => { add_standby_signal(pgdata_path)?; } } @@ -306,7 +380,7 @@ impl ComputeNode { /// Start Postgres as a child process and manage DBs/roles. /// After that this will hang waiting on the postmaster process to exit. - #[instrument(skip(self))] + #[instrument(skip_all)] pub fn start_postgres( &self, storage_auth_token: Option, @@ -330,7 +404,7 @@ impl ComputeNode { } /// Do initial configuration of the already started Postgres. - #[instrument(skip(self, compute_state))] + #[instrument(skip_all)] pub fn apply_config(&self, compute_state: &ComputeState) -> Result<()> { // If connection fails, // it may be the old node with `zenith_admin` superuser. @@ -351,6 +425,8 @@ impl ComputeNode { .map_err(|_| anyhow::anyhow!("invalid connstr"))?; let mut client = Client::connect(zenith_admin_connstr.as_str(), NoTls)?; + // Disable forwarding so that users don't get a cloud_admin role + client.simple_query("SET neon.forward_ddl = false")?; client.simple_query("CREATE USER cloud_admin WITH SUPERUSER")?; client.simple_query("GRANT zenith_admin TO cloud_admin")?; drop(client); @@ -361,31 +437,28 @@ impl ComputeNode { Ok(client) => client, }; - // Proceed with post-startup configuration. Note, that order of operations is important. // Disable DDL forwarding because control plane already knows about these roles/databases. client.simple_query("SET neon.forward_ddl = false")?; + + // Proceed with post-startup configuration. Note, that order of operations is important. let spec = &compute_state.pspec.as_ref().expect("spec must be set").spec; + create_neon_superuser(spec, &mut client)?; handle_roles(spec, &mut client)?; handle_databases(spec, &mut client)?; handle_role_deletions(spec, self.connstr.as_str(), &mut client)?; - handle_grants(spec, self.connstr.as_str(), &mut client)?; + handle_grants(spec, self.connstr.as_str())?; handle_extensions(spec, &mut client)?; // 'Close' connection drop(client); - info!( - "finished configuration of compute for project {}", - spec.cluster.cluster_id - ); - Ok(()) } // We could've wrapped this around `pg_ctl reload`, but right now we don't use // `pg_ctl` for start / stop, so this just seems much easier to do as we already // have opened connection to Postgres and superuser access. - #[instrument(skip(self, client))] + #[instrument(skip_all)] fn pg_reload_conf(&self, client: &mut Client) -> Result<()> { client.simple_query("SELECT pg_reload_conf()")?; Ok(()) @@ -393,7 +466,7 @@ impl ComputeNode { /// Similar to `apply_config()`, but does a bit different sequence of operations, /// as it's used to reconfigure a previously started and configured Postgres node. - #[instrument(skip(self))] + #[instrument(skip_all)] pub fn reconfigure(&self) -> Result<()> { let spec = self.state.lock().unwrap().pspec.clone().unwrap().spec; @@ -411,7 +484,7 @@ impl ComputeNode { handle_roles(&spec, &mut client)?; handle_databases(&spec, &mut client)?; handle_role_deletions(&spec, self.connstr.as_str(), &mut client)?; - handle_grants(&spec, self.connstr.as_str(), &mut client)?; + handle_grants(&spec, self.connstr.as_str())?; handle_extensions(&spec, &mut client)?; } @@ -428,25 +501,25 @@ impl ComputeNode { Ok(()) } - #[instrument(skip(self))] + #[instrument(skip_all)] pub fn start_compute(&self) -> Result { let compute_state = self.state.lock().unwrap().clone(); - let spec = compute_state.pspec.as_ref().expect("spec must be set"); + let pspec = compute_state.pspec.as_ref().expect("spec must be set"); info!( "starting compute for project {}, operation {}, tenant {}, timeline {}", - spec.spec.cluster.cluster_id, - spec.spec.operation_uuid.as_deref().unwrap_or("None"), - spec.tenant_id, - spec.timeline_id, + pspec.spec.cluster.cluster_id.as_deref().unwrap_or("None"), + pspec.spec.operation_uuid.as_deref().unwrap_or("None"), + pspec.tenant_id, + pspec.timeline_id, ); self.prepare_pgdata(&compute_state)?; let start_time = Utc::now(); - let pg = self.start_postgres(spec.storage_auth_token.clone())?; + let pg = self.start_postgres(pspec.storage_auth_token.clone())?; - if spec.spec.mode == ComputeMode::Primary { + if pspec.spec.mode == ComputeMode::Primary && !pspec.spec.skip_pg_catalog_updates { self.apply_config(&compute_state)?; } @@ -466,6 +539,11 @@ impl ComputeNode { } self.set_status(ComputeStatus::Running); + info!( + "finished configuration of compute for project {}", + pspec.spec.cluster.cluster_id.as_deref().unwrap_or("None") + ); + Ok(pg) } diff --git a/compute_tools/src/config.rs b/compute_tools/src/config.rs index 1168f3876a..99346433d0 100644 --- a/compute_tools/src/config.rs +++ b/compute_tools/src/config.rs @@ -5,6 +5,7 @@ use std::path::Path; use anyhow::Result; +use crate::pg_helpers::escape_conf_value; use crate::pg_helpers::PgOptionsSerialize; use compute_api::spec::{ComputeMode, ComputeSpec}; @@ -36,10 +37,44 @@ pub fn write_postgres_conf(path: &Path, spec: &ComputeSpec) -> Result<()> { // File::create() destroys the file content if it exists. let mut file = File::create(path)?; - writeln!(file, "# Managed by compute_ctl: begin")?; + // Write the postgresql.conf content from the spec file as is. + if let Some(conf) = &spec.cluster.postgresql_conf { + writeln!(file, "{}", conf)?; + } write!(file, "{}", &spec.cluster.settings.as_pg_settings())?; + // Add options for connecting to storage + writeln!(file, "# Neon storage settings")?; + if let Some(s) = &spec.pageserver_connstring { + writeln!( + file, + "neon.pageserver_connstring='{}'", + escape_conf_value(s) + )?; + } + if !spec.safekeeper_connstrings.is_empty() { + writeln!( + file, + "neon.safekeepers='{}'", + escape_conf_value(&spec.safekeeper_connstrings.join(",")) + )?; + } + if let Some(s) = &spec.tenant_id { + writeln!( + file, + "neon.tenant_id='{}'", + escape_conf_value(&s.to_string()) + )?; + } + if let Some(s) = &spec.timeline_id { + writeln!( + file, + "neon.timeline_id='{}'", + escape_conf_value(&s.to_string()) + )?; + } + match spec.mode { ComputeMode::Primary => {} ComputeMode::Static(lsn) => { @@ -53,7 +88,12 @@ pub fn write_postgres_conf(path: &Path, spec: &ComputeSpec) -> Result<()> { } } - writeln!(file, "# Managed by compute_ctl: end")?; + // If there are any extra options in the 'settings' field, append those + if spec.cluster.settings.is_some() { + writeln!(file, "# Managed by compute_ctl: begin")?; + write!(file, "{}", spec.cluster.settings.as_pg_settings())?; + writeln!(file, "# Managed by compute_ctl: end")?; + } Ok(()) } diff --git a/compute_tools/src/configurator.rs b/compute_tools/src/configurator.rs index a07fd0b8cd..13550e0176 100644 --- a/compute_tools/src/configurator.rs +++ b/compute_tools/src/configurator.rs @@ -8,7 +8,7 @@ use compute_api::responses::ComputeStatus; use crate::compute::ComputeNode; -#[instrument(skip(compute))] +#[instrument(skip_all)] fn configurator_main_loop(compute: &Arc) { info!("waiting for reconfiguration requests"); loop { diff --git a/compute_tools/src/http/api.rs b/compute_tools/src/http/api.rs index 4468f6f5e4..afd9c2fb54 100644 --- a/compute_tools/src/http/api.rs +++ b/compute_tools/src/http/api.rs @@ -220,8 +220,8 @@ fn render_json_error(e: &str, status: StatusCode) -> Response { // Main Hyper HTTP server function that runs it and blocks waiting on it forever. #[tokio::main] -async fn serve(state: Arc) { - let addr = SocketAddr::from(([0, 0, 0, 0], 3080)); +async fn serve(port: u16, state: Arc) { + let addr = SocketAddr::from(([0, 0, 0, 0], port)); let make_service = make_service_fn(move |_conn| { let state = state.clone(); @@ -256,10 +256,10 @@ async fn serve(state: Arc) { } /// Launch a separate Hyper HTTP API server thread and return its `JoinHandle`. -pub fn launch_http_server(state: &Arc) -> Result> { +pub fn launch_http_server(port: u16, state: &Arc) -> Result> { let state = Arc::clone(state); Ok(thread::Builder::new() .name("http-endpoint".into()) - .spawn(move || serve(state))?) + .spawn(move || serve(port, state))?) } diff --git a/compute_tools/src/logger.rs b/compute_tools/src/logger.rs index 1b5cf647b0..f6fc882968 100644 --- a/compute_tools/src/logger.rs +++ b/compute_tools/src/logger.rs @@ -33,5 +33,7 @@ pub fn init_tracing_and_logging(default_log_level: &str) -> anyhow::Result<()> { .init(); tracing::info!("logging and tracing started"); + utils::logging::replace_panic_hook_with_tracing_panic_hook().forget(); + Ok(()) } diff --git a/compute_tools/src/pg_helpers.rs b/compute_tools/src/pg_helpers.rs index ed00485d5a..6a78bffd1b 100644 --- a/compute_tools/src/pg_helpers.rs +++ b/compute_tools/src/pg_helpers.rs @@ -17,13 +17,13 @@ use compute_api::spec::{Database, GenericOption, GenericOptions, PgIdent, Role}; const POSTGRES_WAIT_TIMEOUT: Duration = Duration::from_millis(60 * 1000); // milliseconds /// Escape a string for including it in a SQL literal -fn escape_literal(s: &str) -> String { +pub fn escape_literal(s: &str) -> String { s.replace('\'', "''").replace('\\', "\\\\") } /// Escape a string so that it can be used in postgresql.conf. /// Same as escape_literal, currently. -fn escape_conf_value(s: &str) -> String { +pub fn escape_conf_value(s: &str) -> String { s.replace('\'', "''").replace('\\', "\\\\") } @@ -215,7 +215,7 @@ pub fn get_existing_dbs(client: &mut Client) -> Result> { /// Wait for Postgres to become ready to accept connections. It's ready to /// accept connections when the state-field in `pgdata/postmaster.pid` says /// 'ready'. -#[instrument(skip(pg))] +#[instrument(skip_all, fields(pgdata = %pgdata.display()))] pub fn wait_for_postgres(pg: &mut Child, pgdata: &Path) -> Result<()> { let pid_path = pgdata.join("postmaster.pid"); diff --git a/compute_tools/src/spec.rs b/compute_tools/src/spec.rs index a2a19ae0da..520696da00 100644 --- a/compute_tools/src/spec.rs +++ b/compute_tools/src/spec.rs @@ -269,17 +269,13 @@ pub fn handle_roles(spec: &ComputeSpec, client: &mut Client) -> Result<()> { xact.execute(query.as_str(), &[])?; } RoleAction::Create => { - let mut query: String = format!("CREATE ROLE {} ", name.pg_quote()); + let mut query: String = format!( + "CREATE ROLE {} CREATEROLE CREATEDB IN ROLE neon_superuser", + name.pg_quote() + ); info!("role create query: '{}'", &query); query.push_str(&role.to_pg_options()); xact.execute(query.as_str(), &[])?; - - let grant_query = format!( - "GRANT pg_read_all_data, pg_write_all_data TO {}", - name.pg_quote() - ); - xact.execute(grant_query.as_str(), &[])?; - info!("role grant query: '{}'", &grant_query); } } @@ -476,6 +472,11 @@ pub fn handle_databases(spec: &ComputeSpec, client: &mut Client) -> Result<()> { query.push_str(&db.to_pg_options()); let _guard = info_span!("executing", query).entered(); client.execute(query.as_str(), &[])?; + let grant_query: String = format!( + "GRANT ALL PRIVILEGES ON DATABASE {} TO neon_superuser", + name.pg_quote() + ); + client.execute(grant_query.as_str(), &[])?; } }; @@ -495,35 +496,9 @@ pub fn handle_databases(spec: &ComputeSpec, client: &mut Client) -> Result<()> { /// Grant CREATE ON DATABASE to the database owner and do some other alters and grants /// to allow users creating trusted extensions and re-creating `public` schema, for example. #[instrument(skip_all)] -pub fn handle_grants(spec: &ComputeSpec, connstr: &str, client: &mut Client) -> Result<()> { +pub fn handle_grants(spec: &ComputeSpec, connstr: &str) -> Result<()> { info!("cluster spec grants:"); - // We now have a separate `web_access` role to connect to the database - // via the web interface and proxy link auth. And also we grant a - // read / write all data privilege to every role. So also grant - // create to everyone. - // XXX: later we should stop messing with Postgres ACL in such horrible - // ways. - let roles = spec - .cluster - .roles - .iter() - .map(|r| r.name.pg_quote()) - .collect::>(); - - for db in &spec.cluster.databases { - let dbname = &db.name; - - let query: String = format!( - "GRANT CREATE ON DATABASE {} TO {}", - dbname.pg_quote(), - roles.join(", ") - ); - info!("grant query {}", &query); - - client.execute(query.as_str(), &[])?; - } - // Do some per-database access adjustments. We'd better do this at db creation time, // but CREATE DATABASE isn't transactional. So we cannot create db + do some grants // atomically. diff --git a/control_plane/src/background_process.rs b/control_plane/src/background_process.rs index 1f3f8f45ea..00af1a1d53 100644 --- a/control_plane/src/background_process.rs +++ b/control_plane/src/background_process.rs @@ -180,6 +180,11 @@ pub fn stop_process(immediate: bool, process_name: &str, pid_file: &Path) -> any } // Wait until process is gone + wait_until_stopped(process_name, pid)?; + Ok(()) +} + +pub fn wait_until_stopped(process_name: &str, pid: Pid) -> anyhow::Result<()> { for retries in 0..RETRIES { match process_has_stopped(pid) { Ok(true) => { diff --git a/control_plane/src/bin/neon_local.rs b/control_plane/src/bin/neon_local.rs index 39551642c0..8995a18564 100644 --- a/control_plane/src/bin/neon_local.rs +++ b/control_plane/src/bin/neon_local.rs @@ -308,7 +308,8 @@ fn handle_init(init_match: &ArgMatches) -> anyhow::Result { let mut env = LocalEnv::parse_config(&toml_file).context("Failed to create neon configuration")?; - env.init(pg_version) + let force = init_match.get_flag("force"); + env.init(pg_version, force) .context("Failed to initialize neon repository")?; // Initialize pageserver, create initial tenant and timeline. @@ -476,10 +477,11 @@ fn handle_timeline(timeline_match: &ArgMatches, env: &mut local_env::LocalEnv) - println!("Creating endpoint for imported timeline ..."); cplane.new_endpoint( - tenant_id, name, + tenant_id, timeline_id, None, + None, pg_version, ComputeMode::Primary, )?; @@ -591,7 +593,7 @@ fn handle_endpoint(ep_match: &ArgMatches, env: &local_env::LocalEnv) -> Result<( table.add_row([ endpoint_id.as_str(), - &endpoint.address.to_string(), + &endpoint.pg_address.to_string(), &endpoint.timeline_id.to_string(), branch_name, lsn_str.as_str(), @@ -620,8 +622,8 @@ fn handle_endpoint(ep_match: &ArgMatches, env: &local_env::LocalEnv) -> Result<( .get_branch_timeline_id(branch_name, tenant_id) .ok_or_else(|| anyhow!("Found no timeline id for branch name '{branch_name}'"))?; - let port: Option = sub_args.get_one::("port").copied(); - + let pg_port: Option = sub_args.get_one::("pg-port").copied(); + let http_port: Option = sub_args.get_one::("http-port").copied(); let pg_version = sub_args .get_one::("pg-version") .copied() @@ -639,14 +641,38 @@ fn handle_endpoint(ep_match: &ArgMatches, env: &local_env::LocalEnv) -> Result<( (Some(_), true) => anyhow::bail!("cannot specify both lsn and hot-standby"), }; - cplane.new_endpoint(tenant_id, &endpoint_id, timeline_id, port, pg_version, mode)?; + cplane.new_endpoint( + &endpoint_id, + tenant_id, + timeline_id, + pg_port, + http_port, + pg_version, + mode, + )?; } "start" => { - let port: Option = sub_args.get_one::("port").copied(); + let pg_port: Option = sub_args.get_one::("pg-port").copied(); + let http_port: Option = sub_args.get_one::("http-port").copied(); let endpoint_id = sub_args .get_one::("endpoint_id") .ok_or_else(|| anyhow!("No endpoint ID was provided to start"))?; + // If --safekeepers argument is given, use only the listed safekeeper nodes. + let safekeepers = + if let Some(safekeepers_str) = sub_args.get_one::("safekeepers") { + let mut safekeepers: Vec = Vec::new(); + for sk_id in safekeepers_str.split(',').map(str::trim) { + let sk_id = NodeId(u64::from_str(sk_id).map_err(|_| { + anyhow!("invalid node ID \"{sk_id}\" in --safekeepers list") + })?); + safekeepers.push(sk_id); + } + safekeepers + } else { + env.safekeepers.iter().map(|sk| sk.id).collect() + }; + let endpoint = cplane.endpoints.get(endpoint_id.as_str()); let auth_token = if matches!(env.pageserver.pg_auth_type, AuthType::NeonJWT) { @@ -673,7 +699,7 @@ fn handle_endpoint(ep_match: &ArgMatches, env: &local_env::LocalEnv) -> Result<( _ => {} } println!("Starting existing endpoint {endpoint_id}..."); - endpoint.start(&auth_token)?; + endpoint.start(&auth_token, safekeepers)?; } else { let branch_name = sub_args .get_one::("branch-name") @@ -709,14 +735,15 @@ fn handle_endpoint(ep_match: &ArgMatches, env: &local_env::LocalEnv) -> Result<( println!("Starting new endpoint {endpoint_id} (PostgreSQL v{pg_version}) on timeline {timeline_id} ..."); let ep = cplane.new_endpoint( - tenant_id, endpoint_id, + tenant_id, timeline_id, - port, + pg_port, + http_port, pg_version, mode, )?; - ep.start(&auth_token)?; + ep.start(&auth_token, safekeepers)?; } } "stop" => { @@ -944,11 +971,22 @@ fn cli() -> Command { .value_parser(value_parser!(u32)) .default_value(DEFAULT_PG_VERSION); - let port_arg = Arg::new("port") - .long("port") + let pg_port_arg = Arg::new("pg-port") + .long("pg-port") .required(false) .value_parser(value_parser!(u16)) - .value_name("port"); + .value_name("pg-port"); + + let http_port_arg = Arg::new("http-port") + .long("http-port") + .required(false) + .value_parser(value_parser!(u16)) + .value_name("http-port"); + + let safekeepers_arg = Arg::new("safekeepers") + .long("safekeepers") + .required(false) + .value_name("safekeepers"); let stop_mode_arg = Arg::new("stop-mode") .short('m') @@ -976,6 +1014,13 @@ fn cli() -> Command { .help("If set, the node will be a hot replica on the specified timeline") .required(false); + let force_arg = Arg::new("force") + .value_parser(value_parser!(bool)) + .long("force") + .action(ArgAction::SetTrue) + .help("Force initialization even if the repository is not empty") + .required(false); + Command::new("Neon CLI") .arg_required_else_help(true) .version(GIT_VERSION) @@ -991,6 +1036,7 @@ fn cli() -> Command { .value_name("config"), ) .arg(pg_version_arg.clone()) + .arg(force_arg) ) .subcommand( Command::new("timeline") @@ -1093,7 +1139,8 @@ fn cli() -> Command { .arg(branch_name_arg.clone()) .arg(tenant_id_arg.clone()) .arg(lsn_arg.clone()) - .arg(port_arg.clone()) + .arg(pg_port_arg.clone()) + .arg(http_port_arg.clone()) .arg( Arg::new("config-only") .help("Don't do basebackup, create endpoint directory with only config files") @@ -1109,9 +1156,11 @@ fn cli() -> Command { .arg(branch_name_arg) .arg(timeline_id_arg) .arg(lsn_arg) - .arg(port_arg) + .arg(pg_port_arg) + .arg(http_port_arg) .arg(pg_version_arg) .arg(hot_standby_arg) + .arg(safekeepers_arg) ) .subcommand( Command::new("stop") diff --git a/control_plane/src/broker.rs b/control_plane/src/broker.rs index 6c0604a076..ad19dfa204 100644 --- a/control_plane/src/broker.rs +++ b/control_plane/src/broker.rs @@ -1,3 +1,9 @@ +//! Code to manage the storage broker +//! +//! In the local test environment, the data for each safekeeper is stored in +//! +//! .neon/safekeepers/ +//! use anyhow::Context; use std::path::PathBuf; diff --git a/control_plane/src/endpoint.rs b/control_plane/src/endpoint.rs index cc5a7a4168..ab921d096f 100644 --- a/control_plane/src/endpoint.rs +++ b/control_plane/src/endpoint.rs @@ -1,41 +1,73 @@ +//! Code to manage compute endpoints +//! +//! In the local test environment, the data for each endpoint is stored in +//! +//! .neon/endpoints/ +//! +//! Some basic information about the endpoint, like the tenant and timeline IDs, +//! are stored in the `endpoint.json` file. The `endpoint.json` file is created +//! when the endpoint is created, and doesn't change afterwards. +//! +//! The endpoint is managed by the `compute_ctl` binary. When an endpoint is +//! started, we launch `compute_ctl` It synchronizes the safekeepers, downloads +//! the basebackup from the pageserver to initialize the the data directory, and +//! finally launches the PostgreSQL process. It watches the PostgreSQL process +//! until it exits. +//! +//! When an endpoint is created, a `postgresql.conf` file is also created in +//! the endpoint's directory. The file can be modified before starting PostgreSQL. +//! However, the `postgresql.conf` file in the endpoint directory is not used directly +//! by PostgreSQL. It is passed to `compute_ctl`, and `compute_ctl` writes another +//! copy of it in the data directory. +//! +//! Directory contents: +//! +//! ```ignore +//! .neon/endpoints/main/ +//! compute.log - log output of `compute_ctl` and `postgres` +//! endpoint.json - serialized `EndpointConf` struct +//! postgresql.conf - postgresql settings +//! spec.json - passed to `compute_ctl` +//! pgdata/ +//! postgresql.conf - copy of postgresql.conf created by `compute_ctl` +//! zenith.signal +//! +//! ``` +//! use std::collections::BTreeMap; -use std::fs::{self, File}; -use std::io::Write; use std::net::SocketAddr; use std::net::TcpStream; -use std::os::unix::fs::PermissionsExt; use std::path::PathBuf; -use std::process::{Command, Stdio}; -use std::str::FromStr; +use std::process::Command; use std::sync::Arc; use std::time::Duration; -use anyhow::{Context, Result}; +use anyhow::{anyhow, bail, Context, Result}; use serde::{Deserialize, Serialize}; use serde_with::{serde_as, DisplayFromStr}; -use utils::{ - id::{TenantId, TimelineId}, - lsn::Lsn, -}; +use utils::id::{NodeId, TenantId, TimelineId}; use crate::local_env::LocalEnv; use crate::pageserver::PageServerNode; use crate::postgresql_conf::PostgresConf; -use compute_api::spec::ComputeMode; +use compute_api::responses::{ComputeState, ComputeStatus}; +use compute_api::spec::{Cluster, ComputeMode, ComputeSpec}; // contents of a endpoint.json file #[serde_as] #[derive(Serialize, Deserialize, PartialEq, Eq, Clone, Debug)] pub struct EndpointConf { - name: String, + endpoint_id: String, #[serde_as(as = "DisplayFromStr")] tenant_id: TenantId, #[serde_as(as = "DisplayFromStr")] timeline_id: TimelineId, mode: ComputeMode, - port: u16, + pg_port: u16, + http_port: u16, pg_version: u32, + skip_pg_catalog_updates: bool, } // @@ -57,11 +89,11 @@ impl ComputeControlPlane { let pageserver = Arc::new(PageServerNode::from_env(&env)); let mut endpoints = BTreeMap::default(); - for endpoint_dir in fs::read_dir(env.endpoints_path()) + for endpoint_dir in std::fs::read_dir(env.endpoints_path()) .with_context(|| format!("failed to list {}", env.endpoints_path().display()))? { let ep = Endpoint::from_dir_entry(endpoint_dir?, &env, &pageserver)?; - endpoints.insert(ep.name.clone(), Arc::new(ep)); + endpoints.insert(ep.endpoint_id.clone(), Arc::new(ep)); } Ok(ComputeControlPlane { @@ -76,47 +108,58 @@ impl ComputeControlPlane { 1 + self .endpoints .values() - .map(|ep| ep.address.port()) + .map(|ep| std::cmp::max(ep.pg_address.port(), ep.http_address.port())) .max() .unwrap_or(self.base_port) } + #[allow(clippy::too_many_arguments)] pub fn new_endpoint( &mut self, + endpoint_id: &str, tenant_id: TenantId, - name: &str, timeline_id: TimelineId, - port: Option, + pg_port: Option, + http_port: Option, pg_version: u32, mode: ComputeMode, ) -> Result> { - let port = port.unwrap_or_else(|| self.get_port()); - + let pg_port = pg_port.unwrap_or_else(|| self.get_port()); + let http_port = http_port.unwrap_or_else(|| self.get_port() + 1); let ep = Arc::new(Endpoint { - name: name.to_owned(), - address: SocketAddr::new("127.0.0.1".parse().unwrap(), port), + endpoint_id: endpoint_id.to_owned(), + pg_address: SocketAddr::new("127.0.0.1".parse().unwrap(), pg_port), + http_address: SocketAddr::new("127.0.0.1".parse().unwrap(), http_port), env: self.env.clone(), pageserver: Arc::clone(&self.pageserver), timeline_id, mode, tenant_id, pg_version, + skip_pg_catalog_updates: false, }); - ep.create_pgdata()?; + + ep.create_endpoint_dir()?; std::fs::write( ep.endpoint_path().join("endpoint.json"), serde_json::to_string_pretty(&EndpointConf { - name: name.to_string(), + endpoint_id: endpoint_id.to_string(), tenant_id, timeline_id, mode, - port, + http_port, + pg_port, pg_version, + skip_pg_catalog_updates: false, })?, )?; - ep.setup_pg_conf()?; + std::fs::write( + ep.endpoint_path().join("postgresql.conf"), + ep.setup_pg_conf()?.to_string(), + )?; - self.endpoints.insert(ep.name.clone(), Arc::clone(&ep)); + self.endpoints + .insert(ep.endpoint_id.clone(), Arc::clone(&ep)); Ok(ep) } @@ -127,13 +170,15 @@ impl ComputeControlPlane { #[derive(Debug)] pub struct Endpoint { /// used as the directory name - name: String, + endpoint_id: String, pub tenant_id: TenantId, pub timeline_id: TimelineId, pub mode: ComputeMode, - // port and address of the Postgres server - pub address: SocketAddr, + // port and address of the Postgres server and `compute_ctl`'s HTTP API + pub pg_address: SocketAddr, + pub http_address: SocketAddr, + // postgres major version in the format: 14, 15, etc. pg_version: u32, @@ -141,6 +186,9 @@ pub struct Endpoint { // the endpoint runs in. pub env: LocalEnv, pageserver: Arc, + + // Optimizations + skip_pg_catalog_updates: bool, } impl Endpoint { @@ -158,123 +206,37 @@ impl Endpoint { // parse data directory name let fname = entry.file_name(); - let name = fname.to_str().unwrap().to_string(); + let endpoint_id = fname.to_str().unwrap().to_string(); // Read the endpoint.json file let conf: EndpointConf = serde_json::from_slice(&std::fs::read(entry.path().join("endpoint.json"))?)?; - // ok now Ok(Endpoint { - address: SocketAddr::new("127.0.0.1".parse().unwrap(), conf.port), - name, + pg_address: SocketAddr::new("127.0.0.1".parse().unwrap(), conf.pg_port), + http_address: SocketAddr::new("127.0.0.1".parse().unwrap(), conf.http_port), + endpoint_id, env: env.clone(), pageserver: Arc::clone(pageserver), timeline_id: conf.timeline_id, mode: conf.mode, tenant_id: conf.tenant_id, pg_version: conf.pg_version, + skip_pg_catalog_updates: conf.skip_pg_catalog_updates, }) } - fn sync_safekeepers(&self, auth_token: &Option, pg_version: u32) -> Result { - let pg_path = self.env.pg_bin_dir(pg_version)?.join("postgres"); - let mut cmd = Command::new(pg_path); - - cmd.arg("--sync-safekeepers") - .env_clear() - .env( - "LD_LIBRARY_PATH", - self.env.pg_lib_dir(pg_version)?.to_str().unwrap(), - ) - .env( - "DYLD_LIBRARY_PATH", - self.env.pg_lib_dir(pg_version)?.to_str().unwrap(), - ) - .env("PGDATA", self.pgdata().to_str().unwrap()) - .stdout(Stdio::piped()) - // Comment this to avoid capturing stderr (useful if command hangs) - .stderr(Stdio::piped()); - - if let Some(token) = auth_token { - cmd.env("NEON_AUTH_TOKEN", token); - } - - let sync_handle = cmd - .spawn() - .expect("postgres --sync-safekeepers failed to start"); - - let sync_output = sync_handle - .wait_with_output() - .expect("postgres --sync-safekeepers failed"); - if !sync_output.status.success() { - anyhow::bail!( - "sync-safekeepers failed: '{}'", - String::from_utf8_lossy(&sync_output.stderr) - ); - } - - let lsn = Lsn::from_str(std::str::from_utf8(&sync_output.stdout)?.trim())?; - println!("Safekeepers synced on {}", lsn); - Ok(lsn) - } - - /// Get basebackup from the pageserver as a tar archive and extract it - /// to the `self.pgdata()` directory. - fn do_basebackup(&self, lsn: Option) -> Result<()> { - println!( - "Extracting base backup to create postgres instance: path={} port={}", - self.pgdata().display(), - self.address.port() - ); - - let sql = if let Some(lsn) = lsn { - format!("basebackup {} {} {}", self.tenant_id, self.timeline_id, lsn) - } else { - format!("basebackup {} {}", self.tenant_id, self.timeline_id) - }; - - let mut client = self - .pageserver - .page_server_psql_client() - .context("connecting to page server failed")?; - - let copyreader = client - .copy_out(sql.as_str()) - .context("page server 'basebackup' command failed")?; - - // Read the archive directly from the `CopyOutReader` - // - // Set `ignore_zeros` so that unpack() reads all the Copy data and - // doesn't stop at the end-of-archive marker. Otherwise, if the server - // sends an Error after finishing the tarball, we will not notice it. - let mut ar = tar::Archive::new(copyreader); - ar.set_ignore_zeros(true); - ar.unpack(&self.pgdata()) - .context("extracting base backup failed")?; - - Ok(()) - } - - fn create_pgdata(&self) -> Result<()> { - fs::create_dir_all(self.pgdata()).with_context(|| { + fn create_endpoint_dir(&self) -> Result<()> { + std::fs::create_dir_all(self.endpoint_path()).with_context(|| { format!( - "could not create data directory {}", - self.pgdata().display() + "could not create endpoint directory {}", + self.endpoint_path().display() ) - })?; - fs::set_permissions(self.pgdata().as_path(), fs::Permissions::from_mode(0o700)) - .with_context(|| { - format!( - "could not set permissions in data directory {}", - self.pgdata().display() - ) - }) + }) } - // Write postgresql.conf with default configuration - // and PG_VERSION file to the data directory of a new endpoint. - fn setup_pg_conf(&self) -> Result<()> { + // Generate postgresql.conf with default configuration + fn setup_pg_conf(&self) -> Result { let mut conf = PostgresConf::new(); conf.append("max_wal_senders", "10"); conf.append("wal_log_hints", "off"); @@ -287,25 +249,14 @@ impl Endpoint { // wal_sender_timeout is the maximum time to wait for WAL replication. // It also defines how often the walreciever will send a feedback message to the wal sender. conf.append("wal_sender_timeout", "5s"); - conf.append("listen_addresses", &self.address.ip().to_string()); - conf.append("port", &self.address.port().to_string()); + conf.append("listen_addresses", &self.pg_address.ip().to_string()); + conf.append("port", &self.pg_address.port().to_string()); conf.append("wal_keep_size", "0"); // walproposer panics when basebackup is invalid, it is pointless to restart in this case. conf.append("restart_after_crash", "off"); - // Configure the Neon Postgres extension to fetch pages from pageserver - let pageserver_connstr = { - let config = &self.pageserver.pg_connection_config; - let (host, port) = (config.host(), config.port()); - - // NOTE: avoid spaces in connection string, because it is less error prone if we forward it somewhere. - format!("postgresql://no_user@{host}:{port}") - }; + // Load the 'neon' extension conf.append("shared_preload_libraries", "neon"); - conf.append_line(""); - conf.append("neon.pageserver_connstring", &pageserver_connstr); - conf.append("neon.tenant_id", &self.tenant_id.to_string()); - conf.append("neon.timeline_id", &self.timeline_id.to_string()); conf.append_line(""); // Replication-related configurations, such as WAL sending @@ -390,46 +341,11 @@ impl Endpoint { } } - let mut file = File::create(self.pgdata().join("postgresql.conf"))?; - file.write_all(conf.to_string().as_bytes())?; - - let mut file = File::create(self.pgdata().join("PG_VERSION"))?; - file.write_all(self.pg_version.to_string().as_bytes())?; - - Ok(()) - } - - fn load_basebackup(&self, auth_token: &Option) -> Result<()> { - let backup_lsn = match &self.mode { - ComputeMode::Primary => { - if !self.env.safekeepers.is_empty() { - // LSN 0 means that it is bootstrap and we need to download just - // latest data from the pageserver. That is a bit clumsy but whole bootstrap - // procedure evolves quite actively right now, so let's think about it again - // when things would be more stable (TODO). - let lsn = self.sync_safekeepers(auth_token, self.pg_version)?; - if lsn == Lsn(0) { - None - } else { - Some(lsn) - } - } else { - None - } - } - ComputeMode::Static(lsn) => Some(*lsn), - ComputeMode::Replica => { - None // Take the latest snapshot available to start with - } - }; - - self.do_basebackup(backup_lsn)?; - - Ok(()) + Ok(conf) } pub fn endpoint_path(&self) -> PathBuf { - self.env.endpoints_path().join(&self.name) + self.env.endpoints_path().join(&self.endpoint_id) } pub fn pgdata(&self) -> PathBuf { @@ -439,7 +355,7 @@ impl Endpoint { pub fn status(&self) -> &str { let timeout = Duration::from_millis(300); let has_pidfile = self.pgdata().join("postmaster.pid").exists(); - let can_connect = TcpStream::connect_timeout(&self.address, timeout).is_ok(); + let can_connect = TcpStream::connect_timeout(&self.pg_address, timeout).is_ok(); match (has_pidfile, can_connect) { (true, true) => "running", @@ -457,8 +373,6 @@ impl Endpoint { &[ "-D", self.pgdata().to_str().unwrap(), - "-l", - self.pgdata().join("pg.log").to_str().unwrap(), "-w", //wait till pg_ctl actually does what was asked ], args, @@ -491,39 +405,203 @@ impl Endpoint { String::from_utf8_lossy(&pg_ctl.stderr), ); } + + // Also wait for the compute_ctl process to die. It might have some cleanup + // work to do after postgres stops, like syncing safekeepers, etc. + // + // TODO use background_process::stop_process instead + let pidfile_path = self.endpoint_path().join("compute_ctl.pid"); + let pid: u32 = std::fs::read_to_string(pidfile_path)?.parse()?; + let pid = nix::unistd::Pid::from_raw(pid as i32); + crate::background_process::wait_until_stopped("compute_ctl", pid)?; + Ok(()) } - pub fn start(&self, auth_token: &Option) -> Result<()> { + pub fn start(&self, auth_token: &Option, safekeepers: Vec) -> Result<()> { if self.status() == "running" { anyhow::bail!("The endpoint is already running"); } - // 1. We always start Postgres from scratch, so - // if old dir exists, preserve 'postgresql.conf' and drop the directory - let postgresql_conf_path = self.pgdata().join("postgresql.conf"); - let postgresql_conf = fs::read(&postgresql_conf_path).with_context(|| { - format!( - "failed to read config file in {}", - postgresql_conf_path.to_str().unwrap() - ) - })?; - fs::remove_dir_all(self.pgdata())?; - self.create_pgdata()?; + // Slurp the endpoints//postgresql.conf file into + // memory. We will include it in the spec file that we pass to + // `compute_ctl`, and `compute_ctl` will write it to the postgresql.conf + // in the data directory. + let postgresql_conf_path = self.endpoint_path().join("postgresql.conf"); + let postgresql_conf = match std::fs::read(&postgresql_conf_path) { + Ok(content) => String::from_utf8(content)?, + Err(e) if e.kind() == std::io::ErrorKind::NotFound => "".to_string(), + Err(e) => { + return Err(anyhow::Error::new(e).context(format!( + "failed to read config file in {}", + postgresql_conf_path.to_str().unwrap() + ))) + } + }; - // 2. Bring back config files - fs::write(&postgresql_conf_path, postgresql_conf)?; - - // 3. Load basebackup - self.load_basebackup(auth_token)?; - - if self.mode != ComputeMode::Primary { - File::create(self.pgdata().join("standby.signal"))?; + // We always start the compute node from scratch, so if the Postgres + // data dir exists from a previous launch, remove it first. + if self.pgdata().exists() { + std::fs::remove_dir_all(self.pgdata())?; } - // 4. Finally start postgres - println!("Starting postgres at '{}'", self.connstr()); - self.pg_ctl(&["start"], auth_token) + let pageserver_connstring = { + let config = &self.pageserver.pg_connection_config; + let (host, port) = (config.host(), config.port()); + + // NOTE: avoid spaces in connection string, because it is less error prone if we forward it somewhere. + format!("postgresql://no_user@{host}:{port}") + }; + let mut safekeeper_connstrings = Vec::new(); + if self.mode == ComputeMode::Primary { + for sk_id in safekeepers { + let sk = self + .env + .safekeepers + .iter() + .find(|node| node.id == sk_id) + .ok_or_else(|| anyhow!("safekeeper {sk_id} does not exist"))?; + safekeeper_connstrings.push(format!("127.0.0.1:{}", sk.pg_port)); + } + } + + // Create spec file + let spec = ComputeSpec { + skip_pg_catalog_updates: self.skip_pg_catalog_updates, + format_version: 1.0, + operation_uuid: None, + cluster: Cluster { + cluster_id: None, // project ID: not used + name: None, // project name: not used + state: None, + roles: vec![], + databases: vec![], + settings: None, + postgresql_conf: Some(postgresql_conf), + }, + delta_operations: None, + tenant_id: Some(self.tenant_id), + timeline_id: Some(self.timeline_id), + mode: self.mode, + pageserver_connstring: Some(pageserver_connstring), + safekeeper_connstrings, + storage_auth_token: auth_token.clone(), + }; + let spec_path = self.endpoint_path().join("spec.json"); + std::fs::write(spec_path, serde_json::to_string_pretty(&spec)?)?; + + // Open log file. We'll redirect the stdout and stderr of `compute_ctl` to it. + let logfile = std::fs::OpenOptions::new() + .create(true) + .append(true) + .open(self.endpoint_path().join("compute.log"))?; + + // Launch compute_ctl + println!("Starting postgres node at '{}'", self.connstr()); + let mut cmd = Command::new(self.env.neon_distrib_dir.join("compute_ctl")); + cmd.args(["--http-port", &self.http_address.port().to_string()]) + .args(["--pgdata", self.pgdata().to_str().unwrap()]) + .args(["--connstr", &self.connstr()]) + .args([ + "--spec-path", + self.endpoint_path().join("spec.json").to_str().unwrap(), + ]) + .args([ + "--pgbin", + self.env + .pg_bin_dir(self.pg_version)? + .join("postgres") + .to_str() + .unwrap(), + ]) + .stdin(std::process::Stdio::null()) + .stderr(logfile.try_clone()?) + .stdout(logfile); + let child = cmd.spawn()?; + + // Write down the pid so we can wait for it when we want to stop + // TODO use background_process::start_process instead + let pid = child.id(); + let pidfile_path = self.endpoint_path().join("compute_ctl.pid"); + std::fs::write(pidfile_path, pid.to_string())?; + + // Wait for it to start + let mut attempt = 0; + const ATTEMPT_INTERVAL: Duration = Duration::from_millis(100); + const MAX_ATTEMPTS: u32 = 10 * 30; // Wait up to 30 s + loop { + attempt += 1; + match self.get_status() { + Ok(state) => { + match state.status { + ComputeStatus::Init => { + if attempt == MAX_ATTEMPTS { + bail!("compute startup timed out; still in Init state"); + } + // keep retrying + } + ComputeStatus::Running => { + // All good! + break; + } + ComputeStatus::Failed => { + bail!( + "compute startup failed: {}", + state + .error + .as_deref() + .unwrap_or("") + ); + } + ComputeStatus::Empty + | ComputeStatus::ConfigurationPending + | ComputeStatus::Configuration => { + bail!("unexpected compute status: {:?}", state.status) + } + } + } + Err(e) => { + if attempt == MAX_ATTEMPTS { + return Err(e).context( + "timed out waiting to connect to compute_ctl HTTP; last error: {e}", + ); + } + } + } + std::thread::sleep(ATTEMPT_INTERVAL); + } + + Ok(()) + } + + // Call the /status HTTP API + pub fn get_status(&self) -> Result { + let client = reqwest::blocking::Client::new(); + + let response = client + .request( + reqwest::Method::GET, + format!( + "http://{}:{}/status", + self.http_address.ip(), + self.http_address.port() + ), + ) + .send()?; + + // Interpret the response + let status = response.status(); + if !(status.is_client_error() || status.is_server_error()) { + Ok(response.json()?) + } else { + // reqwest does not export its error construction utility functions, so let's craft the message ourselves + let url = response.url().to_owned(); + let msg = match response.text() { + Ok(err_body) => format!("Error: {}", err_body), + Err(_) => format!("Http error ({}) at {}.", status.as_u16(), url), + }; + Err(anyhow::anyhow!(msg)) + } } pub fn stop(&self, destroy: bool) -> Result<()> { @@ -540,7 +618,7 @@ impl Endpoint { "Destroying postgres data directory '{}'", self.pgdata().to_str().unwrap() ); - fs::remove_dir_all(self.endpoint_path())?; + std::fs::remove_dir_all(self.endpoint_path())?; } else { self.pg_ctl(&["stop"], &None)?; } @@ -549,10 +627,10 @@ impl Endpoint { pub fn connstr(&self) -> String { format!( - "host={} port={} user={} dbname={}", - self.address.ip(), - self.address.port(), + "postgresql://{}@{}:{}/{}", "cloud_admin", + self.pg_address.ip(), + self.pg_address.port(), "postgres" ) } diff --git a/control_plane/src/local_env.rs b/control_plane/src/local_env.rs index 9286944412..208eb9e7ec 100644 --- a/control_plane/src/local_env.rs +++ b/control_plane/src/local_env.rs @@ -37,7 +37,7 @@ pub const DEFAULT_PG_VERSION: u32 = 15; #[derive(Serialize, Deserialize, PartialEq, Eq, Clone, Debug)] pub struct LocalEnv { // Base directory for all the nodes (the pageserver, safekeepers and - // compute nodes). + // compute endpoints). // // This is not stored in the config file. Rather, this is the path where the // config file itself is. It is read from the NEON_REPO_DIR env variable or @@ -364,7 +364,7 @@ impl LocalEnv { // // Initialize a new Neon repository // - pub fn init(&mut self, pg_version: u32) -> anyhow::Result<()> { + pub fn init(&mut self, pg_version: u32, force: bool) -> anyhow::Result<()> { // check if config already exists let base_path = &self.base_data_dir; ensure!( @@ -372,11 +372,29 @@ impl LocalEnv { "repository base path is missing" ); - ensure!( - !base_path.exists(), - "directory '{}' already exists. Perhaps already initialized?", - base_path.display() - ); + if base_path.exists() { + if force { + println!("removing all contents of '{}'", base_path.display()); + // instead of directly calling `remove_dir_all`, we keep the original dir but removing + // all contents inside. This helps if the developer symbol links another directory (i.e., + // S3 local SSD) to the `.neon` base directory. + for entry in std::fs::read_dir(base_path)? { + let entry = entry?; + let path = entry.path(); + if path.is_dir() { + fs::remove_dir_all(&path)?; + } else { + fs::remove_file(&path)?; + } + } + } else { + bail!( + "directory '{}' already exists. Perhaps already initialized? (Hint: use --force to remove all contents)", + base_path.display() + ); + } + } + if !self.pg_bin_dir(pg_version)?.join("postgres").exists() { bail!( "Can't find postgres binary at {}", @@ -392,7 +410,9 @@ impl LocalEnv { } } - fs::create_dir(base_path)?; + if !base_path.exists() { + fs::create_dir(base_path)?; + } // Generate keypair for JWT. // diff --git a/control_plane/src/pageserver.rs b/control_plane/src/pageserver.rs index 149cfd00cb..2ff09021e5 100644 --- a/control_plane/src/pageserver.rs +++ b/control_plane/src/pageserver.rs @@ -1,3 +1,9 @@ +//! Code to manage pageservers +//! +//! In the local test environment, the pageserver stores its data directly in +//! +//! .neon/ +//! use std::borrow::Cow; use std::collections::HashMap; use std::fs::File; @@ -369,6 +375,11 @@ impl PageServerNode { evictions_low_residence_duration_metric_threshold: settings .remove("evictions_low_residence_duration_metric_threshold") .map(|x| x.to_string()), + gc_feedback: settings + .remove("gc_feedback") + .map(|x| x.parse::()) + .transpose() + .context("Failed to parse 'gc_feedback' as bool")?, }; // If tenant ID was not specified, generate one @@ -463,6 +474,11 @@ impl PageServerNode { evictions_low_residence_duration_metric_threshold: settings .remove("evictions_low_residence_duration_metric_threshold") .map(|x| x.to_string()), + gc_feedback: settings + .remove("gc_feedback") + .map(|x| x.parse::()) + .transpose() + .context("Failed to parse 'gc_feedback' as bool")?, } }; diff --git a/control_plane/src/safekeeper.rs b/control_plane/src/safekeeper.rs index d358f73343..9e053ff1f1 100644 --- a/control_plane/src/safekeeper.rs +++ b/control_plane/src/safekeeper.rs @@ -1,3 +1,9 @@ +//! Code to manage safekeepers +//! +//! In the local test environment, the data for each safekeeper is stored in +//! +//! .neon/safekeepers/ +//! use std::io::Write; use std::path::PathBuf; use std::process::Child; diff --git a/docs/pageserver-thread-mgmt.md b/docs/pageserver-thread-mgmt.md index 0cc897f154..b911933528 100644 --- a/docs/pageserver-thread-mgmt.md +++ b/docs/pageserver-thread-mgmt.md @@ -52,9 +52,7 @@ completion, or shield the rest of the code from surprise cancellations by spawning a separate task. The code that handles incoming HTTP requests, for example, spawns a separate task for each request, because Hyper will drop the request-handling Future if the HTTP -connection is lost. (FIXME: our HTTP handlers do not do that -currently, but we should fix that. See [issue -3478](https://github.com/neondatabase/neon/issues/3478)). +connection is lost. #### How to cancel, then? diff --git a/libs/compute_api/src/responses.rs b/libs/compute_api/src/responses.rs index d181c018b1..ce73dda08a 100644 --- a/libs/compute_api/src/responses.rs +++ b/libs/compute_api/src/responses.rs @@ -5,13 +5,13 @@ use serde::{Deserialize, Serialize, Serializer}; use crate::spec::ComputeSpec; -#[derive(Serialize, Debug)] +#[derive(Serialize, Debug, Deserialize)] pub struct GenericAPIError { pub error: String, } /// Response of the /status API -#[derive(Serialize, Debug)] +#[derive(Serialize, Debug, Deserialize)] #[serde(rename_all = "snake_case")] pub struct ComputeStatusResponse { pub start_time: DateTime, @@ -23,7 +23,7 @@ pub struct ComputeStatusResponse { pub error: Option, } -#[derive(Serialize)] +#[derive(Deserialize, Serialize)] #[serde(rename_all = "snake_case")] pub struct ComputeState { pub status: ComputeStatus, @@ -33,7 +33,7 @@ pub struct ComputeState { pub error: Option, } -#[derive(Serialize, Clone, Copy, Debug, PartialEq, Eq)] +#[derive(Serialize, Clone, Copy, Debug, Deserialize, PartialEq, Eq)] #[serde(rename_all = "snake_case")] pub enum ComputeStatus { // Spec wasn't provided at start, waiting for it to be diff --git a/libs/compute_api/src/spec.rs b/libs/compute_api/src/spec.rs index 6072980ed8..b3f0e9ba43 100644 --- a/libs/compute_api/src/spec.rs +++ b/libs/compute_api/src/spec.rs @@ -5,6 +5,7 @@ //! and connect it to the storage nodes. use serde::{Deserialize, Serialize}; use serde_with::{serde_as, DisplayFromStr}; +use utils::id::{TenantId, TimelineId}; use utils::lsn::Lsn; /// String type alias representing Postgres identifier and @@ -14,7 +15,7 @@ pub type PgIdent = String; /// Cluster spec or configuration represented as an optional number of /// delta operations + final cluster state description. #[serde_as] -#[derive(Clone, Debug, Default, Deserialize)] +#[derive(Clone, Debug, Default, Deserialize, Serialize)] pub struct ComputeSpec { pub format_version: f32, @@ -26,9 +27,38 @@ pub struct ComputeSpec { pub cluster: Cluster, pub delta_operations: Option>, + /// An optinal hint that can be passed to speed up startup time if we know + /// that no pg catalog mutations (like role creation, database creation, + /// extension creation) need to be done on the actual database to start. + #[serde(default)] // Default false + pub skip_pg_catalog_updates: bool, + + // Information needed to connect to the storage layer. + // + // `tenant_id`, `timeline_id` and `pageserver_connstring` are always needed. + // + // Depending on `mode`, this can be a primary read-write node, a read-only + // replica, or a read-only node pinned at an older LSN. + // `safekeeper_connstrings` must be set for a primary. + // + // For backwards compatibility, the control plane may leave out all of + // these, and instead set the "neon.tenant_id", "neon.timeline_id", + // etc. GUCs in cluster.settings. TODO: Once the control plane has been + // updated to fill these fields, we can make these non optional. + #[serde_as(as = "Option")] + pub tenant_id: Option, + #[serde_as(as = "Option")] + pub timeline_id: Option, + #[serde_as(as = "Option")] + pub pageserver_connstring: Option, + #[serde(default)] + pub safekeeper_connstrings: Vec, + #[serde(default)] pub mode: ComputeMode, + /// If set, 'storage_auth_token' is used as the password to authenticate to + /// the pageserver and safekeepers. pub storage_auth_token: Option, } @@ -47,13 +77,19 @@ pub enum ComputeMode { Replica, } -#[derive(Clone, Debug, Default, Deserialize)] +#[derive(Clone, Debug, Default, Deserialize, Serialize)] pub struct Cluster { - pub cluster_id: String, - pub name: String, + pub cluster_id: Option, + pub name: Option, pub state: Option, pub roles: Vec, pub databases: Vec, + + /// Desired contents of 'postgresql.conf' file. (The 'compute_ctl' + /// tool may add additional settings to the final file.) + pub postgresql_conf: Option, + + /// Additional settings that will be appended to the 'postgresql.conf' file. pub settings: GenericOptions, } @@ -63,7 +99,7 @@ pub struct Cluster { /// - DROP ROLE /// - ALTER ROLE name RENAME TO new_name /// - ALTER DATABASE name RENAME TO new_name -#[derive(Clone, Debug, Deserialize)] +#[derive(Clone, Debug, Deserialize, Serialize)] pub struct DeltaOp { pub action: String, pub name: PgIdent, @@ -72,7 +108,7 @@ pub struct DeltaOp { /// Rust representation of Postgres role info with only those fields /// that matter for us. -#[derive(Clone, Debug, Deserialize)] +#[derive(Clone, Debug, Deserialize, Serialize)] pub struct Role { pub name: PgIdent, pub encrypted_password: Option, @@ -81,7 +117,7 @@ pub struct Role { /// Rust representation of Postgres database info with only those fields /// that matter for us. -#[derive(Clone, Debug, Deserialize)] +#[derive(Clone, Debug, Deserialize, Serialize)] pub struct Database { pub name: PgIdent, pub owner: PgIdent, @@ -91,7 +127,7 @@ pub struct Database { /// Common type representing both SQL statement params with or without value, /// like `LOGIN` or `OWNER username` in the `CREATE/ALTER ROLE`, and config /// options like `wal_level = logical`. -#[derive(Clone, Debug, Deserialize)] +#[derive(Clone, Debug, Deserialize, Serialize)] pub struct GenericOption { pub name: String, pub value: Option, @@ -112,4 +148,14 @@ mod tests { let file = File::open("tests/cluster_spec.json").unwrap(); let _spec: ComputeSpec = serde_json::from_reader(file).unwrap(); } + + #[test] + fn parse_unknown_fields() { + // Forward compatibility test + let file = File::open("tests/cluster_spec.json").unwrap(); + let mut json: serde_json::Value = serde_json::from_reader(file).unwrap(); + let ob = json.as_object_mut().unwrap(); + ob.insert("unknown_field_123123123".into(), "hello".into()); + let _spec: ComputeSpec = serde_json::from_value(json).unwrap(); + } } diff --git a/libs/metrics/src/lib.rs b/libs/metrics/src/lib.rs index 6011713c8f..f24488b19d 100644 --- a/libs/metrics/src/lib.rs +++ b/libs/metrics/src/lib.rs @@ -23,6 +23,7 @@ use prometheus::{Registry, Result}; pub mod launch_timestamp; mod wrappers; pub use wrappers::{CountedReader, CountedWriter}; +pub mod metric_vec_duration; pub type UIntGauge = GenericGauge; pub type UIntGaugeVec = GenericGaugeVec; diff --git a/libs/metrics/src/metric_vec_duration.rs b/libs/metrics/src/metric_vec_duration.rs new file mode 100644 index 0000000000..840f60f19b --- /dev/null +++ b/libs/metrics/src/metric_vec_duration.rs @@ -0,0 +1,23 @@ +//! Helpers for observing duration on HistogramVec / CounterVec / GaugeVec / MetricVec. + +use std::{future::Future, time::Instant}; + +pub trait DurationResultObserver { + fn observe_result(&self, res: &Result, duration: std::time::Duration); +} + +pub async fn observe_async_block_duration_by_result< + T, + E, + F: Future>, + O: DurationResultObserver, +>( + observer: &O, + block: F, +) -> Result { + let start = Instant::now(); + let result = block.await; + let duration = start.elapsed(); + observer.observe_result(&result, duration); + result +} diff --git a/libs/pageserver_api/src/models.rs b/libs/pageserver_api/src/models.rs index 540633d113..df5f5896a1 100644 --- a/libs/pageserver_api/src/models.rs +++ b/libs/pageserver_api/src/models.rs @@ -18,7 +18,29 @@ use crate::reltag::RelTag; use anyhow::bail; use bytes::{BufMut, Bytes, BytesMut}; -/// A state of a tenant in pageserver's memory. +/// The state of a tenant in this pageserver. +/// +/// ```mermaid +/// stateDiagram-v2 +/// +/// [*] --> Loading: spawn_load() +/// [*] --> Attaching: spawn_attach() +/// +/// Loading --> Activating: activate() +/// Attaching --> Activating: activate() +/// Activating --> Active: infallible +/// +/// Loading --> Broken: load() failure +/// Attaching --> Broken: attach() failure +/// +/// Active --> Stopping: set_stopping(), part of shutdown & detach +/// Stopping --> Broken: late error in remove_tenant_from_memory +/// +/// Broken --> [*]: ignore / detach / shutdown +/// Stopping --> [*]: remove_from_memory complete +/// +/// Active --> Broken: cfg(testing)-only tenant break point +/// ``` #[derive( Clone, PartialEq, @@ -26,51 +48,73 @@ use bytes::{BufMut, Bytes, BytesMut}; serde::Serialize, serde::Deserialize, strum_macros::Display, - strum_macros::EnumString, strum_macros::EnumVariantNames, strum_macros::AsRefStr, strum_macros::IntoStaticStr, )] #[serde(tag = "slug", content = "data")] pub enum TenantState { - /// This tenant is being loaded from local disk + /// This tenant is being loaded from local disk. + /// + /// `set_stopping()` and `set_broken()` do not work in this state and wait for it to pass. Loading, - /// This tenant is being downloaded from cloud storage. + /// This tenant is being attached to the pageserver. + /// + /// `set_stopping()` and `set_broken()` do not work in this state and wait for it to pass. Attaching, - /// Tenant is fully operational + /// The tenant is transitioning from Loading/Attaching to Active. + /// + /// While in this state, the individual timelines are being activated. + /// + /// `set_stopping()` and `set_broken()` do not work in this state and wait for it to pass. + Activating(ActivatingFrom), + /// The tenant has finished activating and is open for business. + /// + /// Transitions out of this state are possible through `set_stopping()` and `set_broken()`. Active, - /// A tenant is recognized by pageserver, but it is being detached or the + /// The tenant is recognized by pageserver, but it is being detached or the /// system is being shut down. + /// + /// Transitions out of this state are possible through `set_broken()`. Stopping, - /// A tenant is recognized by the pageserver, but can no longer be used for - /// any operations, because it failed to be activated. + /// The tenant is recognized by the pageserver, but can no longer be used for + /// any operations. + /// + /// If the tenant fails to load or attach, it will transition to this state + /// and it is guaranteed that no background tasks are running in its name. + /// + /// The other way to transition into this state is from `Stopping` state + /// through `set_broken()` called from `remove_tenant_from_memory()`. That happens + /// if the cleanup future executed by `remove_tenant_from_memory()` fails. Broken { reason: String, backtrace: String }, } impl TenantState { pub fn attachment_status(&self) -> TenantAttachmentStatus { use TenantAttachmentStatus::*; + + // Below TenantState::Activating is used as "transient" or "transparent" state for + // attachment_status determining. match self { // The attach procedure writes the marker file before adding the Attaching tenant to the tenants map. // So, technically, we can return Attached here. // However, as soon as Console observes Attached, it will proceed with the Postgres-level health check. // But, our attach task might still be fetching the remote timelines, etc. // So, return `Maybe` while Attaching, making Console wait for the attach task to finish. - Self::Attaching => Maybe, + Self::Attaching | Self::Activating(ActivatingFrom::Attaching) => Maybe, // tenant mgr startup distinguishes attaching from loading via marker file. // If it's loading, there is no attach marker file, i.e., attach had finished in the past. - Self::Loading => Attached, + Self::Loading | Self::Activating(ActivatingFrom::Loading) => Attached, // We only reach Active after successful load / attach. // So, call atttachment status Attached. Self::Active => Attached, // If the (initial or resumed) attach procedure fails, the tenant becomes Broken. // However, it also becomes Broken if the regular load fails. - // We would need a separate TenantState variant to distinguish these cases. - // However, there's no practical difference from Console's perspective. - // It will run a Postgres-level health check as soon as it observes Attached. - // That will fail on Broken tenants. - // Console can then rollback the attach, or, wait for operator to fix the Broken tenant. - Self::Broken { .. } => Attached, + // From Console's perspective there's no practical difference + // because attachment_status is polled by console only during attach operation execution. + Self::Broken { reason, .. } => Failed { + reason: reason.to_owned(), + }, // Why is Stopping a Maybe case? Because, during pageserver shutdown, // we set the Stopping state irrespective of whether the tenant // has finished attaching or not. @@ -98,8 +142,17 @@ impl std::fmt::Debug for TenantState { } } +/// The only [`TenantState`] variants we could be `TenantState::Activating` from. +#[derive(Clone, Copy, Debug, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +pub enum ActivatingFrom { + /// Arrived to [`TenantState::Activating`] from [`TenantState::Loading`] + Loading, + /// Arrived to [`TenantState::Activating`] from [`TenantState::Attaching`] + Attaching, +} + /// A state of a timeline in pageserver's memory. -#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] pub enum TimelineState { /// The timeline is recognized by the pageserver but is not yet operational. /// In particular, the walreceiver connection loop is not running for this timeline. @@ -112,7 +165,7 @@ pub enum TimelineState { /// It cannot transition back into any other state. Stopping, /// The timeline is broken and not operational (previous states: Loading or Active). - Broken, + Broken { reason: String, backtrace: String }, } #[serde_as] @@ -169,6 +222,7 @@ pub struct TenantConfig { pub eviction_policy: Option, pub min_resident_size_override: Option, pub evictions_low_residence_duration_metric_threshold: Option, + pub gc_feedback: Option, } #[serde_as] @@ -227,6 +281,7 @@ impl TenantConfigRequest { eviction_policy: None, min_resident_size_override: None, evictions_low_residence_duration_metric_threshold: None, + gc_feedback: None, }; TenantConfigRequest { tenant_id, config } } @@ -256,10 +311,11 @@ impl std::ops::Deref for TenantAttachConfig { /// See [`TenantState::attachment_status`] and the OpenAPI docs for context. #[derive(Serialize, Deserialize, Clone)] -#[serde(rename_all = "snake_case")] +#[serde(tag = "slug", content = "data", rename_all = "snake_case")] pub enum TenantAttachmentStatus { Maybe, Attached, + Failed { reason: String }, } #[serde_as] @@ -753,7 +809,9 @@ mod tests { "slug": "Active", }, "current_physical_size": 42, - "attachment_status": "attached", + "attachment_status": { + "slug":"attached", + } }); let original_broken = TenantInfo { @@ -775,7 +833,9 @@ mod tests { } }, "current_physical_size": 42, - "attachment_status": "attached", + "attachment_status": { + "slug":"attached", + } }); assert_eq!( @@ -829,4 +889,55 @@ mod tests { err ); } + + #[test] + fn tenantstatus_activating_serde() { + let states = [ + TenantState::Activating(ActivatingFrom::Loading), + TenantState::Activating(ActivatingFrom::Attaching), + ]; + let expected = "[{\"slug\":\"Activating\",\"data\":\"Loading\"},{\"slug\":\"Activating\",\"data\":\"Attaching\"}]"; + + let actual = serde_json::to_string(&states).unwrap(); + + assert_eq!(actual, expected); + + let parsed = serde_json::from_str::>(&actual).unwrap(); + + assert_eq!(states.as_slice(), &parsed); + } + + #[test] + fn tenantstatus_activating_strum() { + // tests added, because we use these for metrics + let examples = [ + (line!(), TenantState::Loading, "Loading"), + (line!(), TenantState::Attaching, "Attaching"), + ( + line!(), + TenantState::Activating(ActivatingFrom::Loading), + "Activating", + ), + ( + line!(), + TenantState::Activating(ActivatingFrom::Attaching), + "Activating", + ), + (line!(), TenantState::Active, "Active"), + (line!(), TenantState::Stopping, "Stopping"), + ( + line!(), + TenantState::Broken { + reason: "Example".into(), + backtrace: "Looooong backtrace".into(), + }, + "Broken", + ), + ]; + + for (line, rendered, expected) in examples { + let actual: &'static str = rendered.into(); + assert_eq!(actual, expected, "example on {line}"); + } + } } diff --git a/libs/postgres_ffi/Cargo.toml b/libs/postgres_ffi/Cargo.toml index 159fc5946d..86e72f6bdd 100644 --- a/libs/postgres_ffi/Cargo.toml +++ b/libs/postgres_ffi/Cargo.toml @@ -24,7 +24,6 @@ workspace_hack.workspace = true [dev-dependencies] env_logger.workspace = true postgres.workspace = true -wal_craft = { path = "wal_craft" } [build-dependencies] anyhow.workspace = true diff --git a/libs/postgres_ffi/src/lib.rs b/libs/postgres_ffi/src/lib.rs index b8eb469cb0..cc115664d5 100644 --- a/libs/postgres_ffi/src/lib.rs +++ b/libs/postgres_ffi/src/lib.rs @@ -33,6 +33,7 @@ macro_rules! postgres_ffi { } pub mod controlfile_utils; pub mod nonrelfile_utils; + pub mod wal_craft_test_export; pub mod waldecoder_handler; pub mod xlog_utils; @@ -45,8 +46,15 @@ macro_rules! postgres_ffi { }; } -postgres_ffi!(v14); -postgres_ffi!(v15); +#[macro_export] +macro_rules! for_all_postgres_versions { + ($macro:tt) => { + $macro!(v14); + $macro!(v15); + }; +} + +for_all_postgres_versions! { postgres_ffi } pub mod pg_constants; pub mod relfile_utils; diff --git a/libs/postgres_ffi/src/wal_craft_test_export.rs b/libs/postgres_ffi/src/wal_craft_test_export.rs new file mode 100644 index 0000000000..147567c442 --- /dev/null +++ b/libs/postgres_ffi/src/wal_craft_test_export.rs @@ -0,0 +1,6 @@ +//! This module is for WAL craft to test with postgres_ffi. Should not import any thing in normal usage. + +pub use super::PG_MAJORVERSION; +pub use super::xlog_utils::*; +pub use super::bindings::*; +pub use crate::WAL_SEGMENT_SIZE; diff --git a/libs/postgres_ffi/src/xlog_utils.rs b/libs/postgres_ffi/src/xlog_utils.rs index 4d7bb61883..61a9c38a84 100644 --- a/libs/postgres_ffi/src/xlog_utils.rs +++ b/libs/postgres_ffi/src/xlog_utils.rs @@ -481,220 +481,4 @@ pub fn encode_logical_message(prefix: &str, message: &str) -> Vec { wal } -#[cfg(test)] -mod tests { - use super::super::PG_MAJORVERSION; - use super::*; - use regex::Regex; - use std::cmp::min; - use std::fs; - use std::{env, str::FromStr}; - use utils::const_assert; - - fn init_logging() { - let _ = env_logger::Builder::from_env(env_logger::Env::default().default_filter_or( - format!("wal_craft=info,postgres_ffi::{PG_MAJORVERSION}::xlog_utils=trace"), - )) - .is_test(true) - .try_init(); - } - - fn test_end_of_wal(test_name: &str) { - use wal_craft::*; - - let pg_version = PG_MAJORVERSION[1..3].parse::().unwrap(); - - // Craft some WAL - let top_path = PathBuf::from(env!("CARGO_MANIFEST_DIR")) - .join("..") - .join(".."); - let cfg = Conf { - pg_version, - pg_distrib_dir: top_path.join("pg_install"), - datadir: top_path.join(format!("test_output/{}-{PG_MAJORVERSION}", test_name)), - }; - if cfg.datadir.exists() { - fs::remove_dir_all(&cfg.datadir).unwrap(); - } - cfg.initdb().unwrap(); - let srv = cfg.start_server().unwrap(); - let (intermediate_lsns, expected_end_of_wal_partial) = - C::craft(&mut srv.connect_with_timeout().unwrap()).unwrap(); - let intermediate_lsns: Vec = intermediate_lsns - .iter() - .map(|&lsn| u64::from(lsn).into()) - .collect(); - let expected_end_of_wal: Lsn = u64::from(expected_end_of_wal_partial).into(); - srv.kill(); - - // Check find_end_of_wal on the initial WAL - let last_segment = cfg - .wal_dir() - .read_dir() - .unwrap() - .map(|f| f.unwrap().file_name().into_string().unwrap()) - .filter(|fname| IsXLogFileName(fname)) - .max() - .unwrap(); - check_pg_waldump_end_of_wal(&cfg, &last_segment, expected_end_of_wal); - for start_lsn in intermediate_lsns - .iter() - .chain(std::iter::once(&expected_end_of_wal)) - { - // Erase all WAL before `start_lsn` to ensure it's not used by `find_end_of_wal`. - // We assume that `start_lsn` is non-decreasing. - info!( - "Checking with start_lsn={}, erasing WAL before it", - start_lsn - ); - for file in fs::read_dir(cfg.wal_dir()).unwrap().flatten() { - let fname = file.file_name().into_string().unwrap(); - if !IsXLogFileName(&fname) { - continue; - } - let (segno, _) = XLogFromFileName(&fname, WAL_SEGMENT_SIZE); - let seg_start_lsn = XLogSegNoOffsetToRecPtr(segno, 0, WAL_SEGMENT_SIZE); - if seg_start_lsn > u64::from(*start_lsn) { - continue; - } - let mut f = File::options().write(true).open(file.path()).unwrap(); - const ZEROS: [u8; WAL_SEGMENT_SIZE] = [0u8; WAL_SEGMENT_SIZE]; - f.write_all( - &ZEROS[0..min( - WAL_SEGMENT_SIZE, - (u64::from(*start_lsn) - seg_start_lsn) as usize, - )], - ) - .unwrap(); - } - check_end_of_wal(&cfg, &last_segment, *start_lsn, expected_end_of_wal); - } - } - - fn check_pg_waldump_end_of_wal( - cfg: &wal_craft::Conf, - last_segment: &str, - expected_end_of_wal: Lsn, - ) { - // Get the actual end of WAL by pg_waldump - let waldump_output = cfg - .pg_waldump("000000010000000000000001", last_segment) - .unwrap() - .stderr; - let waldump_output = std::str::from_utf8(&waldump_output).unwrap(); - let caps = match Regex::new(r"invalid record length at (.+):") - .unwrap() - .captures(waldump_output) - { - Some(caps) => caps, - None => { - error!("Unable to parse pg_waldump's stderr:\n{}", waldump_output); - panic!(); - } - }; - let waldump_wal_end = Lsn::from_str(caps.get(1).unwrap().as_str()).unwrap(); - info!( - "waldump erred on {}, expected wal end at {}", - waldump_wal_end, expected_end_of_wal - ); - assert_eq!(waldump_wal_end, expected_end_of_wal); - } - - fn check_end_of_wal( - cfg: &wal_craft::Conf, - last_segment: &str, - start_lsn: Lsn, - expected_end_of_wal: Lsn, - ) { - // Check end_of_wal on non-partial WAL segment (we treat it as fully populated) - // let wal_end = find_end_of_wal(&cfg.wal_dir(), WAL_SEGMENT_SIZE, start_lsn).unwrap(); - // info!( - // "find_end_of_wal returned wal_end={} with non-partial WAL segment", - // wal_end - // ); - // assert_eq!(wal_end, expected_end_of_wal_non_partial); - - // Rename file to partial to actually find last valid lsn, then rename it back. - fs::rename( - cfg.wal_dir().join(last_segment), - cfg.wal_dir().join(format!("{}.partial", last_segment)), - ) - .unwrap(); - let wal_end = find_end_of_wal(&cfg.wal_dir(), WAL_SEGMENT_SIZE, start_lsn).unwrap(); - info!( - "find_end_of_wal returned wal_end={} with partial WAL segment", - wal_end - ); - assert_eq!(wal_end, expected_end_of_wal); - fs::rename( - cfg.wal_dir().join(format!("{}.partial", last_segment)), - cfg.wal_dir().join(last_segment), - ) - .unwrap(); - } - - const_assert!(WAL_SEGMENT_SIZE == 16 * 1024 * 1024); - - #[test] - pub fn test_find_end_of_wal_simple() { - init_logging(); - test_end_of_wal::("test_find_end_of_wal_simple"); - } - - #[test] - pub fn test_find_end_of_wal_crossing_segment_followed_by_small_one() { - init_logging(); - test_end_of_wal::( - "test_find_end_of_wal_crossing_segment_followed_by_small_one", - ); - } - - #[test] - pub fn test_find_end_of_wal_last_crossing_segment() { - init_logging(); - test_end_of_wal::( - "test_find_end_of_wal_last_crossing_segment", - ); - } - - /// Check the math in update_next_xid - /// - /// NOTE: These checks are sensitive to the value of XID_CHECKPOINT_INTERVAL, - /// currently 1024. - #[test] - pub fn test_update_next_xid() { - let checkpoint_buf = [0u8; std::mem::size_of::()]; - let mut checkpoint = CheckPoint::decode(&checkpoint_buf).unwrap(); - - checkpoint.nextXid = FullTransactionId { value: 10 }; - assert_eq!(checkpoint.nextXid.value, 10); - - // The input XID gets rounded up to the next XID_CHECKPOINT_INTERVAL - // boundary - checkpoint.update_next_xid(100); - assert_eq!(checkpoint.nextXid.value, 1024); - - // No change - checkpoint.update_next_xid(500); - assert_eq!(checkpoint.nextXid.value, 1024); - checkpoint.update_next_xid(1023); - assert_eq!(checkpoint.nextXid.value, 1024); - - // The function returns the *next* XID, given the highest XID seen so - // far. So when we pass 1024, the nextXid gets bumped up to the next - // XID_CHECKPOINT_INTERVAL boundary. - checkpoint.update_next_xid(1024); - assert_eq!(checkpoint.nextXid.value, 2048); - } - - #[test] - pub fn test_encode_logical_message() { - let expected = [ - 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 170, 34, 166, 227, 255, - 38, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 112, 114, - 101, 102, 105, 120, 0, 109, 101, 115, 115, 97, 103, 101, - ]; - let actual = encode_logical_message("prefix", "message"); - assert_eq!(expected, actual[..]); - } -} +// If you need to craft WAL and write tests for this module, put it at wal_craft crate. diff --git a/libs/postgres_ffi/wal_craft/Cargo.toml b/libs/postgres_ffi/wal_craft/Cargo.toml index 992bf7460b..bea888b23e 100644 --- a/libs/postgres_ffi/wal_craft/Cargo.toml +++ b/libs/postgres_ffi/wal_craft/Cargo.toml @@ -15,3 +15,7 @@ postgres_ffi.workspace = true tempfile.workspace = true workspace_hack.workspace = true + +[dev-dependencies] +regex.workspace = true +utils.workspace = true diff --git a/libs/postgres_ffi/wal_craft/src/lib.rs b/libs/postgres_ffi/wal_craft/src/lib.rs index 9f3f4dc20d..d4aed88048 100644 --- a/libs/postgres_ffi/wal_craft/src/lib.rs +++ b/libs/postgres_ffi/wal_craft/src/lib.rs @@ -10,6 +10,20 @@ use std::process::Command; use std::time::{Duration, Instant}; use tempfile::{tempdir, TempDir}; +macro_rules! xlog_utils_test { + ($version:ident) => { + #[path = "."] + mod $version { + pub use postgres_ffi::$version::wal_craft_test_export::*; + #[allow(clippy::duplicate_mod)] + #[cfg(test)] + mod xlog_utils_test; + } + }; +} + +postgres_ffi::for_all_postgres_versions! { xlog_utils_test } + #[derive(Debug, Clone, PartialEq, Eq)] pub struct Conf { pub pg_version: u32, diff --git a/libs/postgres_ffi/wal_craft/src/xlog_utils_test.rs b/libs/postgres_ffi/wal_craft/src/xlog_utils_test.rs new file mode 100644 index 0000000000..6ff4c563b2 --- /dev/null +++ b/libs/postgres_ffi/wal_craft/src/xlog_utils_test.rs @@ -0,0 +1,219 @@ +//! Tests for postgres_ffi xlog_utils module. Put it here to break cyclic dependency. + +use super::*; +use crate::{error, info}; +use regex::Regex; +use std::cmp::min; +use std::fs::{self, File}; +use std::io::Write; +use std::{env, str::FromStr}; +use utils::const_assert; +use utils::lsn::Lsn; + +fn init_logging() { + let _ = env_logger::Builder::from_env(env_logger::Env::default().default_filter_or( + format!("crate=info,postgres_ffi::{PG_MAJORVERSION}::xlog_utils=trace"), + )) + .is_test(true) + .try_init(); +} + +fn test_end_of_wal(test_name: &str) { + use crate::*; + + let pg_version = PG_MAJORVERSION[1..3].parse::().unwrap(); + + // Craft some WAL + let top_path = PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("..") + .join("..") + .join(".."); + let cfg = Conf { + pg_version, + pg_distrib_dir: top_path.join("pg_install"), + datadir: top_path.join(format!("test_output/{}-{PG_MAJORVERSION}", test_name)), + }; + if cfg.datadir.exists() { + fs::remove_dir_all(&cfg.datadir).unwrap(); + } + cfg.initdb().unwrap(); + let srv = cfg.start_server().unwrap(); + let (intermediate_lsns, expected_end_of_wal_partial) = + C::craft(&mut srv.connect_with_timeout().unwrap()).unwrap(); + let intermediate_lsns: Vec = intermediate_lsns + .iter() + .map(|&lsn| u64::from(lsn).into()) + .collect(); + let expected_end_of_wal: Lsn = u64::from(expected_end_of_wal_partial).into(); + srv.kill(); + + // Check find_end_of_wal on the initial WAL + let last_segment = cfg + .wal_dir() + .read_dir() + .unwrap() + .map(|f| f.unwrap().file_name().into_string().unwrap()) + .filter(|fname| IsXLogFileName(fname)) + .max() + .unwrap(); + check_pg_waldump_end_of_wal(&cfg, &last_segment, expected_end_of_wal); + for start_lsn in intermediate_lsns + .iter() + .chain(std::iter::once(&expected_end_of_wal)) + { + // Erase all WAL before `start_lsn` to ensure it's not used by `find_end_of_wal`. + // We assume that `start_lsn` is non-decreasing. + info!( + "Checking with start_lsn={}, erasing WAL before it", + start_lsn + ); + for file in fs::read_dir(cfg.wal_dir()).unwrap().flatten() { + let fname = file.file_name().into_string().unwrap(); + if !IsXLogFileName(&fname) { + continue; + } + let (segno, _) = XLogFromFileName(&fname, WAL_SEGMENT_SIZE); + let seg_start_lsn = XLogSegNoOffsetToRecPtr(segno, 0, WAL_SEGMENT_SIZE); + if seg_start_lsn > u64::from(*start_lsn) { + continue; + } + let mut f = File::options().write(true).open(file.path()).unwrap(); + const ZEROS: [u8; WAL_SEGMENT_SIZE] = [0u8; WAL_SEGMENT_SIZE]; + f.write_all( + &ZEROS[0..min( + WAL_SEGMENT_SIZE, + (u64::from(*start_lsn) - seg_start_lsn) as usize, + )], + ) + .unwrap(); + } + check_end_of_wal(&cfg, &last_segment, *start_lsn, expected_end_of_wal); + } +} + +fn check_pg_waldump_end_of_wal( + cfg: &crate::Conf, + last_segment: &str, + expected_end_of_wal: Lsn, +) { + // Get the actual end of WAL by pg_waldump + let waldump_output = cfg + .pg_waldump("000000010000000000000001", last_segment) + .unwrap() + .stderr; + let waldump_output = std::str::from_utf8(&waldump_output).unwrap(); + let caps = match Regex::new(r"invalid record length at (.+):") + .unwrap() + .captures(waldump_output) + { + Some(caps) => caps, + None => { + error!("Unable to parse pg_waldump's stderr:\n{}", waldump_output); + panic!(); + } + }; + let waldump_wal_end = Lsn::from_str(caps.get(1).unwrap().as_str()).unwrap(); + info!( + "waldump erred on {}, expected wal end at {}", + waldump_wal_end, expected_end_of_wal + ); + assert_eq!(waldump_wal_end, expected_end_of_wal); +} + +fn check_end_of_wal( + cfg: &crate::Conf, + last_segment: &str, + start_lsn: Lsn, + expected_end_of_wal: Lsn, +) { + // Check end_of_wal on non-partial WAL segment (we treat it as fully populated) + // let wal_end = find_end_of_wal(&cfg.wal_dir(), WAL_SEGMENT_SIZE, start_lsn).unwrap(); + // info!( + // "find_end_of_wal returned wal_end={} with non-partial WAL segment", + // wal_end + // ); + // assert_eq!(wal_end, expected_end_of_wal_non_partial); + + // Rename file to partial to actually find last valid lsn, then rename it back. + fs::rename( + cfg.wal_dir().join(last_segment), + cfg.wal_dir().join(format!("{}.partial", last_segment)), + ) + .unwrap(); + let wal_end = find_end_of_wal(&cfg.wal_dir(), WAL_SEGMENT_SIZE, start_lsn).unwrap(); + info!( + "find_end_of_wal returned wal_end={} with partial WAL segment", + wal_end + ); + assert_eq!(wal_end, expected_end_of_wal); + fs::rename( + cfg.wal_dir().join(format!("{}.partial", last_segment)), + cfg.wal_dir().join(last_segment), + ) + .unwrap(); +} + +const_assert!(WAL_SEGMENT_SIZE == 16 * 1024 * 1024); + +#[test] +pub fn test_find_end_of_wal_simple() { + init_logging(); + test_end_of_wal::("test_find_end_of_wal_simple"); +} + +#[test] +pub fn test_find_end_of_wal_crossing_segment_followed_by_small_one() { + init_logging(); + test_end_of_wal::( + "test_find_end_of_wal_crossing_segment_followed_by_small_one", + ); +} + +#[test] +pub fn test_find_end_of_wal_last_crossing_segment() { + init_logging(); + test_end_of_wal::( + "test_find_end_of_wal_last_crossing_segment", + ); +} + +/// Check the math in update_next_xid +/// +/// NOTE: These checks are sensitive to the value of XID_CHECKPOINT_INTERVAL, +/// currently 1024. +#[test] +pub fn test_update_next_xid() { + let checkpoint_buf = [0u8; std::mem::size_of::()]; + let mut checkpoint = CheckPoint::decode(&checkpoint_buf).unwrap(); + + checkpoint.nextXid = FullTransactionId { value: 10 }; + assert_eq!(checkpoint.nextXid.value, 10); + + // The input XID gets rounded up to the next XID_CHECKPOINT_INTERVAL + // boundary + checkpoint.update_next_xid(100); + assert_eq!(checkpoint.nextXid.value, 1024); + + // No change + checkpoint.update_next_xid(500); + assert_eq!(checkpoint.nextXid.value, 1024); + checkpoint.update_next_xid(1023); + assert_eq!(checkpoint.nextXid.value, 1024); + + // The function returns the *next* XID, given the highest XID seen so + // far. So when we pass 1024, the nextXid gets bumped up to the next + // XID_CHECKPOINT_INTERVAL boundary. + checkpoint.update_next_xid(1024); + assert_eq!(checkpoint.nextXid.value, 2048); +} + +#[test] +pub fn test_encode_logical_message() { + let expected = [ + 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 170, 34, 166, 227, 255, + 38, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 112, 114, + 101, 102, 105, 120, 0, 109, 101, 115, 115, 97, 103, 101, + ]; + let actual = encode_logical_message("prefix", "message"); + assert_eq!(expected, actual[..]); +} diff --git a/libs/remote_storage/src/lib.rs b/libs/remote_storage/src/lib.rs index e0cc3ca543..0e9c237e1e 100644 --- a/libs/remote_storage/src/lib.rs +++ b/libs/remote_storage/src/lib.rs @@ -70,6 +70,14 @@ impl RemotePath { pub fn join(&self, segment: &Path) -> Self { Self(self.0.join(segment)) } + + pub fn get_path(&self) -> &PathBuf { + &self.0 + } + + pub fn extension(&self) -> Option<&str> { + self.0.extension()?.to_str() + } } /// Storage (potentially remote) API to manage its state. @@ -86,6 +94,19 @@ pub trait RemoteStorage: Send + Sync + 'static { prefix: Option<&RemotePath>, ) -> Result, DownloadError>; + /// Lists all files in directory "recursively" + /// (not really recursively, because AWS has a flat namespace) + /// Note: This is subtely different than list_prefixes, + /// because it is for listing files instead of listing + /// names sharing common prefixes. + /// For example, + /// list_files("foo/bar") = ["foo/bar/cat123.txt", + /// "foo/bar/cat567.txt", "foo/bar/dog123.txt", "foo/bar/dog456.txt"] + /// whereas, + /// list_prefixes("foo/bar/") = ["cat", "dog"] + /// See `test_real_s3.rs` for more details. + async fn list_files(&self, folder: Option<&RemotePath>) -> anyhow::Result>; + /// Streams the local file contents into remote into the remote storage entry. async fn upload( &self, @@ -111,6 +132,8 @@ pub trait RemoteStorage: Send + Sync + 'static { ) -> Result; async fn delete(&self, path: &RemotePath) -> anyhow::Result<()>; + + async fn delete_objects<'a>(&self, paths: &'a [RemotePath]) -> anyhow::Result<()>; } pub struct Download { @@ -172,6 +195,14 @@ impl GenericRemoteStorage { } } + pub async fn list_files(&self, folder: Option<&RemotePath>) -> anyhow::Result> { + match self { + Self::LocalFs(s) => s.list_files(folder).await, + Self::AwsS3(s) => s.list_files(folder).await, + Self::Unreliable(s) => s.list_files(folder).await, + } + } + pub async fn upload( &self, from: impl io::AsyncRead + Unpin + Send + Sync + 'static, @@ -223,6 +254,14 @@ impl GenericRemoteStorage { Self::Unreliable(s) => s.delete(path).await, } } + + pub async fn delete_objects<'a>(&self, paths: &'a [RemotePath]) -> anyhow::Result<()> { + match self { + Self::LocalFs(s) => s.delete_objects(paths).await, + Self::AwsS3(s) => s.delete_objects(paths).await, + Self::Unreliable(s) => s.delete_objects(paths).await, + } + } } impl GenericRemoteStorage { diff --git a/libs/remote_storage/src/local_fs.rs b/libs/remote_storage/src/local_fs.rs index c081a6d361..ca5fbd5de5 100644 --- a/libs/remote_storage/src/local_fs.rs +++ b/libs/remote_storage/src/local_fs.rs @@ -17,7 +17,7 @@ use tokio::{ io::{self, AsyncReadExt, AsyncSeekExt, AsyncWriteExt}, }; use tracing::*; -use utils::crashsafe::path_with_suffix_extension; +use utils::{crashsafe::path_with_suffix_extension, fs_ext::is_directory_empty}; use crate::{Download, DownloadError, RemotePath}; @@ -48,6 +48,14 @@ impl LocalFs { Ok(Self { storage_root }) } + // mirrors S3Bucket::s3_object_to_relative_path + fn local_file_to_relative_path(&self, key: PathBuf) -> RemotePath { + let relative_path = key + .strip_prefix(&self.storage_root) + .expect("relative path must contain storage_root as prefix"); + RemotePath(relative_path.into()) + } + async fn read_storage_metadata( &self, file_path: &Path, @@ -101,19 +109,63 @@ impl RemoteStorage for LocalFs { Some(prefix) => Cow::Owned(prefix.with_base(&self.storage_root)), None => Cow::Borrowed(&self.storage_root), }; - Ok(get_all_files(path.as_ref(), false) + + let prefixes_to_filter = get_all_files(path.as_ref(), false) .await - .map_err(DownloadError::Other)? - .into_iter() - .map(|path| { - path.strip_prefix(&self.storage_root) - .context("Failed to strip preifix") + .map_err(DownloadError::Other)?; + + let mut prefixes = Vec::with_capacity(prefixes_to_filter.len()); + + // filter out empty directories to mirror s3 behavior. + for prefix in prefixes_to_filter { + if prefix.is_dir() + && is_directory_empty(&prefix) + .await + .map_err(DownloadError::Other)? + { + continue; + } + + prefixes.push( + prefix + .strip_prefix(&self.storage_root) + .context("Failed to strip prefix") .and_then(RemotePath::new) .expect( "We list files for storage root, hence should be able to remote the prefix", - ) - }) - .collect()) + ), + ) + } + + Ok(prefixes) + } + + // recursively lists all files in a directory, + // mirroring the `list_files` for `s3_bucket` + async fn list_files(&self, folder: Option<&RemotePath>) -> anyhow::Result> { + let full_path = match folder { + Some(folder) => folder.with_base(&self.storage_root), + None => self.storage_root.clone(), + }; + let mut files = vec![]; + let mut directory_queue = vec![full_path.clone()]; + + while !directory_queue.is_empty() { + let cur_folder = directory_queue + .pop() + .expect("queue cannot be empty: we just checked"); + let mut entries = fs::read_dir(cur_folder.clone()).await?; + while let Some(entry) = entries.next_entry().await? { + let file_name: PathBuf = entry.file_name().into(); + let full_file_name = cur_folder.clone().join(&file_name); + let file_remote_path = self.local_file_to_relative_path(full_file_name.clone()); + files.push(file_remote_path.clone()); + if full_file_name.is_dir() { + directory_queue.push(full_file_name); + } + } + } + Ok(files) } async fn upload( @@ -291,11 +343,25 @@ impl RemoteStorage for LocalFs { async fn delete(&self, path: &RemotePath) -> anyhow::Result<()> { let file_path = path.with_base(&self.storage_root); - if file_path.exists() && file_path.is_file() { - Ok(fs::remove_file(file_path).await?) - } else { - bail!("File {file_path:?} either does not exist or is not a file") + if !file_path.exists() { + // See https://docs.aws.amazon.com/AmazonS3/latest/API/API_DeleteObject.html + // > If there isn't a null version, Amazon S3 does not remove any objects but will still respond that the command was successful. + return Ok(()); } + + if !file_path.is_file() { + anyhow::bail!("{file_path:?} is not a file"); + } + Ok(fs::remove_file(file_path) + .await + .map_err(|e| anyhow::anyhow!(e))?) + } + + async fn delete_objects<'a>(&self, paths: &'a [RemotePath]) -> anyhow::Result<()> { + for path in paths { + self.delete(path).await? + } + Ok(()) } } @@ -320,7 +386,7 @@ where let file_type = dir_entry.file_type().await?; let entry_path = dir_entry.path(); if file_type.is_symlink() { - debug!("{entry_path:?} us a symlink, skipping") + debug!("{entry_path:?} is a symlink, skipping") } else if file_type.is_dir() { if recursive { paths.extend(get_all_files(&entry_path, true).await?.into_iter()) @@ -595,15 +661,11 @@ mod fs_tests { storage.delete(&upload_target).await?; assert!(storage.list().await?.is_empty()); - match storage.delete(&upload_target).await { - Ok(()) => panic!("Should not allow deleting non-existing storage files"), - Err(e) => { - let error_string = e.to_string(); - assert!(error_string.contains("does not exist")); - let expected_path = upload_target.with_base(&storage.storage_root); - assert!(error_string.contains(expected_path.to_str().unwrap())); - } - } + storage + .delete(&upload_target) + .await + .expect("Should allow deleting non-existing storage files"); + Ok(()) } diff --git a/libs/remote_storage/src/s3_bucket.rs b/libs/remote_storage/src/s3_bucket.rs index 0be8c72fe0..43d818dfb9 100644 --- a/libs/remote_storage/src/s3_bucket.rs +++ b/libs/remote_storage/src/s3_bucket.rs @@ -17,6 +17,7 @@ use aws_sdk_s3::{ error::SdkError, operation::get_object::GetObjectError, primitives::ByteStream, + types::{Delete, ObjectIdentifier}, Client, }; use aws_smithy_http::body::SdkBody; @@ -33,6 +34,8 @@ use crate::{ Download, DownloadError, RemotePath, RemoteStorage, S3Config, REMOTE_STORAGE_PREFIX_SEPARATOR, }; +const MAX_DELETE_OBJECTS_REQUEST_SIZE: usize = 1000; + pub(super) mod metrics { use metrics::{register_int_counter_vec, IntCounterVec}; use once_cell::sync::Lazy; @@ -81,12 +84,24 @@ pub(super) mod metrics { .inc(); } + pub fn inc_delete_objects(count: u64) { + S3_REQUESTS_COUNT + .with_label_values(&["delete_object"]) + .inc_by(count); + } + pub fn inc_delete_object_fail() { S3_REQUESTS_FAIL_COUNT .with_label_values(&["delete_object"]) .inc(); } + pub fn inc_delete_objects_fail(count: u64) { + S3_REQUESTS_FAIL_COUNT + .with_label_values(&["delete_object"]) + .inc_by(count); + } + pub fn inc_list_objects() { S3_REQUESTS_COUNT.with_label_values(&["list_objects"]).inc(); } @@ -332,6 +347,51 @@ impl RemoteStorage for S3Bucket { Ok(document_keys) } + /// See the doc for `RemoteStorage::list_files` + async fn list_files(&self, folder: Option<&RemotePath>) -> anyhow::Result> { + let folder_name = folder + .map(|p| self.relative_path_to_s3_object(p)) + .or_else(|| self.prefix_in_bucket.clone()); + + // AWS may need to break the response into several parts + let mut continuation_token = None; + let mut all_files = vec![]; + loop { + let _guard = self + .concurrency_limiter + .acquire() + .await + .context("Concurrency limiter semaphore got closed during S3 list_files")?; + metrics::inc_list_objects(); + + let response = self + .client + .list_objects_v2() + .bucket(self.bucket_name.clone()) + .set_prefix(folder_name.clone()) + .set_continuation_token(continuation_token) + .set_max_keys(self.max_keys_per_list_response) + .send() + .await + .map_err(|e| { + metrics::inc_list_objects_fail(); + e + }) + .context("Failed to list files in S3 bucket")?; + + for object in response.contents().unwrap_or_default() { + let object_path = object.key().expect("response does not contain a key"); + let remote_path = self.s3_object_to_relative_path(object_path); + all_files.push(remote_path); + } + match response.next_continuation_token { + Some(new_token) => continuation_token = Some(new_token), + None => break, + } + } + Ok(all_files) + } + async fn upload( &self, from: impl io::AsyncRead + Unpin + Send + Sync + 'static, @@ -396,6 +456,50 @@ impl RemoteStorage for S3Bucket { }) .await } + async fn delete_objects<'a>(&self, paths: &'a [RemotePath]) -> anyhow::Result<()> { + let _guard = self + .concurrency_limiter + .acquire() + .await + .context("Concurrency limiter semaphore got closed during S3 delete")?; + + let mut delete_objects = Vec::with_capacity(paths.len()); + for path in paths { + let obj_id = ObjectIdentifier::builder() + .set_key(Some(self.relative_path_to_s3_object(path))) + .build(); + delete_objects.push(obj_id); + } + + for chunk in delete_objects.chunks(MAX_DELETE_OBJECTS_REQUEST_SIZE) { + metrics::inc_delete_objects(chunk.len() as u64); + + let resp = self + .client + .delete_objects() + .bucket(self.bucket_name.clone()) + .delete(Delete::builder().set_objects(Some(chunk.to_vec())).build()) + .send() + .await; + + match resp { + Ok(resp) => { + if let Some(errors) = resp.errors { + metrics::inc_delete_objects_fail(errors.len() as u64); + return Err(anyhow::format_err!( + "Failed to delete {} objects", + errors.len() + )); + } + } + Err(e) => { + metrics::inc_delete_objects_fail(chunk.len() as u64); + return Err(e.into()); + } + } + } + Ok(()) + } async fn delete(&self, path: &RemotePath) -> anyhow::Result<()> { let _guard = self diff --git a/libs/remote_storage/src/simulate_failures.rs b/libs/remote_storage/src/simulate_failures.rs index cb40859831..c46ca14ace 100644 --- a/libs/remote_storage/src/simulate_failures.rs +++ b/libs/remote_storage/src/simulate_failures.rs @@ -24,6 +24,7 @@ enum RemoteOp { Upload(RemotePath), Download(RemotePath), Delete(RemotePath), + DeleteObjects(Vec), } impl UnreliableWrapper { @@ -82,6 +83,11 @@ impl RemoteStorage for UnreliableWrapper { self.inner.list_prefixes(prefix).await } + async fn list_files(&self, folder: Option<&RemotePath>) -> anyhow::Result> { + self.attempt(RemoteOp::ListPrefixes(folder.cloned()))?; + self.inner.list_files(folder).await + } + async fn upload( &self, data: impl tokio::io::AsyncRead + Unpin + Send + Sync + 'static, @@ -119,4 +125,21 @@ impl RemoteStorage for UnreliableWrapper { self.attempt(RemoteOp::Delete(path.clone()))?; self.inner.delete(path).await } + + async fn delete_objects<'a>(&self, paths: &'a [RemotePath]) -> anyhow::Result<()> { + self.attempt(RemoteOp::DeleteObjects(paths.to_vec()))?; + let mut error_counter = 0; + for path in paths { + if (self.delete(path).await).is_err() { + error_counter += 1; + } + } + if error_counter > 0 { + return Err(anyhow::anyhow!( + "failed to delete {} objects", + error_counter + )); + } + Ok(()) + } } diff --git a/libs/remote_storage/tests/pagination_tests.rs b/libs/remote_storage/tests/pagination_tests.rs deleted file mode 100644 index 86a6888f98..0000000000 --- a/libs/remote_storage/tests/pagination_tests.rs +++ /dev/null @@ -1,274 +0,0 @@ -use std::collections::HashSet; -use std::env; -use std::num::{NonZeroU32, NonZeroUsize}; -use std::ops::ControlFlow; -use std::path::{Path, PathBuf}; -use std::sync::Arc; -use std::time::UNIX_EPOCH; - -use anyhow::Context; -use remote_storage::{ - GenericRemoteStorage, RemotePath, RemoteStorageConfig, RemoteStorageKind, S3Config, -}; -use test_context::{test_context, AsyncTestContext}; -use tokio::task::JoinSet; -use tracing::{debug, error, info}; - -const ENABLE_REAL_S3_REMOTE_STORAGE_ENV_VAR_NAME: &str = "ENABLE_REAL_S3_REMOTE_STORAGE"; - -/// Tests that S3 client can list all prefixes, even if the response come paginated and requires multiple S3 queries. -/// Uses real S3 and requires [`ENABLE_REAL_S3_REMOTE_STORAGE_ENV_VAR_NAME`] and related S3 cred env vars specified. -/// See the client creation in [`create_s3_client`] for details on the required env vars. -/// If real S3 tests are disabled, the test passes, skipping any real test run: currently, there's no way to mark the test ignored in runtime with the -/// deafult test framework, see https://github.com/rust-lang/rust/issues/68007 for details. -/// -/// First, the test creates a set of S3 objects with keys `/${random_prefix_part}/${base_prefix_str}/sub_prefix_${i}/blob_${i}` in [`upload_s3_data`] -/// where -/// * `random_prefix_part` is set for the entire S3 client during the S3 client creation in [`create_s3_client`], to avoid multiple test runs interference -/// * `base_prefix_str` is a common prefix to use in the client requests: we would want to ensure that the client is able to list nested prefixes inside the bucket -/// -/// Then, verifies that the client does return correct prefixes when queried: -/// * with no prefix, it lists everything after its `${random_prefix_part}/` — that should be `${base_prefix_str}` value only -/// * with `${base_prefix_str}/` prefix, it lists every `sub_prefix_${i}` -/// -/// With the real S3 enabled and `#[cfg(test)]` Rust configuration used, the S3 client test adds a `max-keys` param to limit the response keys. -/// This way, we are able to test the pagination implicitly, by ensuring all results are returned from the remote storage and avoid uploading too many blobs to S3, -/// since current default AWS S3 pagination limit is 1000. -/// (see https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListObjectsV2.html#API_ListObjectsV2_RequestSyntax) -/// -/// Lastly, the test attempts to clean up and remove all uploaded S3 files. -/// If any errors appear during the clean up, they get logged, but the test is not failed or stopped until clean up is finished. -#[test_context(MaybeEnabledS3)] -#[tokio::test] -async fn s3_pagination_should_work(ctx: &mut MaybeEnabledS3) -> anyhow::Result<()> { - let ctx = match ctx { - MaybeEnabledS3::Enabled(ctx) => ctx, - MaybeEnabledS3::Disabled => return Ok(()), - MaybeEnabledS3::UploadsFailed(e, _) => anyhow::bail!("S3 init failed: {e:?}"), - }; - - let test_client = Arc::clone(&ctx.client_with_excessive_pagination); - let expected_remote_prefixes = ctx.remote_prefixes.clone(); - - let base_prefix = - RemotePath::new(Path::new(ctx.base_prefix_str)).context("common_prefix construction")?; - let root_remote_prefixes = test_client - .list_prefixes(None) - .await - .context("client list root prefixes failure")? - .into_iter() - .collect::>(); - assert_eq!( - root_remote_prefixes, HashSet::from([base_prefix.clone()]), - "remote storage root prefixes list mismatches with the uploads. Returned prefixes: {root_remote_prefixes:?}" - ); - - let nested_remote_prefixes = test_client - .list_prefixes(Some(&base_prefix)) - .await - .context("client list nested prefixes failure")? - .into_iter() - .collect::>(); - let remote_only_prefixes = nested_remote_prefixes - .difference(&expected_remote_prefixes) - .collect::>(); - let missing_uploaded_prefixes = expected_remote_prefixes - .difference(&nested_remote_prefixes) - .collect::>(); - assert_eq!( - remote_only_prefixes.len() + missing_uploaded_prefixes.len(), 0, - "remote storage nested prefixes list mismatches with the uploads. Remote only prefixes: {remote_only_prefixes:?}, missing uploaded prefixes: {missing_uploaded_prefixes:?}", - ); - - Ok(()) -} - -enum MaybeEnabledS3 { - Enabled(S3WithTestBlobs), - Disabled, - UploadsFailed(anyhow::Error, S3WithTestBlobs), -} - -struct S3WithTestBlobs { - client_with_excessive_pagination: Arc, - base_prefix_str: &'static str, - remote_prefixes: HashSet, - remote_blobs: HashSet, -} - -#[async_trait::async_trait] -impl AsyncTestContext for MaybeEnabledS3 { - async fn setup() -> Self { - utils::logging::init( - utils::logging::LogFormat::Test, - utils::logging::TracingErrorLayerEnablement::Disabled, - ) - .expect("logging init failed"); - if env::var(ENABLE_REAL_S3_REMOTE_STORAGE_ENV_VAR_NAME).is_err() { - info!( - "`{}` env variable is not set, skipping the test", - ENABLE_REAL_S3_REMOTE_STORAGE_ENV_VAR_NAME - ); - return Self::Disabled; - } - - let max_keys_in_list_response = 10; - let upload_tasks_count = 1 + (2 * usize::try_from(max_keys_in_list_response).unwrap()); - - let client_with_excessive_pagination = create_s3_client(max_keys_in_list_response) - .context("S3 client creation") - .expect("S3 client creation failed"); - - let base_prefix_str = "test/"; - match upload_s3_data( - &client_with_excessive_pagination, - base_prefix_str, - upload_tasks_count, - ) - .await - { - ControlFlow::Continue(uploads) => { - info!("Remote objects created successfully"); - Self::Enabled(S3WithTestBlobs { - client_with_excessive_pagination, - base_prefix_str, - remote_prefixes: uploads.prefixes, - remote_blobs: uploads.blobs, - }) - } - ControlFlow::Break(uploads) => Self::UploadsFailed( - anyhow::anyhow!("One or multiple blobs failed to upload to S3"), - S3WithTestBlobs { - client_with_excessive_pagination, - base_prefix_str, - remote_prefixes: uploads.prefixes, - remote_blobs: uploads.blobs, - }, - ), - } - } - - async fn teardown(self) { - match self { - Self::Disabled => {} - Self::Enabled(ctx) | Self::UploadsFailed(_, ctx) => { - cleanup(&ctx.client_with_excessive_pagination, ctx.remote_blobs).await; - } - } - } -} - -fn create_s3_client(max_keys_per_list_response: i32) -> anyhow::Result> { - let remote_storage_s3_bucket = env::var("REMOTE_STORAGE_S3_BUCKET") - .context("`REMOTE_STORAGE_S3_BUCKET` env var is not set, but real S3 tests are enabled")?; - let remote_storage_s3_region = env::var("REMOTE_STORAGE_S3_REGION") - .context("`REMOTE_STORAGE_S3_REGION` env var is not set, but real S3 tests are enabled")?; - let random_prefix_part = std::time::SystemTime::now() - .duration_since(UNIX_EPOCH) - .context("random s3 test prefix part calculation")? - .as_millis(); - let remote_storage_config = RemoteStorageConfig { - max_concurrent_syncs: NonZeroUsize::new(100).unwrap(), - max_sync_errors: NonZeroU32::new(5).unwrap(), - storage: RemoteStorageKind::AwsS3(S3Config { - bucket_name: remote_storage_s3_bucket, - bucket_region: remote_storage_s3_region, - prefix_in_bucket: Some(format!("pagination_should_work_test_{random_prefix_part}/")), - endpoint: None, - concurrency_limit: NonZeroUsize::new(100).unwrap(), - max_keys_per_list_response: Some(max_keys_per_list_response), - }), - }; - Ok(Arc::new( - GenericRemoteStorage::from_config(&remote_storage_config).context("remote storage init")?, - )) -} - -struct Uploads { - prefixes: HashSet, - blobs: HashSet, -} - -async fn upload_s3_data( - client: &Arc, - base_prefix_str: &'static str, - upload_tasks_count: usize, -) -> ControlFlow { - info!("Creating {upload_tasks_count} S3 files"); - let mut upload_tasks = JoinSet::new(); - for i in 1..upload_tasks_count + 1 { - let task_client = Arc::clone(client); - upload_tasks.spawn(async move { - let prefix = PathBuf::from(format!("{base_prefix_str}/sub_prefix_{i}/")); - let blob_prefix = RemotePath::new(&prefix) - .with_context(|| format!("{prefix:?} to RemotePath conversion"))?; - let blob_path = blob_prefix.join(Path::new(&format!("blob_{i}"))); - debug!("Creating remote item {i} at path {blob_path:?}"); - - let data = format!("remote blob data {i}").into_bytes(); - let data_len = data.len(); - task_client - .upload(std::io::Cursor::new(data), data_len, &blob_path, None) - .await?; - - Ok::<_, anyhow::Error>((blob_prefix, blob_path)) - }); - } - - let mut upload_tasks_failed = false; - let mut uploaded_prefixes = HashSet::with_capacity(upload_tasks_count); - let mut uploaded_blobs = HashSet::with_capacity(upload_tasks_count); - while let Some(task_run_result) = upload_tasks.join_next().await { - match task_run_result - .context("task join failed") - .and_then(|task_result| task_result.context("upload task failed")) - { - Ok((upload_prefix, upload_path)) => { - uploaded_prefixes.insert(upload_prefix); - uploaded_blobs.insert(upload_path); - } - Err(e) => { - error!("Upload task failed: {e:?}"); - upload_tasks_failed = true; - } - } - } - - let uploads = Uploads { - prefixes: uploaded_prefixes, - blobs: uploaded_blobs, - }; - if upload_tasks_failed { - ControlFlow::Break(uploads) - } else { - ControlFlow::Continue(uploads) - } -} - -async fn cleanup(client: &Arc, objects_to_delete: HashSet) { - info!( - "Removing {} objects from the remote storage during cleanup", - objects_to_delete.len() - ); - let mut delete_tasks = JoinSet::new(); - for object_to_delete in objects_to_delete { - let task_client = Arc::clone(client); - delete_tasks.spawn(async move { - debug!("Deleting remote item at path {object_to_delete:?}"); - task_client - .delete(&object_to_delete) - .await - .with_context(|| format!("{object_to_delete:?} removal")) - }); - } - - while let Some(task_run_result) = delete_tasks.join_next().await { - match task_run_result { - Ok(task_result) => match task_result { - Ok(()) => {} - Err(e) => error!("Delete task failed: {e:?}"), - }, - Err(join_err) => error!("Delete task did not finish correctly: {join_err}"), - } - } -} diff --git a/libs/remote_storage/tests/test_real_s3.rs b/libs/remote_storage/tests/test_real_s3.rs new file mode 100644 index 0000000000..0917bab39c --- /dev/null +++ b/libs/remote_storage/tests/test_real_s3.rs @@ -0,0 +1,542 @@ +use std::collections::HashSet; +use std::env; +use std::num::{NonZeroU32, NonZeroUsize}; +use std::ops::ControlFlow; +use std::path::{Path, PathBuf}; +use std::sync::Arc; +use std::time::UNIX_EPOCH; + +use anyhow::Context; +use once_cell::sync::OnceCell; +use remote_storage::{ + GenericRemoteStorage, RemotePath, RemoteStorageConfig, RemoteStorageKind, S3Config, +}; +use test_context::{test_context, AsyncTestContext}; +use tokio::task::JoinSet; +use tracing::{debug, error, info}; + +static LOGGING_DONE: OnceCell<()> = OnceCell::new(); + +const ENABLE_REAL_S3_REMOTE_STORAGE_ENV_VAR_NAME: &str = "ENABLE_REAL_S3_REMOTE_STORAGE"; + +const BASE_PREFIX: &str = "test/"; + +/// Tests that S3 client can list all prefixes, even if the response come paginated and requires multiple S3 queries. +/// Uses real S3 and requires [`ENABLE_REAL_S3_REMOTE_STORAGE_ENV_VAR_NAME`] and related S3 cred env vars specified. +/// See the client creation in [`create_s3_client`] for details on the required env vars. +/// If real S3 tests are disabled, the test passes, skipping any real test run: currently, there's no way to mark the test ignored in runtime with the +/// deafult test framework, see https://github.com/rust-lang/rust/issues/68007 for details. +/// +/// First, the test creates a set of S3 objects with keys `/${random_prefix_part}/${base_prefix_str}/sub_prefix_${i}/blob_${i}` in [`upload_s3_data`] +/// where +/// * `random_prefix_part` is set for the entire S3 client during the S3 client creation in [`create_s3_client`], to avoid multiple test runs interference +/// * `base_prefix_str` is a common prefix to use in the client requests: we would want to ensure that the client is able to list nested prefixes inside the bucket +/// +/// Then, verifies that the client does return correct prefixes when queried: +/// * with no prefix, it lists everything after its `${random_prefix_part}/` — that should be `${base_prefix_str}` value only +/// * with `${base_prefix_str}/` prefix, it lists every `sub_prefix_${i}` +/// +/// With the real S3 enabled and `#[cfg(test)]` Rust configuration used, the S3 client test adds a `max-keys` param to limit the response keys. +/// This way, we are able to test the pagination implicitly, by ensuring all results are returned from the remote storage and avoid uploading too many blobs to S3, +/// since current default AWS S3 pagination limit is 1000. +/// (see https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListObjectsV2.html#API_ListObjectsV2_RequestSyntax) +/// +/// Lastly, the test attempts to clean up and remove all uploaded S3 files. +/// If any errors appear during the clean up, they get logged, but the test is not failed or stopped until clean up is finished. +#[test_context(MaybeEnabledS3WithTestBlobs)] +#[tokio::test] +async fn s3_pagination_should_work(ctx: &mut MaybeEnabledS3WithTestBlobs) -> anyhow::Result<()> { + let ctx = match ctx { + MaybeEnabledS3WithTestBlobs::Enabled(ctx) => ctx, + MaybeEnabledS3WithTestBlobs::Disabled => return Ok(()), + MaybeEnabledS3WithTestBlobs::UploadsFailed(e, _) => anyhow::bail!("S3 init failed: {e:?}"), + }; + + let test_client = Arc::clone(&ctx.enabled.client); + let expected_remote_prefixes = ctx.remote_prefixes.clone(); + + let base_prefix = RemotePath::new(Path::new(ctx.enabled.base_prefix)) + .context("common_prefix construction")?; + let root_remote_prefixes = test_client + .list_prefixes(None) + .await + .context("client list root prefixes failure")? + .into_iter() + .collect::>(); + assert_eq!( + root_remote_prefixes, HashSet::from([base_prefix.clone()]), + "remote storage root prefixes list mismatches with the uploads. Returned prefixes: {root_remote_prefixes:?}" + ); + + let nested_remote_prefixes = test_client + .list_prefixes(Some(&base_prefix)) + .await + .context("client list nested prefixes failure")? + .into_iter() + .collect::>(); + let remote_only_prefixes = nested_remote_prefixes + .difference(&expected_remote_prefixes) + .collect::>(); + let missing_uploaded_prefixes = expected_remote_prefixes + .difference(&nested_remote_prefixes) + .collect::>(); + assert_eq!( + remote_only_prefixes.len() + missing_uploaded_prefixes.len(), 0, + "remote storage nested prefixes list mismatches with the uploads. Remote only prefixes: {remote_only_prefixes:?}, missing uploaded prefixes: {missing_uploaded_prefixes:?}", + ); + + Ok(()) +} + +/// Tests that S3 client can list all files in a folder, even if the response comes paginated and requirees multiple S3 queries. +/// Uses real S3 and requires [`ENABLE_REAL_S3_REMOTE_STORAGE_ENV_VAR_NAME`] and related S3 cred env vars specified. Test will skip real code and pass if env vars not set. +/// See `s3_pagination_should_work` for more information. +/// +/// First, create a set of S3 objects with keys `random_prefix/folder{j}/blob_{i}.txt` in [`upload_s3_data`] +/// Then performs the following queries: +/// 1. `list_files(None)`. This should return all files `random_prefix/folder{j}/blob_{i}.txt` +/// 2. `list_files("folder1")`. This should return all files `random_prefix/folder1/blob_{i}.txt` +#[test_context(MaybeEnabledS3WithSimpleTestBlobs)] +#[tokio::test] +async fn s3_list_files_works(ctx: &mut MaybeEnabledS3WithSimpleTestBlobs) -> anyhow::Result<()> { + let ctx = match ctx { + MaybeEnabledS3WithSimpleTestBlobs::Enabled(ctx) => ctx, + MaybeEnabledS3WithSimpleTestBlobs::Disabled => return Ok(()), + MaybeEnabledS3WithSimpleTestBlobs::UploadsFailed(e, _) => { + anyhow::bail!("S3 init failed: {e:?}") + } + }; + let test_client = Arc::clone(&ctx.enabled.client); + let base_prefix = + RemotePath::new(Path::new("folder1")).context("common_prefix construction")?; + let root_files = test_client + .list_files(None) + .await + .context("client list root files failure")? + .into_iter() + .collect::>(); + assert_eq!( + root_files, + ctx.remote_blobs.clone(), + "remote storage list_files on root mismatches with the uploads." + ); + let nested_remote_files = test_client + .list_files(Some(&base_prefix)) + .await + .context("client list nested files failure")? + .into_iter() + .collect::>(); + let trim_remote_blobs: HashSet<_> = ctx + .remote_blobs + .iter() + .map(|x| x.get_path().to_str().expect("must be valid name")) + .filter(|x| x.starts_with("folder1")) + .map(|x| RemotePath::new(Path::new(x)).expect("must be valid name")) + .collect(); + assert_eq!( + nested_remote_files, trim_remote_blobs, + "remote storage list_files on subdirrectory mismatches with the uploads." + ); + Ok(()) +} + +#[test_context(MaybeEnabledS3)] +#[tokio::test] +async fn s3_delete_non_exising_works(ctx: &mut MaybeEnabledS3) -> anyhow::Result<()> { + let ctx = match ctx { + MaybeEnabledS3::Enabled(ctx) => ctx, + MaybeEnabledS3::Disabled => return Ok(()), + }; + + let path = RemotePath::new(&PathBuf::from(format!( + "{}/for_sure_there_is_nothing_there_really", + ctx.base_prefix, + ))) + .with_context(|| "RemotePath conversion")?; + + ctx.client.delete(&path).await.expect("should succeed"); + + Ok(()) +} + +#[test_context(MaybeEnabledS3)] +#[tokio::test] +async fn s3_delete_objects_works(ctx: &mut MaybeEnabledS3) -> anyhow::Result<()> { + let ctx = match ctx { + MaybeEnabledS3::Enabled(ctx) => ctx, + MaybeEnabledS3::Disabled => return Ok(()), + }; + + let path1 = RemotePath::new(&PathBuf::from(format!("{}/path1", ctx.base_prefix,))) + .with_context(|| "RemotePath conversion")?; + + let path2 = RemotePath::new(&PathBuf::from(format!("{}/path2", ctx.base_prefix,))) + .with_context(|| "RemotePath conversion")?; + + let path3 = RemotePath::new(&PathBuf::from(format!("{}/path3", ctx.base_prefix,))) + .with_context(|| "RemotePath conversion")?; + + let data1 = "remote blob data1".as_bytes(); + let data1_len = data1.len(); + let data2 = "remote blob data2".as_bytes(); + let data2_len = data2.len(); + let data3 = "remote blob data3".as_bytes(); + let data3_len = data3.len(); + ctx.client + .upload(std::io::Cursor::new(data1), data1_len, &path1, None) + .await?; + + ctx.client + .upload(std::io::Cursor::new(data2), data2_len, &path2, None) + .await?; + + ctx.client + .upload(std::io::Cursor::new(data3), data3_len, &path3, None) + .await?; + + ctx.client.delete_objects(&[path1, path2]).await?; + + let prefixes = ctx.client.list_prefixes(None).await?; + + assert_eq!(prefixes.len(), 1); + + ctx.client.delete_objects(&[path3]).await?; + + Ok(()) +} + +fn ensure_logging_ready() { + LOGGING_DONE.get_or_init(|| { + utils::logging::init( + utils::logging::LogFormat::Test, + utils::logging::TracingErrorLayerEnablement::Disabled, + ) + .expect("logging init failed"); + }); +} + +struct EnabledS3 { + client: Arc, + base_prefix: &'static str, +} + +impl EnabledS3 { + async fn setup(max_keys_in_list_response: Option) -> Self { + let client = create_s3_client(max_keys_in_list_response) + .context("S3 client creation") + .expect("S3 client creation failed"); + + EnabledS3 { + client, + base_prefix: BASE_PREFIX, + } + } +} + +enum MaybeEnabledS3 { + Enabled(EnabledS3), + Disabled, +} + +#[async_trait::async_trait] +impl AsyncTestContext for MaybeEnabledS3 { + async fn setup() -> Self { + ensure_logging_ready(); + + if env::var(ENABLE_REAL_S3_REMOTE_STORAGE_ENV_VAR_NAME).is_err() { + info!( + "`{}` env variable is not set, skipping the test", + ENABLE_REAL_S3_REMOTE_STORAGE_ENV_VAR_NAME + ); + return Self::Disabled; + } + + Self::Enabled(EnabledS3::setup(None).await) + } +} + +enum MaybeEnabledS3WithTestBlobs { + Enabled(S3WithTestBlobs), + Disabled, + UploadsFailed(anyhow::Error, S3WithTestBlobs), +} + +struct S3WithTestBlobs { + enabled: EnabledS3, + remote_prefixes: HashSet, + remote_blobs: HashSet, +} + +#[async_trait::async_trait] +impl AsyncTestContext for MaybeEnabledS3WithTestBlobs { + async fn setup() -> Self { + ensure_logging_ready(); + if env::var(ENABLE_REAL_S3_REMOTE_STORAGE_ENV_VAR_NAME).is_err() { + info!( + "`{}` env variable is not set, skipping the test", + ENABLE_REAL_S3_REMOTE_STORAGE_ENV_VAR_NAME + ); + return Self::Disabled; + } + + let max_keys_in_list_response = 10; + let upload_tasks_count = 1 + (2 * usize::try_from(max_keys_in_list_response).unwrap()); + + let enabled = EnabledS3::setup(Some(max_keys_in_list_response)).await; + + match upload_s3_data(&enabled.client, enabled.base_prefix, upload_tasks_count).await { + ControlFlow::Continue(uploads) => { + info!("Remote objects created successfully"); + + Self::Enabled(S3WithTestBlobs { + enabled, + remote_prefixes: uploads.prefixes, + remote_blobs: uploads.blobs, + }) + } + ControlFlow::Break(uploads) => Self::UploadsFailed( + anyhow::anyhow!("One or multiple blobs failed to upload to S3"), + S3WithTestBlobs { + enabled, + remote_prefixes: uploads.prefixes, + remote_blobs: uploads.blobs, + }, + ), + } + } + + async fn teardown(self) { + match self { + Self::Disabled => {} + Self::Enabled(ctx) | Self::UploadsFailed(_, ctx) => { + cleanup(&ctx.enabled.client, ctx.remote_blobs).await; + } + } + } +} + +// NOTE: the setups for the list_prefixes test and the list_files test are very similar +// However, they are not idential. The list_prefixes function is concerned with listing prefixes, +// whereas the list_files function is concerned with listing files. +// See `RemoteStorage::list_files` documentation for more details +enum MaybeEnabledS3WithSimpleTestBlobs { + Enabled(S3WithSimpleTestBlobs), + Disabled, + UploadsFailed(anyhow::Error, S3WithSimpleTestBlobs), +} +struct S3WithSimpleTestBlobs { + enabled: EnabledS3, + remote_blobs: HashSet, +} + +#[async_trait::async_trait] +impl AsyncTestContext for MaybeEnabledS3WithSimpleTestBlobs { + async fn setup() -> Self { + ensure_logging_ready(); + if env::var(ENABLE_REAL_S3_REMOTE_STORAGE_ENV_VAR_NAME).is_err() { + info!( + "`{}` env variable is not set, skipping the test", + ENABLE_REAL_S3_REMOTE_STORAGE_ENV_VAR_NAME + ); + return Self::Disabled; + } + + let max_keys_in_list_response = 10; + let upload_tasks_count = 1 + (2 * usize::try_from(max_keys_in_list_response).unwrap()); + + let enabled = EnabledS3::setup(Some(max_keys_in_list_response)).await; + + match upload_simple_s3_data(&enabled.client, upload_tasks_count).await { + ControlFlow::Continue(uploads) => { + info!("Remote objects created successfully"); + + Self::Enabled(S3WithSimpleTestBlobs { + enabled, + remote_blobs: uploads, + }) + } + ControlFlow::Break(uploads) => Self::UploadsFailed( + anyhow::anyhow!("One or multiple blobs failed to upload to S3"), + S3WithSimpleTestBlobs { + enabled, + remote_blobs: uploads, + }, + ), + } + } + + async fn teardown(self) { + match self { + Self::Disabled => {} + Self::Enabled(ctx) | Self::UploadsFailed(_, ctx) => { + cleanup(&ctx.enabled.client, ctx.remote_blobs).await; + } + } + } +} + +fn create_s3_client( + max_keys_per_list_response: Option, +) -> anyhow::Result> { + let remote_storage_s3_bucket = env::var("REMOTE_STORAGE_S3_BUCKET") + .context("`REMOTE_STORAGE_S3_BUCKET` env var is not set, but real S3 tests are enabled")?; + let remote_storage_s3_region = env::var("REMOTE_STORAGE_S3_REGION") + .context("`REMOTE_STORAGE_S3_REGION` env var is not set, but real S3 tests are enabled")?; + let random_prefix_part = std::time::SystemTime::now() + .duration_since(UNIX_EPOCH) + .context("random s3 test prefix part calculation")? + .as_nanos(); + let remote_storage_config = RemoteStorageConfig { + max_concurrent_syncs: NonZeroUsize::new(100).unwrap(), + max_sync_errors: NonZeroU32::new(5).unwrap(), + storage: RemoteStorageKind::AwsS3(S3Config { + bucket_name: remote_storage_s3_bucket, + bucket_region: remote_storage_s3_region, + prefix_in_bucket: Some(format!("pagination_should_work_test_{random_prefix_part}/")), + endpoint: None, + concurrency_limit: NonZeroUsize::new(100).unwrap(), + max_keys_per_list_response, + }), + }; + Ok(Arc::new( + GenericRemoteStorage::from_config(&remote_storage_config).context("remote storage init")?, + )) +} + +struct Uploads { + prefixes: HashSet, + blobs: HashSet, +} + +async fn upload_s3_data( + client: &Arc, + base_prefix_str: &'static str, + upload_tasks_count: usize, +) -> ControlFlow { + info!("Creating {upload_tasks_count} S3 files"); + let mut upload_tasks = JoinSet::new(); + for i in 1..upload_tasks_count + 1 { + let task_client = Arc::clone(client); + upload_tasks.spawn(async move { + let prefix = PathBuf::from(format!("{base_prefix_str}/sub_prefix_{i}/")); + let blob_prefix = RemotePath::new(&prefix) + .with_context(|| format!("{prefix:?} to RemotePath conversion"))?; + let blob_path = blob_prefix.join(Path::new(&format!("blob_{i}"))); + debug!("Creating remote item {i} at path {blob_path:?}"); + + let data = format!("remote blob data {i}").into_bytes(); + let data_len = data.len(); + task_client + .upload(std::io::Cursor::new(data), data_len, &blob_path, None) + .await?; + + Ok::<_, anyhow::Error>((blob_prefix, blob_path)) + }); + } + + let mut upload_tasks_failed = false; + let mut uploaded_prefixes = HashSet::with_capacity(upload_tasks_count); + let mut uploaded_blobs = HashSet::with_capacity(upload_tasks_count); + while let Some(task_run_result) = upload_tasks.join_next().await { + match task_run_result + .context("task join failed") + .and_then(|task_result| task_result.context("upload task failed")) + { + Ok((upload_prefix, upload_path)) => { + uploaded_prefixes.insert(upload_prefix); + uploaded_blobs.insert(upload_path); + } + Err(e) => { + error!("Upload task failed: {e:?}"); + upload_tasks_failed = true; + } + } + } + + let uploads = Uploads { + prefixes: uploaded_prefixes, + blobs: uploaded_blobs, + }; + if upload_tasks_failed { + ControlFlow::Break(uploads) + } else { + ControlFlow::Continue(uploads) + } +} + +async fn cleanup(client: &Arc, objects_to_delete: HashSet) { + info!( + "Removing {} objects from the remote storage during cleanup", + objects_to_delete.len() + ); + let mut delete_tasks = JoinSet::new(); + for object_to_delete in objects_to_delete { + let task_client = Arc::clone(client); + delete_tasks.spawn(async move { + debug!("Deleting remote item at path {object_to_delete:?}"); + task_client + .delete(&object_to_delete) + .await + .with_context(|| format!("{object_to_delete:?} removal")) + }); + } + + while let Some(task_run_result) = delete_tasks.join_next().await { + match task_run_result { + Ok(task_result) => match task_result { + Ok(()) => {} + Err(e) => error!("Delete task failed: {e:?}"), + }, + Err(join_err) => error!("Delete task did not finish correctly: {join_err}"), + } + } +} + +// Uploads files `folder{j}/blob{i}.txt`. See test description for more details. +async fn upload_simple_s3_data( + client: &Arc, + upload_tasks_count: usize, +) -> ControlFlow, HashSet> { + info!("Creating {upload_tasks_count} S3 files"); + let mut upload_tasks = JoinSet::new(); + for i in 1..upload_tasks_count + 1 { + let task_client = Arc::clone(client); + upload_tasks.spawn(async move { + let blob_path = PathBuf::from(format!("folder{}/blob_{}.txt", i / 7, i)); + let blob_path = RemotePath::new(&blob_path) + .with_context(|| format!("{blob_path:?} to RemotePath conversion"))?; + debug!("Creating remote item {i} at path {blob_path:?}"); + + let data = format!("remote blob data {i}").into_bytes(); + let data_len = data.len(); + task_client + .upload(std::io::Cursor::new(data), data_len, &blob_path, None) + .await?; + + Ok::<_, anyhow::Error>(blob_path) + }); + } + + let mut upload_tasks_failed = false; + let mut uploaded_blobs = HashSet::with_capacity(upload_tasks_count); + while let Some(task_run_result) = upload_tasks.join_next().await { + match task_run_result + .context("task join failed") + .and_then(|task_result| task_result.context("upload task failed")) + { + Ok(upload_path) => { + uploaded_blobs.insert(upload_path); + } + Err(e) => { + error!("Upload task failed: {e:?}"); + upload_tasks_failed = true; + } + } + } + + if upload_tasks_failed { + ControlFlow::Break(uploaded_blobs) + } else { + ControlFlow::Continue(uploaded_blobs) + } +} diff --git a/libs/utils/src/completion.rs b/libs/utils/src/completion.rs new file mode 100644 index 0000000000..2cdaee548e --- /dev/null +++ b/libs/utils/src/completion.rs @@ -0,0 +1,33 @@ +use std::sync::Arc; + +use tokio::sync::{mpsc, Mutex}; + +/// While a reference is kept around, the associated [`Barrier::wait`] will wait. +/// +/// Can be cloned, moved and kept around in futures as "guard objects". +#[derive(Clone)] +pub struct Completion(mpsc::Sender<()>); + +/// Barrier will wait until all clones of [`Completion`] have been dropped. +#[derive(Clone)] +pub struct Barrier(Arc>>); + +impl Barrier { + pub async fn wait(self) { + self.0.lock().await.recv().await; + } + + pub async fn maybe_wait(barrier: Option) { + if let Some(b) = barrier { + b.wait().await + } + } +} + +/// Create new Guard and Barrier pair. +pub fn channel() -> (Completion, Barrier) { + let (tx, rx) = mpsc::channel::<()>(1); + let rx = Mutex::new(rx); + let rx = Arc::new(rx); + (Completion(tx), Barrier(rx)) +} diff --git a/libs/utils/src/fs_ext.rs b/libs/utils/src/fs_ext.rs index d2cb7be816..0ef0464267 100644 --- a/libs/utils/src/fs_ext.rs +++ b/libs/utils/src/fs_ext.rs @@ -1,6 +1,8 @@ /// Extensions to `std::fs` types. use std::{fs, io, path::Path}; +use anyhow::Context; + pub trait PathExt { /// Returns an error if `self` is not a directory. fn is_empty_dir(&self) -> io::Result; @@ -15,10 +17,19 @@ where } } +pub async fn is_directory_empty(path: impl AsRef) -> anyhow::Result { + let mut dir = tokio::fs::read_dir(&path) + .await + .context(format!("read_dir({})", path.as_ref().display()))?; + Ok(dir.next_entry().await?.is_none()) +} + #[cfg(test)] mod test { use std::path::PathBuf; + use crate::fs_ext::is_directory_empty; + #[test] fn is_empty_dir() { use super::PathExt; @@ -42,4 +53,26 @@ mod test { std::fs::remove_file(&file_path).unwrap(); assert!(file_path.is_empty_dir().is_err()); } + + #[tokio::test] + async fn is_empty_dir_async() { + let dir = tempfile::tempdir().unwrap(); + let dir_path = dir.path(); + + // test positive case + assert!( + is_directory_empty(dir_path).await.expect("test failure"), + "new tempdir should be empty" + ); + + // invoke on a file to ensure it returns an error + let file_path: PathBuf = dir_path.join("testfile"); + let f = std::fs::File::create(&file_path).unwrap(); + drop(f); + assert!(is_directory_empty(&file_path).await.is_err()); + + // do it again on a path, we know to be nonexistent + std::fs::remove_file(&file_path).unwrap(); + assert!(is_directory_empty(file_path).await.is_err()); + } } diff --git a/libs/utils/src/http/endpoint.rs b/libs/utils/src/http/endpoint.rs index 4bfb5bf994..33241dbdf7 100644 --- a/libs/utils/src/http/endpoint.rs +++ b/libs/utils/src/http/endpoint.rs @@ -1,23 +1,20 @@ use crate::auth::{Claims, JwtAuth}; -use crate::http::error; -use anyhow::{anyhow, Context}; +use crate::http::error::{api_error_handler, route_error_handler, ApiError}; +use anyhow::Context; use hyper::header::{HeaderName, AUTHORIZATION}; use hyper::http::HeaderValue; use hyper::Method; -use hyper::{header::CONTENT_TYPE, Body, Request, Response, Server}; +use hyper::{header::CONTENT_TYPE, Body, Request, Response}; use metrics::{register_int_counter, Encoder, IntCounter, TextEncoder}; use once_cell::sync::Lazy; use routerify::ext::RequestExt; -use routerify::{Middleware, RequestInfo, Router, RouterBuilder, RouterService}; +use routerify::{Middleware, RequestInfo, Router, RouterBuilder}; use tokio::task::JoinError; use tracing::{self, debug, info, info_span, warn, Instrument}; use std::future::Future; -use std::net::TcpListener; use std::str::FromStr; -use super::error::ApiError; - static SERVE_METRICS_COUNT: Lazy = Lazy::new(|| { register_int_counter!( "libmetrics_metric_handler_requests_total", @@ -35,8 +32,18 @@ struct RequestId(String); /// Adds a tracing info_span! instrumentation around the handler events, /// logs the request start and end events for non-GET requests and non-200 responses. /// +/// Usage: Replace `my_handler` with `|r| request_span(r, my_handler)` +/// /// Use this to distinguish between logs of different HTTP requests: every request handler wrapped -/// in this type will get request info logged in the wrapping span, including the unique request ID. +/// with this will get request info logged in the wrapping span, including the unique request ID. +/// +/// This also handles errors, logging them and converting them to an HTTP error response. +/// +/// NB: If the client disconnects, Hyper will drop the Future, without polling it to +/// completion. In other words, the handler must be async cancellation safe! request_span +/// prints a warning to the log when that happens, so that you have some trace of it in +/// the log. +/// /// /// There could be other ways to implement similar functionality: /// @@ -54,60 +61,56 @@ struct RequestId(String); /// tries to achive with its `.instrument` used in the current approach. /// /// If needed, a declarative macro to substitute the |r| ... closure boilerplate could be introduced. -pub struct RequestSpan(pub H) +pub async fn request_span(request: Request, handler: H) -> R::Output where - E: Into> + 'static, - R: Future, E>> + Send + 'static, - H: Fn(Request) -> R + Send + Sync + 'static; - -impl RequestSpan -where - E: Into> + 'static, - R: Future, E>> + Send + 'static, - H: Fn(Request) -> R + Send + Sync + 'static, + R: Future, ApiError>> + Send + 'static, + H: FnOnce(Request) -> R + Send + Sync + 'static, { - /// Creates a tracing span around inner request handler and executes the request handler in the contex of that span. - /// Use as `|r| RequestSpan(my_handler).handle(r)` instead of `my_handler` as the request handler to get the span enabled. - pub async fn handle(self, request: Request) -> Result, E> { - let request_id = request.context::().unwrap_or_default().0; - let method = request.method(); - let path = request.uri().path(); - let request_span = info_span!("request", %method, %path, %request_id); + let request_id = request.context::().unwrap_or_default().0; + let method = request.method(); + let path = request.uri().path(); + let request_span = info_span!("request", %method, %path, %request_id); - let log_quietly = method == Method::GET; - async move { - let cancellation_guard = RequestCancelled::warn_when_dropped_without_responding(); - if log_quietly { - debug!("Handling request"); - } else { - info!("Handling request"); - } - - // Note that we reuse `error::handler` here and not returning and error at all, - // yet cannot use `!` directly in the method signature due to `routerify::RouterBuilder` limitation. - // Usage of the error handler also means that we expect only the `ApiError` errors to be raised in this call. - // - // Panics are not handled separately, there's a `tracing_panic_hook` from another module to do that globally. - let res = (self.0)(request).await; - - cancellation_guard.disarm(); - - match res { - Ok(response) => { - let response_status = response.status(); - if log_quietly && response_status.is_success() { - debug!("Request handled, status: {response_status}"); - } else { - info!("Request handled, status: {response_status}"); - } - Ok(response) - } - Err(e) => Ok(error::handler(e.into()).await), - } + let log_quietly = method == Method::GET; + async move { + let cancellation_guard = RequestCancelled::warn_when_dropped_without_responding(); + if log_quietly { + debug!("Handling request"); + } else { + info!("Handling request"); + } + + // No special handling for panics here. There's a `tracing_panic_hook` from another + // module to do that globally. + let res = handler(request).await; + + cancellation_guard.disarm(); + + // Log the result if needed. + // + // We also convert any errors into an Ok response with HTTP error code here. + // `make_router` sets a last-resort error handler that would do the same, but + // we prefer to do it here, before we exit the request span, so that the error + // is still logged with the span. + // + // (Because we convert errors to Ok response, we never actually return an error, + // and we could declare the function to return the never type (`!`). However, + // using `routerify::RouterBuilder` requires a proper error type.) + match res { + Ok(response) => { + let response_status = response.status(); + if log_quietly && response_status.is_success() { + debug!("Request handled, status: {response_status}"); + } else { + info!("Request handled, status: {response_status}"); + } + Ok(response) + } + Err(err) => Ok(api_error_handler(err)), } - .instrument(request_span) - .await } + .instrument(request_span) + .await } /// Drop guard to WARN in case the request was dropped before completion. @@ -207,10 +210,8 @@ pub fn make_router() -> RouterBuilder { .middleware(Middleware::post_with_info( add_request_id_header_to_response, )) - .get("/metrics", |r| { - RequestSpan(prometheus_metrics_handler).handle(r) - }) - .err_handler(error::handler) + .get("/metrics", |r| request_span(r, prometheus_metrics_handler)) + .err_handler(route_error_handler) } pub fn attach_openapi_ui( @@ -220,12 +221,14 @@ pub fn attach_openapi_ui( ui_mount_path: &'static str, ) -> RouterBuilder { router_builder - .get(spec_mount_path, move |r| { - RequestSpan(move |_| async move { Ok(Response::builder().body(Body::from(spec)).unwrap()) }) - .handle(r) - }) - .get(ui_mount_path, move |r| RequestSpan( move |_| async move { - Ok(Response::builder().body(Body::from(format!(r#" + .get(spec_mount_path, + move |r| request_span(r, move |_| async move { + Ok(Response::builder().body(Body::from(spec)).unwrap()) + }) + ) + .get(ui_mount_path, + move |r| request_span(r, move |_| async move { + Ok(Response::builder().body(Body::from(format!(r#" @@ -255,7 +258,8 @@ pub fn attach_openapi_ui( "#, spec_mount_path))).unwrap()) - }).handle(r)) + }) + ) } fn parse_token(header_value: &str) -> Result<&str, ApiError> { @@ -343,40 +347,6 @@ pub fn check_permission_with( } } -/// -/// Start listening for HTTP requests on given socket. -/// -/// 'shutdown_future' can be used to stop. If the Future becomes -/// ready, we stop listening for new requests, and the function returns. -/// -pub fn serve_thread_main( - router_builder: RouterBuilder, - listener: TcpListener, - shutdown_future: S, -) -> anyhow::Result<()> -where - S: Future + Send + Sync, -{ - info!("Starting an HTTP endpoint at {}", listener.local_addr()?); - - // Create a Service from the router above to handle incoming requests. - let service = RouterService::new(router_builder.build().map_err(|err| anyhow!(err))?).unwrap(); - - // Enter a single-threaded tokio runtime bound to the current thread - let runtime = tokio::runtime::Builder::new_current_thread() - .enable_all() - .build()?; - - let _guard = runtime.enter(); - - let server = Server::from_tcp(listener)? - .serve(service) - .with_graceful_shutdown(shutdown_future); - - runtime.block_on(server)?; - - Ok(()) -} #[cfg(test)] mod tests { use super::*; diff --git a/libs/utils/src/http/error.rs b/libs/utils/src/http/error.rs index 3c6023eb80..527e486fd0 100644 --- a/libs/utils/src/http/error.rs +++ b/libs/utils/src/http/error.rs @@ -1,5 +1,6 @@ use hyper::{header, Body, Response, StatusCode}; use serde::{Deserialize, Serialize}; +use std::error::Error as StdError; use thiserror::Error; use tracing::error; @@ -15,13 +16,13 @@ pub enum ApiError { Unauthorized(String), #[error("NotFound: {0}")] - NotFound(anyhow::Error), + NotFound(Box), #[error("Conflict: {0}")] Conflict(String), #[error("Precondition failed: {0}")] - PreconditionFailed(&'static str), + PreconditionFailed(Box), #[error(transparent)] InternalServerError(anyhow::Error), @@ -83,13 +84,24 @@ impl HttpErrorBody { } } -pub async fn handler(err: routerify::RouteError) -> Response { - let api_error = err - .downcast::() - .expect("handler should always return api error"); +pub async fn route_error_handler(err: routerify::RouteError) -> Response { + match err.downcast::() { + Ok(api_error) => api_error_handler(*api_error), + Err(other_error) => { + // We expect all the request handlers to return an ApiError, so this should + // not be reached. But just in case. + error!("Error processing HTTP request: {other_error:?}"); + HttpErrorBody::response_from_msg_and_status( + other_error.to_string(), + StatusCode::INTERNAL_SERVER_ERROR, + ) + } + } +} +pub fn api_error_handler(api_error: ApiError) -> Response { // Print a stack trace for Internal Server errors - if let ApiError::InternalServerError(_) = api_error.as_ref() { + if let ApiError::InternalServerError(_) = api_error { error!("Error processing HTTP request: {api_error:?}"); } else { error!("Error processing HTTP request: {api_error:#}"); diff --git a/libs/utils/src/lib.rs b/libs/utils/src/lib.rs index 4e4f79ab6b..69d3a1b9f2 100644 --- a/libs/utils/src/lib.rs +++ b/libs/utils/src/lib.rs @@ -60,6 +60,9 @@ pub mod tracing_span_assert; pub mod rate_limit; +/// Simple once-barrier and a guard which keeps barrier awaiting. +pub mod completion; + mod failpoint_macro_helpers { /// use with fail::cfg("$name", "return(2000)") diff --git a/pageserver/benches/bench_layer_map.rs b/pageserver/benches/bench_layer_map.rs index ee5980212e..03bb7a5bfd 100644 --- a/pageserver/benches/bench_layer_map.rs +++ b/pageserver/benches/bench_layer_map.rs @@ -1,22 +1,23 @@ use pageserver::keyspace::{KeyPartitioning, KeySpace}; use pageserver::repository::Key; use pageserver::tenant::layer_map::LayerMap; -use pageserver::tenant::storage_layer::{Layer, LayerDescriptor, LayerFileName}; +use pageserver::tenant::storage_layer::{tests::LayerDescriptor, Layer, LayerFileName}; +use pageserver::tenant::storage_layer::{PersistentLayer, PersistentLayerDesc}; use rand::prelude::{SeedableRng, SliceRandom, StdRng}; use std::cmp::{max, min}; use std::fs::File; use std::io::{BufRead, BufReader}; use std::path::PathBuf; use std::str::FromStr; -use std::sync::Arc; use std::time::Instant; +use utils::id::{TenantId, TimelineId}; use utils::lsn::Lsn; use criterion::{black_box, criterion_group, criterion_main, Criterion}; -fn build_layer_map(filename_dump: PathBuf) -> LayerMap { - let mut layer_map = LayerMap::::default(); +fn build_layer_map(filename_dump: PathBuf) -> LayerMap { + let mut layer_map = LayerMap::default(); let mut min_lsn = Lsn(u64::MAX); let mut max_lsn = Lsn(0); @@ -33,7 +34,7 @@ fn build_layer_map(filename_dump: PathBuf) -> LayerMap { min_lsn = min(min_lsn, lsn_range.start); max_lsn = max(max_lsn, Lsn(lsn_range.end.0 - 1)); - updates.insert_historic(Arc::new(layer)); + updates.insert_historic(layer.layer_desc().clone()); } println!("min: {min_lsn}, max: {max_lsn}"); @@ -43,7 +44,7 @@ fn build_layer_map(filename_dump: PathBuf) -> LayerMap { } /// Construct a layer map query pattern for benchmarks -fn uniform_query_pattern(layer_map: &LayerMap) -> Vec<(Key, Lsn)> { +fn uniform_query_pattern(layer_map: &LayerMap) -> Vec<(Key, Lsn)> { // For each image layer we query one of the pages contained, at LSN right // before the image layer was created. This gives us a somewhat uniform // coverage of both the lsn and key space because image layers have @@ -69,7 +70,7 @@ fn uniform_query_pattern(layer_map: &LayerMap) -> Vec<(Key, Lsn // Construct a partitioning for testing get_difficulty map when we // don't have an exact result of `collect_keyspace` to work with. -fn uniform_key_partitioning(layer_map: &LayerMap, _lsn: Lsn) -> KeyPartitioning { +fn uniform_key_partitioning(layer_map: &LayerMap, _lsn: Lsn) -> KeyPartitioning { let mut parts = Vec::new(); // We add a partition boundary at the start of each image layer, @@ -209,13 +210,15 @@ fn bench_sequential(c: &mut Criterion) { for i in 0..100_000 { let i32 = (i as u32) % 100; let zero = Key::from_hex("000000000000000000000000000000000000").unwrap(); - let layer = LayerDescriptor { - key: zero.add(10 * i32)..zero.add(10 * i32 + 1), - lsn: Lsn(i)..Lsn(i + 1), - is_incremental: false, - short_id: format!("Layer {}", i), - }; - updates.insert_historic(Arc::new(layer)); + let layer = LayerDescriptor::from(PersistentLayerDesc::new_img( + TenantId::generate(), + TimelineId::generate(), + zero.add(10 * i32)..zero.add(10 * i32 + 1), + Lsn(i), + false, + 0, + )); + updates.insert_historic(layer.layer_desc().clone()); } updates.flush(); println!("Finished layer map init in {:?}", now.elapsed()); diff --git a/pageserver/src/bin/pageserver.rs b/pageserver/src/bin/pageserver.rs index d9d3d9d662..b01ace63e4 100644 --- a/pageserver/src/bin/pageserver.rs +++ b/pageserver/src/bin/pageserver.rs @@ -335,13 +335,119 @@ fn start_pageserver( // Set up remote storage client let remote_storage = create_remote_storage_client(conf)?; + // Startup staging or optimizing: + // + // We want to minimize downtime for `page_service` connections, and trying not to overload + // BACKGROUND_RUNTIME by doing initial compactions and initial logical sizes at the same time. + // + // init_done_rx will notify when all initial load operations have completed. + // + // background_jobs_can_start (same name used to hold off background jobs from starting at + // consumer side) will be dropped once we can start the background jobs. Currently it is behind + // completing all initial logical size calculations (init_logical_size_done_rx) and a timeout + // (background_task_maximum_delay). + let (init_done_tx, init_done_rx) = utils::completion::channel(); + + let (init_logical_size_done_tx, init_logical_size_done_rx) = utils::completion::channel(); + + let (background_jobs_can_start, background_jobs_barrier) = utils::completion::channel(); + + let order = pageserver::InitializationOrder { + initial_tenant_load: Some(init_done_tx), + initial_logical_size_can_start: init_done_rx.clone(), + initial_logical_size_attempt: init_logical_size_done_tx, + background_jobs_can_start: background_jobs_barrier.clone(), + }; + // Scan the local 'tenants/' directory and start loading the tenants + let init_started_at = std::time::Instant::now(); + let shutdown_pageserver = tokio_util::sync::CancellationToken::new(); + BACKGROUND_RUNTIME.block_on(mgr::init_tenant_mgr( conf, broker_client.clone(), remote_storage.clone(), + order, ))?; + BACKGROUND_RUNTIME.spawn({ + let init_done_rx = init_done_rx; + let shutdown_pageserver = shutdown_pageserver.clone(); + let drive_init = async move { + // NOTE: unlike many futures in pageserver, this one is cancellation-safe + let guard = scopeguard::guard_on_success((), |_| tracing::info!("Cancelled before initial load completed")); + + init_done_rx.wait().await; + // initial logical sizes can now start, as they were waiting on init_done_rx. + + scopeguard::ScopeGuard::into_inner(guard); + + let init_done = std::time::Instant::now(); + let elapsed = init_done - init_started_at; + + tracing::info!( + elapsed_millis = elapsed.as_millis(), + "Initial load completed" + ); + + let mut init_sizes_done = std::pin::pin!(init_logical_size_done_rx.wait()); + + let timeout = conf.background_task_maximum_delay; + + let guard = scopeguard::guard_on_success((), |_| tracing::info!("Cancelled before initial logical sizes completed")); + + let init_sizes_done = tokio::select! { + _ = &mut init_sizes_done => { + let now = std::time::Instant::now(); + tracing::info!( + from_init_done_millis = (now - init_done).as_millis(), + from_init_millis = (now - init_started_at).as_millis(), + "Initial logical sizes completed" + ); + None + } + _ = tokio::time::sleep(timeout) => { + tracing::info!( + timeout_millis = timeout.as_millis(), + "Initial logical size timeout elapsed; starting background jobs" + ); + Some(init_sizes_done) + } + }; + + scopeguard::ScopeGuard::into_inner(guard); + + // allow background jobs to start + drop(background_jobs_can_start); + + if let Some(init_sizes_done) = init_sizes_done { + // ending up here is not a bug; at the latest logical sizes will be queried by + // consumption metrics. + let guard = scopeguard::guard_on_success((), |_| tracing::info!("Cancelled before initial logical sizes completed")); + init_sizes_done.await; + + scopeguard::ScopeGuard::into_inner(guard); + + let now = std::time::Instant::now(); + tracing::info!( + from_init_done_millis = (now - init_done).as_millis(), + from_init_millis = (now - init_started_at).as_millis(), + "Initial logical sizes completed after timeout (background jobs already started)" + ); + + } + }; + + async move { + let mut drive_init = std::pin::pin!(drive_init); + // just race these tasks + tokio::select! { + _ = shutdown_pageserver.cancelled() => {}, + _ = &mut drive_init => {}, + } + } + }); + // shared state between the disk-usage backed eviction background task and the http endpoint // that allows triggering disk-usage based eviction manually. note that the http endpoint // is still accessible even if background task is not configured as long as remote storage has @@ -353,6 +459,7 @@ fn start_pageserver( conf, remote_storage.clone(), disk_usage_eviction_state.clone(), + background_jobs_barrier.clone(), )?; } @@ -388,37 +495,50 @@ fn start_pageserver( Ok(()) }, ); + } - if let Some(metric_collection_endpoint) = &conf.metric_collection_endpoint { - let metrics_ctx = RequestContext::todo_child( - TaskKind::MetricsCollection, - // This task itself shouldn't download anything. - // The actual size calculation does need downloads, and - // creates a child context with the right DownloadBehavior. - DownloadBehavior::Error, - ); - task_mgr::spawn( - MGMT_REQUEST_RUNTIME.handle(), - TaskKind::MetricsCollection, - None, - None, - "consumption metrics collection", - true, - async move { - pageserver::consumption_metrics::collect_metrics( - metric_collection_endpoint, - conf.metric_collection_interval, - conf.cached_metric_collection_interval, - conf.synthetic_size_calculation_interval, - conf.id, - metrics_ctx, - ) - .instrument(info_span!("metrics_collection")) - .await?; - Ok(()) - }, - ); - } + if let Some(metric_collection_endpoint) = &conf.metric_collection_endpoint { + let background_jobs_barrier = background_jobs_barrier; + let metrics_ctx = RequestContext::todo_child( + TaskKind::MetricsCollection, + // This task itself shouldn't download anything. + // The actual size calculation does need downloads, and + // creates a child context with the right DownloadBehavior. + DownloadBehavior::Error, + ); + task_mgr::spawn( + crate::BACKGROUND_RUNTIME.handle(), + TaskKind::MetricsCollection, + None, + None, + "consumption metrics collection", + true, + async move { + // first wait until background jobs are cleared to launch. + // + // this is because we only process active tenants and timelines, and the + // Timeline::get_current_logical_size will spawn the logical size calculation, + // which will not be rate-limited. + let cancel = task_mgr::shutdown_token(); + + tokio::select! { + _ = cancel.cancelled() => { return Ok(()); }, + _ = background_jobs_barrier.wait() => {} + }; + + pageserver::consumption_metrics::collect_metrics( + metric_collection_endpoint, + conf.metric_collection_interval, + conf.cached_metric_collection_interval, + conf.synthetic_size_calculation_interval, + conf.id, + metrics_ctx, + ) + .instrument(info_span!("metrics_collection")) + .await?; + Ok(()) + }, + ); } // Spawn a task to listen for libpq connections. It will spawn further tasks @@ -453,6 +573,8 @@ fn start_pageserver( ); } + let mut shutdown_pageserver = Some(shutdown_pageserver.drop_guard()); + // All started up! Now just sit and wait for shutdown signal. ShutdownSignals::handle(|signal| match signal { Signal::Quit => { @@ -468,6 +590,11 @@ fn start_pageserver( "Got {}. Terminating gracefully in fast shutdown mode", signal.name() ); + + // This cancels the `shutdown_pageserver` cancellation tree. + // Right now that tree doesn't reach very far, and `task_mgr` is used instead. + // The plan is to change that over time. + shutdown_pageserver.take(); BACKGROUND_RUNTIME.block_on(pageserver::shutdown_pageserver(0)); unreachable!() } diff --git a/pageserver/src/config.rs b/pageserver/src/config.rs index 88a7f15b21..2046d27b1e 100644 --- a/pageserver/src/config.rs +++ b/pageserver/src/config.rs @@ -63,6 +63,7 @@ pub mod defaults { pub const DEFAULT_CACHED_METRIC_COLLECTION_INTERVAL: &str = "1 hour"; pub const DEFAULT_METRIC_COLLECTION_ENDPOINT: Option = None; pub const DEFAULT_SYNTHETIC_SIZE_CALCULATION_INTERVAL: &str = "10 min"; + pub const DEFAULT_BACKGROUND_TASK_MAXIMUM_DELAY: &str = "10s"; /// /// Default built-in configuration file. @@ -91,15 +92,16 @@ pub mod defaults { #cached_metric_collection_interval = '{DEFAULT_CACHED_METRIC_COLLECTION_INTERVAL}' #synthetic_size_calculation_interval = '{DEFAULT_SYNTHETIC_SIZE_CALCULATION_INTERVAL}' - #disk_usage_based_eviction = {{ max_usage_pct = .., min_avail_bytes = .., period = "10s"}} -# [tenant_config] +#background_task_maximum_delay = '{DEFAULT_BACKGROUND_TASK_MAXIMUM_DELAY}' + +[tenant_config] #checkpoint_distance = {DEFAULT_CHECKPOINT_DISTANCE} # in bytes #checkpoint_timeout = {DEFAULT_CHECKPOINT_TIMEOUT} #compaction_target_size = {DEFAULT_COMPACTION_TARGET_SIZE} # in bytes #compaction_period = '{DEFAULT_COMPACTION_PERIOD}' -#compaction_threshold = '{DEFAULT_COMPACTION_THRESHOLD}' +#compaction_threshold = {DEFAULT_COMPACTION_THRESHOLD} #gc_period = '{DEFAULT_GC_PERIOD}' #gc_horizon = {DEFAULT_GC_HORIZON} @@ -108,8 +110,9 @@ pub mod defaults { #min_resident_size_override = .. # in bytes #evictions_low_residence_duration_metric_threshold = '{DEFAULT_EVICTIONS_LOW_RESIDENCE_DURATION_METRIC_THRESHOLD}' +#gc_feedback = false -# [remote_storage] +[remote_storage] "### ); @@ -187,6 +190,15 @@ pub struct PageServerConf { pub test_remote_failures: u64, pub ondemand_download_behavior_treat_error_as_warn: bool, + + /// How long will background tasks be delayed at most after initial load of tenants. + /// + /// Our largest initialization completions are in the range of 100-200s, so perhaps 10s works + /// as we now isolate initial loading, initial logical size calculation and background tasks. + /// Smaller nodes will have background tasks "not running" for this long unless every timeline + /// has it's initial logical size calculated. Not running background tasks for some seconds is + /// not terrible. + pub background_task_maximum_delay: Duration, } /// We do not want to store this in a PageServerConf because the latter may be logged @@ -259,6 +271,8 @@ struct PageServerConfigBuilder { test_remote_failures: BuilderValue, ondemand_download_behavior_treat_error_as_warn: BuilderValue, + + background_task_maximum_delay: BuilderValue, } impl Default for PageServerConfigBuilder { @@ -316,6 +330,11 @@ impl Default for PageServerConfigBuilder { test_remote_failures: Set(0), ondemand_download_behavior_treat_error_as_warn: Set(false), + + background_task_maximum_delay: Set(humantime::parse_duration( + DEFAULT_BACKGROUND_TASK_MAXIMUM_DELAY, + ) + .unwrap()), } } } @@ -440,6 +459,10 @@ impl PageServerConfigBuilder { BuilderValue::Set(ondemand_download_behavior_treat_error_as_warn); } + pub fn background_task_maximum_delay(&mut self, delay: Duration) { + self.background_task_maximum_delay = BuilderValue::Set(delay); + } + pub fn build(self) -> anyhow::Result { let concurrent_tenant_size_logical_size_queries = self .concurrent_tenant_size_logical_size_queries @@ -522,6 +545,9 @@ impl PageServerConfigBuilder { .ok_or(anyhow!( "missing ondemand_download_behavior_treat_error_as_warn" ))?, + background_task_maximum_delay: self + .background_task_maximum_delay + .ok_or(anyhow!("missing background_task_maximum_delay"))?, }) } } @@ -710,6 +736,7 @@ impl PageServerConf { ) }, "ondemand_download_behavior_treat_error_as_warn" => builder.ondemand_download_behavior_treat_error_as_warn(parse_toml_bool(key, item)?), + "background_task_maximum_delay" => builder.background_task_maximum_delay(parse_toml_duration(key, item)?), _ => bail!("unrecognized pageserver option '{key}'"), } } @@ -828,6 +855,14 @@ impl PageServerConf { )?); } + if let Some(gc_feedback) = item.get("gc_feedback") { + t_conf.gc_feedback = Some( + gc_feedback + .as_bool() + .with_context(|| "configure option gc_feedback is not a bool".to_string())?, + ); + } + Ok(t_conf) } @@ -869,6 +904,7 @@ impl PageServerConf { disk_usage_based_eviction: None, test_remote_failures: 0, ondemand_download_behavior_treat_error_as_warn: false, + background_task_maximum_delay: Duration::ZERO, } } } @@ -1028,6 +1064,7 @@ metric_collection_endpoint = 'http://localhost:80/metrics' synthetic_size_calculation_interval = '333 s' log_format = 'json' +background_task_maximum_delay = '334 s' "#; @@ -1086,6 +1123,9 @@ log_format = 'json' disk_usage_based_eviction: None, test_remote_failures: 0, ondemand_download_behavior_treat_error_as_warn: false, + background_task_maximum_delay: humantime::parse_duration( + defaults::DEFAULT_BACKGROUND_TASK_MAXIMUM_DELAY + )?, }, "Correct defaults should be used when no config values are provided" ); @@ -1140,6 +1180,7 @@ log_format = 'json' disk_usage_based_eviction: None, test_remote_failures: 0, ondemand_download_behavior_treat_error_as_warn: false, + background_task_maximum_delay: Duration::from_secs(334), }, "Should be able to parse all basic config values correctly" ); diff --git a/pageserver/src/context.rs b/pageserver/src/context.rs index e826d28e6d..f53b7736ab 100644 --- a/pageserver/src/context.rs +++ b/pageserver/src/context.rs @@ -88,6 +88,7 @@ use crate::task_mgr::TaskKind; // The main structure of this module, see module-level comment. +#[derive(Clone, Debug)] pub struct RequestContext { task_kind: TaskKind, download_behavior: DownloadBehavior, @@ -95,7 +96,7 @@ pub struct RequestContext { /// Desired behavior if the operation requires an on-demand download /// to proceed. -#[derive(Clone, Copy, PartialEq, Eq)] +#[derive(Clone, Copy, PartialEq, Eq, Debug)] pub enum DownloadBehavior { /// Download the layer file. It can take a while. Download, diff --git a/pageserver/src/disk_usage_eviction_task.rs b/pageserver/src/disk_usage_eviction_task.rs index f4a0f3f18e..61cbd5066f 100644 --- a/pageserver/src/disk_usage_eviction_task.rs +++ b/pageserver/src/disk_usage_eviction_task.rs @@ -54,6 +54,7 @@ use serde::{Deserialize, Serialize}; use tokio::time::Instant; use tokio_util::sync::CancellationToken; use tracing::{debug, error, info, instrument, warn, Instrument}; +use utils::completion; use utils::serde_percent::Percent; use crate::{ @@ -82,6 +83,7 @@ pub fn launch_disk_usage_global_eviction_task( conf: &'static PageServerConf, storage: GenericRemoteStorage, state: Arc, + background_jobs_barrier: completion::Barrier, ) -> anyhow::Result<()> { let Some(task_config) = &conf.disk_usage_based_eviction else { info!("disk usage based eviction task not configured"); @@ -98,14 +100,16 @@ pub fn launch_disk_usage_global_eviction_task( "disk usage based eviction", false, async move { - disk_usage_eviction_task( - &state, - task_config, - storage, - &conf.tenants_path(), - task_mgr::shutdown_token(), - ) - .await; + let cancel = task_mgr::shutdown_token(); + + // wait until initial load is complete, because we cannot evict from loading tenants. + tokio::select! { + _ = cancel.cancelled() => { return Ok(()); }, + _ = background_jobs_barrier.wait() => { } + }; + + disk_usage_eviction_task(&state, task_config, storage, &conf.tenants_path(), cancel) + .await; info!("disk usage based eviction task finishing"); Ok(()) }, @@ -512,7 +516,7 @@ async fn collect_eviction_candidates( if !tl.is_active() { continue; } - let info = tl.get_local_layers_for_disk_usage_eviction(); + let info = tl.get_local_layers_for_disk_usage_eviction().await; debug!(tenant_id=%tl.tenant_id, timeline_id=%tl.timeline_id, "timeline resident layers count: {}", info.resident_layers.len()); tenant_candidates.extend( info.resident_layers diff --git a/pageserver/src/http/openapi_spec.yml b/pageserver/src/http/openapi_spec.yml index 0d912c95e0..33a2e32141 100644 --- a/pageserver/src/http/openapi_spec.yml +++ b/pageserver/src/http/openapi_spec.yml @@ -186,10 +186,8 @@ paths: schema: $ref: "#/components/schemas/Error" delete: - description: "Attempts to delete specified timeline. On 500 errors should be retried" + description: "Attempts to delete specified timeline. 500 and 409 errors should be retried" responses: - "200": - description: Ok "400": description: Error when no tenant id found in path or no timeline id content: @@ -214,8 +212,14 @@ paths: application/json: schema: $ref: "#/components/schemas/NotFoundError" + "409": + description: Deletion is already in progress, continue polling + content: + application/json: + schema: + $ref: "#/components/schemas/ConflictError" "412": - description: Tenant is missing + description: Tenant is missing, or timeline has children content: application/json: schema: @@ -386,6 +390,7 @@ paths: "202": description: Tenant attaching scheduled "400": + description: Bad Request content: application/json: schema: @@ -928,12 +933,28 @@ components: writing to the tenant's S3 state, so, DO NOT ATTACH the tenant to any other pageserver, or we risk split-brain. - `attached` means that the attach operation has completed, - maybe successfully, maybe not. Perform a health check at - the Postgres level to determine healthiness of the tenant. + successfully + - `failed` means that attach has failed. For reason check corresponding `reason` failed. + `failed` is the terminal state, retrying attach call wont resolve the issue. + For example this can be caused by s3 being unreachable. The retry may be implemented + with call to detach, though it would be better to not automate it and inspec failed state + manually before proceeding with a retry. See the tenant `/attach` endpoint for more information. - type: string - enum: [ "maybe", "attached" ] + type: object + required: + - slug + - data + properties: + slug: + type: string + enum: [ "maybe", "attached", "failed" ] + data: + type: object + properties: + reason: + type: string + TenantCreateRequest: allOf: - $ref: '#/components/schemas/TenantConfig' diff --git a/pageserver/src/http/routes.rs b/pageserver/src/http/routes.rs index 30c219f773..08d3fcf8df 100644 --- a/pageserver/src/http/routes.rs +++ b/pageserver/src/http/routes.rs @@ -1,3 +1,6 @@ +//! +//! Management HTTP API +//! use std::collections::HashMap; use std::sync::Arc; @@ -11,7 +14,7 @@ use storage_broker::BrokerClientChannel; use tenant_size_model::{SizeResult, StorageModel}; use tokio_util::sync::CancellationToken; use tracing::*; -use utils::http::endpoint::RequestSpan; +use utils::http::endpoint::request_span; use utils::http::json::json_request_or_empty_body; use utils::http::request::{get_request_param, must_get_query_param, parse_query_param}; @@ -20,7 +23,6 @@ use super::models::{ TimelineCreateRequest, TimelineGcRequest, TimelineInfo, }; use crate::context::{DownloadBehavior, RequestContext}; -use crate::disk_usage_eviction_task; use crate::metrics::{StorageTimeOperation, STORAGE_TIME_GLOBAL}; use crate::pgdatadir_mapping::LsnForTimestamp; use crate::task_mgr::TaskKind; @@ -32,6 +34,7 @@ use crate::tenant::size::ModelInputs; use crate::tenant::storage_layer::LayerAccessStatsReset; use crate::tenant::{LogicalSizeCalculationCause, PageReconstructError, Timeline}; use crate::{config::PageServerConf, tenant::mgr}; +use crate::{disk_usage_eviction_task, tenant}; use utils::{ auth::JwtAuth, http::{ @@ -46,7 +49,6 @@ use utils::{ }; // Imports only used for testing APIs -#[cfg(feature = "testing")] use super::models::ConfigureFailpointsRequest; struct State { @@ -140,7 +142,7 @@ impl From for ApiError { impl From for ApiError { fn from(tse: TenantStateError) -> ApiError { match tse { - TenantStateError::NotFound(tid) => ApiError::NotFound(anyhow!("tenant {}", tid)), + TenantStateError::NotFound(tid) => ApiError::NotFound(anyhow!("tenant {}", tid).into()), _ => ApiError::InternalServerError(anyhow::Error::new(tse)), } } @@ -149,7 +151,7 @@ impl From for ApiError { impl From for ApiError { fn from(tse: GetTenantError) -> ApiError { match tse { - GetTenantError::NotFound(tid) => ApiError::NotFound(anyhow!("tenant {}", tid)), + GetTenantError::NotFound(tid) => ApiError::NotFound(anyhow!("tenant {}", tid).into()), e @ GetTenantError::NotActive(_) => { // Why is this not `ApiError::NotFound`? // Because we must be careful to never return 404 for a tenant if it does @@ -167,7 +169,7 @@ impl From for ApiError { fn from(e: SetNewTenantConfigError) -> ApiError { match e { SetNewTenantConfigError::GetTenant(tid) => { - ApiError::NotFound(anyhow!("tenant {}", tid)) + ApiError::NotFound(anyhow!("tenant {}", tid).into()) } e @ SetNewTenantConfigError::Persist(_) => { ApiError::InternalServerError(anyhow::Error::new(e)) @@ -180,10 +182,12 @@ impl From for ApiError { fn from(value: crate::tenant::DeleteTimelineError) -> Self { use crate::tenant::DeleteTimelineError::*; match value { - NotFound => ApiError::NotFound(anyhow::anyhow!("timeline not found")), - HasChildren => ApiError::BadRequest(anyhow::anyhow!( - "Cannot delete timeline which has child timelines" - )), + NotFound => ApiError::NotFound(anyhow::anyhow!("timeline not found").into()), + HasChildren(children) => ApiError::PreconditionFailed( + format!("Cannot delete timeline which has child timelines: {children:?}") + .into_boxed_str(), + ), + a @ AlreadyInProgress => ApiError::Conflict(a.to_string()), Other(e) => ApiError::InternalServerError(e), } } @@ -195,9 +199,9 @@ impl From for ApiError { match value { // Report Precondition failed so client can distinguish between // "tenant is missing" case from "timeline is missing" - Tenant(GetTenantError::NotFound(..)) => { - ApiError::PreconditionFailed("Requested tenant is missing") - } + Tenant(GetTenantError::NotFound(..)) => ApiError::PreconditionFailed( + "Requested tenant is missing".to_owned().into_boxed_str(), + ), Tenant(t) => ApiError::from(t), Timeline(t) => ApiError::from(t), } @@ -212,7 +216,7 @@ async fn build_timeline_info( ) -> anyhow::Result { crate::tenant::debug_assert_current_span_has_tenant_and_timeline_id(); - let mut info = build_timeline_info_common(timeline, ctx)?; + let mut info = build_timeline_info_common(timeline, ctx).await?; if include_non_incremental_logical_size { // XXX we should be using spawn_ondemand_logical_size_calculation here. // Otherwise, if someone deletes the timeline / detaches the tenant while @@ -230,7 +234,7 @@ async fn build_timeline_info( Ok(info) } -fn build_timeline_info_common( +async fn build_timeline_info_common( timeline: &Arc, ctx: &RequestContext, ) -> anyhow::Result { @@ -261,7 +265,7 @@ fn build_timeline_info_common( None } }; - let current_physical_size = Some(timeline.layer_size_sum()); + let current_physical_size = Some(timeline.layer_size_sum().await); let state = timeline.current_state(); let remote_consistent_lsn = timeline.get_remote_consistent_lsn().unwrap_or(Lsn(0)); @@ -290,13 +294,19 @@ fn build_timeline_info_common( } // healthcheck handler -async fn status_handler(request: Request) -> Result, ApiError> { +async fn status_handler( + request: Request, + _cancel: CancellationToken, +) -> Result, ApiError> { check_permission(&request, None)?; let config = get_config(&request); json_response(StatusCode::OK, StatusResponse { id: config.id }) } -async fn timeline_create_handler(mut request: Request) -> Result, ApiError> { +async fn timeline_create_handler( + mut request: Request, + _cancel: CancellationToken, +) -> Result, ApiError> { let tenant_id: TenantId = parse_request_param(&request, "tenant_id")?; let request_data: TimelineCreateRequest = json_request(&mut request).await?; check_permission(&request, Some(tenant_id))?; @@ -318,21 +328,27 @@ async fn timeline_create_handler(mut request: Request) -> Result { + Ok(new_timeline) => { // Created. Construct a TimelineInfo for it. let timeline_info = build_timeline_info_common(&new_timeline, &ctx) + .await .map_err(ApiError::InternalServerError)?; json_response(StatusCode::CREATED, timeline_info) } - Ok(None) => json_response(StatusCode::CONFLICT, ()), // timeline already exists - Err(err) => Err(ApiError::InternalServerError(err)), + Err(tenant::CreateTimelineError::AlreadyExists) => { + json_response(StatusCode::CONFLICT, ()) + } + Err(tenant::CreateTimelineError::Other(err)) => Err(ApiError::InternalServerError(err)), } } .instrument(info_span!("timeline_create", tenant = %tenant_id, timeline_id = %new_timeline_id, lsn=?request_data.ancestor_start_lsn, pg_version=?request_data.pg_version)) .await } -async fn timeline_list_handler(request: Request) -> Result, ApiError> { +async fn timeline_list_handler( + request: Request, + _cancel: CancellationToken, +) -> Result, ApiError> { let tenant_id: TenantId = parse_request_param(&request, "tenant_id")?; let include_non_incremental_logical_size: Option = parse_query_param(&request, "include-non-incremental-logical-size")?; @@ -366,7 +382,10 @@ async fn timeline_list_handler(request: Request) -> Result, json_response(StatusCode::OK, response_data) } -async fn timeline_detail_handler(request: Request) -> Result, ApiError> { +async fn timeline_detail_handler( + request: Request, + _cancel: CancellationToken, +) -> Result, ApiError> { let tenant_id: TenantId = parse_request_param(&request, "tenant_id")?; let timeline_id: TimelineId = parse_request_param(&request, "timeline_id")?; let include_non_incremental_logical_size: Option = @@ -381,7 +400,7 @@ async fn timeline_detail_handler(request: Request) -> Result) -> Result) -> Result, ApiError> { +async fn get_lsn_by_timestamp_handler( + request: Request, + _cancel: CancellationToken, +) -> Result, ApiError> { let tenant_id: TenantId = parse_request_param(&request, "tenant_id")?; check_permission(&request, Some(tenant_id))?; @@ -424,7 +446,10 @@ async fn get_lsn_by_timestamp_handler(request: Request) -> Result) -> Result, ApiError> { +async fn tenant_attach_handler( + mut request: Request, + _cancel: CancellationToken, +) -> Result, ApiError> { let tenant_id: TenantId = parse_request_param(&request, "tenant_id")?; check_permission(&request, Some(tenant_id))?; @@ -460,7 +485,10 @@ async fn tenant_attach_handler(mut request: Request) -> Result) -> Result, ApiError> { +async fn timeline_delete_handler( + request: Request, + _cancel: CancellationToken, +) -> Result, ApiError> { let tenant_id: TenantId = parse_request_param(&request, "tenant_id")?; let timeline_id: TimelineId = parse_request_param(&request, "timeline_id")?; check_permission(&request, Some(tenant_id))?; @@ -471,10 +499,14 @@ async fn timeline_delete_handler(request: Request) -> Result) -> Result, ApiError> { +async fn tenant_detach_handler( + request: Request, + _cancel: CancellationToken, +) -> Result, ApiError> { let tenant_id: TenantId = parse_request_param(&request, "tenant_id")?; check_permission(&request, Some(tenant_id))?; let detach_ignored: Option = parse_query_param(&request, "detach_ignored")?; @@ -488,7 +520,10 @@ async fn tenant_detach_handler(request: Request) -> Result, json_response(StatusCode::OK, ()) } -async fn tenant_load_handler(request: Request) -> Result, ApiError> { +async fn tenant_load_handler( + request: Request, + _cancel: CancellationToken, +) -> Result, ApiError> { let tenant_id: TenantId = parse_request_param(&request, "tenant_id")?; check_permission(&request, Some(tenant_id))?; @@ -508,7 +543,10 @@ async fn tenant_load_handler(request: Request) -> Result, A json_response(StatusCode::ACCEPTED, ()) } -async fn tenant_ignore_handler(request: Request) -> Result, ApiError> { +async fn tenant_ignore_handler( + request: Request, + _cancel: CancellationToken, +) -> Result, ApiError> { let tenant_id: TenantId = parse_request_param(&request, "tenant_id")?; check_permission(&request, Some(tenant_id))?; @@ -521,7 +559,10 @@ async fn tenant_ignore_handler(request: Request) -> Result, json_response(StatusCode::OK, ()) } -async fn tenant_list_handler(request: Request) -> Result, ApiError> { +async fn tenant_list_handler( + request: Request, + _cancel: CancellationToken, +) -> Result, ApiError> { check_permission(&request, None)?; let response_data = mgr::list_tenants() @@ -541,7 +582,10 @@ async fn tenant_list_handler(request: Request) -> Result, A json_response(StatusCode::OK, response_data) } -async fn tenant_status(request: Request) -> Result, ApiError> { +async fn tenant_status( + request: Request, + _cancel: CancellationToken, +) -> Result, ApiError> { let tenant_id: TenantId = parse_request_param(&request, "tenant_id")?; check_permission(&request, Some(tenant_id))?; @@ -551,7 +595,7 @@ async fn tenant_status(request: Request) -> Result, ApiErro // Calculate total physical size of all timelines let mut current_physical_size = 0; for timeline in tenant.list_timelines().iter() { - current_physical_size += timeline.layer_size_sum(); + current_physical_size += timeline.layer_size_sum().await; } let state = tenant.current_state(); @@ -581,7 +625,10 @@ async fn tenant_status(request: Request) -> Result, ApiErro /// Note: we don't update the cached size and prometheus metric here. /// The retention period might be different, and it's nice to have a method to just calculate it /// without modifying anything anyway. -async fn tenant_size_handler(request: Request) -> Result, ApiError> { +async fn tenant_size_handler( + request: Request, + _cancel: CancellationToken, +) -> Result, ApiError> { let tenant_id: TenantId = parse_request_param(&request, "tenant_id")?; check_permission(&request, Some(tenant_id))?; let inputs_only: Option = parse_query_param(&request, "inputs_only")?; @@ -646,7 +693,10 @@ async fn tenant_size_handler(request: Request) -> Result, A ) } -async fn layer_map_info_handler(request: Request) -> Result, ApiError> { +async fn layer_map_info_handler( + request: Request, + _cancel: CancellationToken, +) -> Result, ApiError> { let tenant_id: TenantId = parse_request_param(&request, "tenant_id")?; let timeline_id: TimelineId = parse_request_param(&request, "timeline_id")?; let reset: LayerAccessStatsReset = @@ -655,12 +705,15 @@ async fn layer_map_info_handler(request: Request) -> Result check_permission(&request, Some(tenant_id))?; let timeline = active_timeline_of_active_tenant(tenant_id, timeline_id).await?; - let layer_map_info = timeline.layer_map_info(reset); + let layer_map_info = timeline.layer_map_info(reset).await; json_response(StatusCode::OK, layer_map_info) } -async fn layer_download_handler(request: Request) -> Result, ApiError> { +async fn layer_download_handler( + request: Request, + _cancel: CancellationToken, +) -> Result, ApiError> { let tenant_id: TenantId = parse_request_param(&request, "tenant_id")?; check_permission(&request, Some(tenant_id))?; let timeline_id: TimelineId = parse_request_param(&request, "timeline_id")?; @@ -683,7 +736,10 @@ async fn layer_download_handler(request: Request) -> Result } } -async fn evict_timeline_layer_handler(request: Request) -> Result, ApiError> { +async fn evict_timeline_layer_handler( + request: Request, + _cancel: CancellationToken, +) -> Result, ApiError> { let tenant_id: TenantId = parse_request_param(&request, "tenant_id")?; check_permission(&request, Some(tenant_id))?; let timeline_id: TimelineId = parse_request_param(&request, "timeline_id")?; @@ -761,7 +817,10 @@ pub fn html_response(status: StatusCode, data: String) -> Result, Ok(response) } -async fn tenant_create_handler(mut request: Request) -> Result, ApiError> { +async fn tenant_create_handler( + mut request: Request, + _cancel: CancellationToken, +) -> Result, ApiError> { let request_data: TenantCreateRequest = json_request(&mut request).await?; let target_tenant_id = request_data.new_tenant_id; check_permission(&request, None)?; @@ -808,7 +867,10 @@ async fn tenant_create_handler(mut request: Request) -> Result) -> Result, ApiError> { +async fn get_tenant_config_handler( + request: Request, + _cancel: CancellationToken, +) -> Result, ApiError> { let tenant_id: TenantId = parse_request_param(&request, "tenant_id")?; check_permission(&request, Some(tenant_id))?; @@ -834,6 +896,7 @@ async fn get_tenant_config_handler(request: Request) -> Result, + _cancel: CancellationToken, ) -> Result, ApiError> { let request_data: TenantConfigRequest = json_request(&mut request).await?; let tenant_id = request_data.tenant_id; @@ -851,21 +914,25 @@ async fn update_tenant_config_handler( } /// Testing helper to transition a tenant to [`crate::tenant::TenantState::Broken`]. -#[cfg(feature = "testing")] -async fn handle_tenant_break(r: Request) -> Result, ApiError> { +async fn handle_tenant_break( + r: Request, + _cancel: CancellationToken, +) -> Result, ApiError> { let tenant_id: TenantId = parse_request_param(&r, "tenant_id")?; let tenant = crate::tenant::mgr::get_tenant(tenant_id, true) .await .map_err(|_| ApiError::Conflict(String::from("no active tenant found")))?; - tenant.set_broken("broken from test".to_owned()); + tenant.set_broken("broken from test".to_owned()).await; json_response(StatusCode::OK, ()) } -#[cfg(feature = "testing")] -async fn failpoints_handler(mut request: Request) -> Result, ApiError> { +async fn failpoints_handler( + mut request: Request, + _cancel: CancellationToken, +) -> Result, ApiError> { if !fail::has_failpoints() { return Err(ApiError::BadRequest(anyhow!( "Cannot manage failpoints because pageserver was compiled without failpoints support" @@ -898,7 +965,10 @@ async fn failpoints_handler(mut request: Request) -> Result } // Run GC immediately on given timeline. -async fn timeline_gc_handler(mut request: Request) -> Result, ApiError> { +async fn timeline_gc_handler( + mut request: Request, + _cancel: CancellationToken, +) -> Result, ApiError> { let tenant_id: TenantId = parse_request_param(&request, "tenant_id")?; let timeline_id: TimelineId = parse_request_param(&request, "timeline_id")?; check_permission(&request, Some(tenant_id))?; @@ -917,8 +987,10 @@ async fn timeline_gc_handler(mut request: Request) -> Result) -> Result, ApiError> { +async fn timeline_compact_handler( + request: Request, + _cancel: CancellationToken, +) -> Result, ApiError> { let tenant_id: TenantId = parse_request_param(&request, "tenant_id")?; let timeline_id: TimelineId = parse_request_param(&request, "timeline_id")?; check_permission(&request, Some(tenant_id))?; @@ -939,8 +1011,10 @@ async fn timeline_compact_handler(request: Request) -> Result) -> Result, ApiError> { +async fn timeline_checkpoint_handler( + request: Request, + _cancel: CancellationToken, +) -> Result, ApiError> { let tenant_id: TenantId = parse_request_param(&request, "tenant_id")?; let timeline_id: TimelineId = parse_request_param(&request, "timeline_id")?; check_permission(&request, Some(tenant_id))?; @@ -964,6 +1038,7 @@ async fn timeline_checkpoint_handler(request: Request) -> Result, + _cancel: CancellationToken, ) -> Result, ApiError> { let tenant_id: TenantId = parse_request_param(&request, "tenant_id")?; let timeline_id: TimelineId = parse_request_param(&request, "timeline_id")?; @@ -979,6 +1054,7 @@ async fn timeline_download_remote_layers_handler_post( async fn timeline_download_remote_layers_handler_get( request: Request, + _cancel: CancellationToken, ) -> Result, ApiError> { let tenant_id: TenantId = parse_request_param(&request, "tenant_id")?; check_permission(&request, Some(tenant_id))?; @@ -988,7 +1064,7 @@ async fn timeline_download_remote_layers_handler_get( let info = timeline .get_download_all_remote_layers_task_info() .context("task never started since last pageserver process start") - .map_err(ApiError::NotFound)?; + .map_err(|e| ApiError::NotFound(e.into()))?; json_response(StatusCode::OK, info) } @@ -999,10 +1075,13 @@ async fn active_timeline_of_active_tenant( let tenant = mgr::get_tenant(tenant_id, true).await?; tenant .get_timeline(timeline_id, true) - .map_err(ApiError::NotFound) + .map_err(|e| ApiError::NotFound(e.into())) } -async fn always_panic_handler(req: Request) -> Result, ApiError> { +async fn always_panic_handler( + req: Request, + _cancel: CancellationToken, +) -> Result, ApiError> { // Deliberately cause a panic to exercise the panic hook registered via std::panic::set_hook(). // For pageserver, the relevant panic hook is `tracing_panic_hook` , and the `sentry` crate's wrapper around it. // Use catch_unwind to ensure that tokio nor hyper are distracted by our panic. @@ -1013,7 +1092,10 @@ async fn always_panic_handler(req: Request) -> Result, ApiE json_response(StatusCode::NO_CONTENT, ()) } -async fn disk_usage_eviction_run(mut r: Request) -> Result, ApiError> { +async fn disk_usage_eviction_run( + mut r: Request, + _cancel: CancellationToken, +) -> Result, ApiError> { check_permission(&r, None)?; #[derive(Debug, Clone, Copy, serde::Serialize, serde::Deserialize)] @@ -1049,8 +1131,6 @@ async fn disk_usage_eviction_run(mut r: Request) -> Result, freed_bytes: 0, }; - use crate::task_mgr::MGMT_REQUEST_RUNTIME; - let (tx, rx) = tokio::sync::oneshot::channel(); let state = get_state(&r); @@ -1068,7 +1148,7 @@ async fn disk_usage_eviction_run(mut r: Request) -> Result, let _g = cancel.drop_guard(); crate::task_mgr::spawn( - MGMT_REQUEST_RUNTIME.handle(), + crate::task_mgr::BACKGROUND_RUNTIME.handle(), TaskKind::DiskUsageEviction, None, None, @@ -1103,8 +1183,10 @@ async fn handler_404(_: Request) -> Result, ApiError> { ) } -#[cfg(feature = "testing")] -async fn post_tracing_event_handler(mut r: Request) -> Result, ApiError> { +async fn post_tracing_event_handler( + mut r: Request, + _cancel: CancellationToken, +) -> Result, ApiError> { #[derive(Debug, serde::Deserialize)] #[serde(rename_all = "lowercase")] enum Level { @@ -1134,6 +1216,85 @@ async fn post_tracing_event_handler(mut r: Request) -> Result(request: Request, handler: H) -> Result, ApiError> +where + R: std::future::Future, ApiError>> + Send + 'static, + H: FnOnce(Request, CancellationToken) -> R + Send + Sync + 'static, +{ + // Spawn a new task to handle the request, to protect the handler from unexpected + // async cancellations. Most pageserver functions are not async cancellation safe. + // We arm a drop-guard, so that if Hyper drops the Future, we signal the task + // with the cancellation token. + let token = CancellationToken::new(); + let cancel_guard = token.clone().drop_guard(); + let result = request_span(request, move |r| async { + let handle = tokio::spawn( + async { + let token_cloned = token.clone(); + let result = handler(r, token).await; + if token_cloned.is_cancelled() { + info!("Cancelled request finished"); + } + result + } + .in_current_span(), + ); + + match handle.await { + Ok(result) => result, + Err(e) => { + // The handler task panicked. We have a global panic handler that logs the + // panic with its backtrace, so no need to log that here. Only log a brief + // message to make it clear that we returned the error to the client. + error!("HTTP request handler task panicked: {e:#}"); + + // Don't return an Error here, because then fallback error handler that was + // installed in make_router() will print the error. Instead, construct the + // HTTP error response and return that. + Ok( + ApiError::InternalServerError(anyhow!("HTTP request handler task panicked")) + .into_response(), + ) + } + } + }) + .await; + + cancel_guard.disarm(); + + result +} + +/// Like api_handler, but returns an error response if the server is built without +/// the 'testing' feature. +async fn testing_api_handler( + desc: &str, + request: Request, + handler: H, +) -> Result, ApiError> +where + R: std::future::Future, ApiError>> + Send + 'static, + H: FnOnce(Request, CancellationToken) -> R + Send + Sync + 'static, +{ + if cfg!(feature = "testing") { + api_handler(request, handler).await + } else { + std::future::ready(Err(ApiError::BadRequest(anyhow!( + "Cannot {desc} because pageserver was compiled without testing APIs", + )))) + .await + } +} + pub fn make_router( conf: &'static PageServerConf, launch_ts: &'static LaunchTimestamp, @@ -1163,26 +1324,6 @@ pub fn make_router( .expect("construct launch timestamp header middleware"), ); - macro_rules! testing_api { - ($handler_desc:literal, $handler:path $(,)?) => {{ - #[cfg(not(feature = "testing"))] - async fn cfg_disabled(_req: Request) -> Result, ApiError> { - Err(ApiError::BadRequest(anyhow!(concat!( - "Cannot ", - $handler_desc, - " because pageserver was compiled without testing APIs", - )))) - } - - #[cfg(feature = "testing")] - let handler = $handler; - #[cfg(not(feature = "testing"))] - let handler = cfg_disabled; - - move |r| RequestSpan(handler).handle(r) - }}; - } - Ok(router .data(Arc::new( State::new( @@ -1194,96 +1335,88 @@ pub fn make_router( ) .context("Failed to initialize router state")?, )) - .get("/v1/status", |r| RequestSpan(status_handler).handle(r)) - .put( - "/v1/failpoints", - testing_api!("manage failpoints", failpoints_handler), - ) - .get("/v1/tenant", |r| RequestSpan(tenant_list_handler).handle(r)) - .post("/v1/tenant", |r| { - RequestSpan(tenant_create_handler).handle(r) - }) - .get("/v1/tenant/:tenant_id", |r| { - RequestSpan(tenant_status).handle(r) + .get("/v1/status", |r| api_handler(r, status_handler)) + .put("/v1/failpoints", |r| { + testing_api_handler("manage failpoints", r, failpoints_handler) }) + .get("/v1/tenant", |r| api_handler(r, tenant_list_handler)) + .post("/v1/tenant", |r| api_handler(r, tenant_create_handler)) + .get("/v1/tenant/:tenant_id", |r| api_handler(r, tenant_status)) .get("/v1/tenant/:tenant_id/synthetic_size", |r| { - RequestSpan(tenant_size_handler).handle(r) + api_handler(r, tenant_size_handler) }) .put("/v1/tenant/config", |r| { - RequestSpan(update_tenant_config_handler).handle(r) + api_handler(r, update_tenant_config_handler) }) .get("/v1/tenant/:tenant_id/config", |r| { - RequestSpan(get_tenant_config_handler).handle(r) + api_handler(r, get_tenant_config_handler) }) .get("/v1/tenant/:tenant_id/timeline", |r| { - RequestSpan(timeline_list_handler).handle(r) + api_handler(r, timeline_list_handler) }) .post("/v1/tenant/:tenant_id/timeline", |r| { - RequestSpan(timeline_create_handler).handle(r) + api_handler(r, timeline_create_handler) }) .post("/v1/tenant/:tenant_id/attach", |r| { - RequestSpan(tenant_attach_handler).handle(r) + api_handler(r, tenant_attach_handler) }) .post("/v1/tenant/:tenant_id/detach", |r| { - RequestSpan(tenant_detach_handler).handle(r) + api_handler(r, tenant_detach_handler) }) .post("/v1/tenant/:tenant_id/load", |r| { - RequestSpan(tenant_load_handler).handle(r) + api_handler(r, tenant_load_handler) }) .post("/v1/tenant/:tenant_id/ignore", |r| { - RequestSpan(tenant_ignore_handler).handle(r) + api_handler(r, tenant_ignore_handler) }) .get("/v1/tenant/:tenant_id/timeline/:timeline_id", |r| { - RequestSpan(timeline_detail_handler).handle(r) + api_handler(r, timeline_detail_handler) }) .get( "/v1/tenant/:tenant_id/timeline/:timeline_id/get_lsn_by_timestamp", - |r| RequestSpan(get_lsn_by_timestamp_handler).handle(r), + |r| api_handler(r, get_lsn_by_timestamp_handler), ) .put("/v1/tenant/:tenant_id/timeline/:timeline_id/do_gc", |r| { - RequestSpan(timeline_gc_handler).handle(r) + api_handler(r, timeline_gc_handler) + }) + .put("/v1/tenant/:tenant_id/timeline/:timeline_id/compact", |r| { + testing_api_handler("run timeline compaction", r, timeline_compact_handler) }) - .put( - "/v1/tenant/:tenant_id/timeline/:timeline_id/compact", - testing_api!("run timeline compaction", timeline_compact_handler), - ) .put( "/v1/tenant/:tenant_id/timeline/:timeline_id/checkpoint", - testing_api!("run timeline checkpoint", timeline_checkpoint_handler), + |r| testing_api_handler("run timeline checkpoint", r, timeline_checkpoint_handler), ) .post( "/v1/tenant/:tenant_id/timeline/:timeline_id/download_remote_layers", - |r| RequestSpan(timeline_download_remote_layers_handler_post).handle(r), + |r| api_handler(r, timeline_download_remote_layers_handler_post), ) .get( "/v1/tenant/:tenant_id/timeline/:timeline_id/download_remote_layers", - |r| RequestSpan(timeline_download_remote_layers_handler_get).handle(r), + |r| api_handler(r, timeline_download_remote_layers_handler_get), ) .delete("/v1/tenant/:tenant_id/timeline/:timeline_id", |r| { - RequestSpan(timeline_delete_handler).handle(r) + api_handler(r, timeline_delete_handler) }) .get("/v1/tenant/:tenant_id/timeline/:timeline_id/layer", |r| { - RequestSpan(layer_map_info_handler).handle(r) + api_handler(r, layer_map_info_handler) }) .get( "/v1/tenant/:tenant_id/timeline/:timeline_id/layer/:layer_file_name", - |r| RequestSpan(layer_download_handler).handle(r), + |r| api_handler(r, layer_download_handler), ) .delete( "/v1/tenant/:tenant_id/timeline/:timeline_id/layer/:layer_file_name", - |r| RequestSpan(evict_timeline_layer_handler).handle(r), + |r| api_handler(r, evict_timeline_layer_handler), ) .put("/v1/disk_usage_eviction/run", |r| { - RequestSpan(disk_usage_eviction_run).handle(r) + api_handler(r, disk_usage_eviction_run) + }) + .put("/v1/tenant/:tenant_id/break", |r| { + testing_api_handler("set tenant state to broken", r, handle_tenant_break) + }) + .get("/v1/panic", |r| api_handler(r, always_panic_handler)) + .post("/v1/tracing/event", |r| { + testing_api_handler("emit a tracing event", r, post_tracing_event_handler) }) - .put( - "/v1/tenant/:tenant_id/break", - testing_api!("set tenant state to broken", handle_tenant_break), - ) - .get("/v1/panic", |r| RequestSpan(always_panic_handler).handle(r)) - .post( - "/v1/tracing/event", - testing_api!("emit a tracing event", post_tracing_event_handler), - ) .any(handler_404)) } diff --git a/pageserver/src/import_datadir.rs b/pageserver/src/import_datadir.rs index 936de35eb9..5bff5337bd 100644 --- a/pageserver/src/import_datadir.rs +++ b/pageserver/src/import_datadir.rs @@ -75,12 +75,12 @@ pub async fn import_timeline_from_postgres_datadir( { pg_control = Some(control_file); } - modification.flush()?; + modification.flush().await?; } } // We're done importing all the data files. - modification.commit()?; + modification.commit().await?; // We expect the Postgres server to be shut down cleanly. let pg_control = pg_control.context("pg_control file not found")?; @@ -148,17 +148,17 @@ async fn import_rel( // because there is no guarantee about the order in which we are processing segments. // ignore "relation already exists" error // - // FIXME: use proper error type for this, instead of parsing the error message. - // Or better yet, keep track of which relations we've already created + // FIXME: Keep track of which relations we've already created? // https://github.com/neondatabase/neon/issues/3309 if let Err(e) = modification .put_rel_creation(rel, nblocks as u32, ctx) .await { - if e.to_string().contains("already exists") { - debug!("relation {} already exists. we must be extending it", rel); - } else { - return Err(e); + match e { + RelationError::AlreadyExists => { + debug!("Relation {} already exist. We must be extending it.", rel) + } + _ => return Err(e.into()), } } @@ -359,7 +359,7 @@ pub async fn import_basebackup_from_tar( // We found the pg_control file. pg_control = Some(res); } - modification.flush()?; + modification.flush().await?; } tokio_tar::EntryType::Directory => { debug!("directory {:?}", file_path); @@ -377,7 +377,7 @@ pub async fn import_basebackup_from_tar( // sanity check: ensure that pg_control is loaded let _pg_control = pg_control.context("pg_control file not found")?; - modification.commit()?; + modification.commit().await?; Ok(()) } @@ -594,7 +594,7 @@ async fn import_file( // zenith.signal is not necessarily the last file, that we handle // but it is ok to call `finish_write()`, because final `modification.commit()` // will update lsn once more to the final one. - let writer = modification.tline.writer(); + let writer = modification.tline.writer().await; writer.finish_write(prev_lsn); debug!("imported zenith signal {}", prev_lsn); diff --git a/pageserver/src/lib.rs b/pageserver/src/lib.rs index 36578ee4e0..5831091098 100644 --- a/pageserver/src/lib.rs +++ b/pageserver/src/lib.rs @@ -45,6 +45,7 @@ static ZERO_PAGE: bytes::Bytes = bytes::Bytes::from_static(&[0u8; 8192]); pub use crate::metrics::preinitialize_metrics; +#[tracing::instrument] pub async fn shutdown_pageserver(exit_code: i32) { // Shut down the libpq endpoint task. This prevents new connections from // being accepted. @@ -57,12 +58,6 @@ pub async fn shutdown_pageserver(exit_code: i32) { // the checkpoint and GC tasks. tenant::mgr::shutdown_all_tenants().await; - // Stop syncing with remote storage. - // - // FIXME: Does this wait for the sync tasks to finish syncing what's queued up? - // Should it? - task_mgr::shutdown_tasks(Some(TaskKind::RemoteUploadTask), None, None).await; - // Shut down the HTTP endpoint last, so that you can still check the server's // status while it's shutting down. // FIXME: We should probably stop accepting commands like attach/detach earlier. @@ -137,6 +132,29 @@ pub fn is_uninit_mark(path: &Path) -> bool { } } +/// During pageserver startup, we need to order operations not to exhaust tokio worker threads by +/// blocking. +/// +/// The instances of this value exist only during startup, otherwise `None` is provided, meaning no +/// delaying is needed. +#[derive(Clone)] +pub struct InitializationOrder { + /// Each initial tenant load task carries this until completion. + pub initial_tenant_load: Option, + + /// Barrier for when we can start initial logical size calculations. + pub initial_logical_size_can_start: utils::completion::Barrier, + + /// Each timeline owns a clone of this to be consumed on the initial logical size calculation + /// attempt. It is important to drop this once the attempt has completed. + pub initial_logical_size_attempt: utils::completion::Completion, + + /// Barrier for when we can start any background jobs. + /// + /// This can be broken up later on, but right now there is just one class of a background job. + pub background_jobs_can_start: utils::completion::Barrier, +} + #[cfg(test)] mod backoff_defaults_tests { use super::*; diff --git a/pageserver/src/metrics.rs b/pageserver/src/metrics.rs index 75bea9dbab..00745143bd 100644 --- a/pageserver/src/metrics.rs +++ b/pageserver/src/metrics.rs @@ -1,4 +1,4 @@ -use metrics::core::{AtomicU64, GenericCounter}; +use metrics::metric_vec_duration::DurationResultObserver; use metrics::{ register_counter_vec, register_histogram, register_histogram_vec, register_int_counter, register_int_counter_vec, register_int_gauge, register_int_gauge_vec, register_uint_gauge_vec, @@ -84,22 +84,48 @@ pub static STORAGE_TIME_GLOBAL: Lazy = Lazy::new(|| { .expect("failed to define a metric") }); -// Metrics collected on operations on the storage repository. -static RECONSTRUCT_TIME: Lazy = Lazy::new(|| { +static READ_NUM_FS_LAYERS: Lazy = Lazy::new(|| { register_histogram_vec!( + "pageserver_read_num_fs_layers", + "Number of persistent layers accessed for processing a read request, including those in the cache", + &["tenant_id", "timeline_id"], + vec![1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 10.0, 20.0, 50.0, 100.0], + ) + .expect("failed to define a metric") +}); + +// Metrics collected on operations on the storage repository. +pub static RECONSTRUCT_TIME: Lazy = Lazy::new(|| { + register_histogram!( "pageserver_getpage_reconstruct_seconds", - "Time spent in reconstruct_value", + "Time spent in reconstruct_value (reconstruct a page from deltas)", + CRITICAL_OP_BUCKETS.into(), + ) + .expect("failed to define a metric") +}); + +pub static MATERIALIZED_PAGE_CACHE_HIT_DIRECT: Lazy = Lazy::new(|| { + register_int_counter!( + "pageserver_materialized_cache_hits_direct_total", + "Number of cache hits from materialized page cache without redo", + ) + .expect("failed to define a metric") +}); + +static GET_RECONSTRUCT_DATA_TIME: Lazy = Lazy::new(|| { + register_histogram_vec!( + "pageserver_getpage_get_reconstruct_data_seconds", + "Time spent in get_reconstruct_value_data", &["tenant_id", "timeline_id"], CRITICAL_OP_BUCKETS.into(), ) .expect("failed to define a metric") }); -static MATERIALIZED_PAGE_CACHE_HIT: Lazy = Lazy::new(|| { - register_int_counter_vec!( +pub static MATERIALIZED_PAGE_CACHE_HIT: Lazy = Lazy::new(|| { + register_int_counter!( "pageserver_materialized_cache_hits_total", "Number of cache hits from materialized page cache", - &["tenant_id", "timeline_id"] ) .expect("failed to define a metric") }); @@ -354,6 +380,7 @@ const STORAGE_IO_TIME_BUCKETS: &[f64] = &[ 0.001000, // 1000 usec 0.030, // 30 ms 1.000, // 1000 ms + 30.000, // 30000 ms ]; const STORAGE_IO_TIME_OPERATIONS: &[&str] = &[ @@ -398,6 +425,27 @@ pub static SMGR_QUERY_TIME: Lazy = Lazy::new(|| { .expect("failed to define a metric") }); +pub struct BasebackupQueryTime(HistogramVec); +pub static BASEBACKUP_QUERY_TIME: Lazy = Lazy::new(|| { + BasebackupQueryTime({ + register_histogram_vec!( + "pageserver_basebackup_query_seconds", + "Histogram of basebackup queries durations, by result type", + &["result"], + CRITICAL_OP_BUCKETS.into(), + ) + .expect("failed to define a metric") + }) +}); + +impl DurationResultObserver for BasebackupQueryTime { + fn observe_result(&self, res: &Result, duration: std::time::Duration) { + let label_value = if res.is_ok() { "ok" } else { "error" }; + let metric = self.0.get_metric_with_label_values(&[label_value]).unwrap(); + metric.observe(duration.as_secs_f64()); + } +} + pub static LIVE_CONNECTIONS_COUNT: Lazy = Lazy::new(|| { register_int_gauge_vec!( "pageserver_live_connections", @@ -622,7 +670,7 @@ pub static WAL_REDO_TIME: Lazy = Lazy::new(|| { pub static WAL_REDO_WAIT_TIME: Lazy = Lazy::new(|| { register_histogram!( "pageserver_wal_redo_wait_seconds", - "Time spent waiting for access to the WAL redo process", + "Time spent waiting for access to the Postgres WAL redo process", redo_histogram_time_buckets!(), ) .expect("failed to define a metric") @@ -631,7 +679,7 @@ pub static WAL_REDO_WAIT_TIME: Lazy = Lazy::new(|| { pub static WAL_REDO_RECORDS_HISTOGRAM: Lazy = Lazy::new(|| { register_histogram!( "pageserver_wal_redo_records_histogram", - "Histogram of number of records replayed per redo", + "Histogram of number of records replayed per redo in the Postgres WAL redo process", redo_histogram_count_buckets!(), ) .expect("failed to define a metric") @@ -640,7 +688,7 @@ pub static WAL_REDO_RECORDS_HISTOGRAM: Lazy = Lazy::new(|| { pub static WAL_REDO_BYTES_HISTOGRAM: Lazy = Lazy::new(|| { register_histogram!( "pageserver_wal_redo_bytes_histogram", - "Histogram of number of records replayed per redo", + "Histogram of number of records replayed per redo sent to Postgres", redo_bytes_histogram_count_buckets!(), ) .expect("failed to define a metric") @@ -722,8 +770,7 @@ impl StorageTimeMetrics { pub struct TimelineMetrics { tenant_id: String, timeline_id: String, - pub reconstruct_time_histo: Histogram, - pub materialized_page_cache_hit_counter: GenericCounter, + pub get_reconstruct_data_time_histo: Histogram, pub flush_time_histo: StorageTimeMetrics, pub compact_time_histo: StorageTimeMetrics, pub create_images_time_histo: StorageTimeMetrics, @@ -734,6 +781,7 @@ pub struct TimelineMetrics { pub last_record_gauge: IntGauge, pub wait_lsn_time_histo: Histogram, pub resident_physical_size_gauge: UIntGauge, + pub read_num_fs_layers: Histogram, /// copy of LayeredTimeline.current_logical_size pub current_logical_size_gauge: UIntGauge, pub num_persistent_files_created: IntCounter, @@ -750,10 +798,7 @@ impl TimelineMetrics { ) -> Self { let tenant_id = tenant_id.to_string(); let timeline_id = timeline_id.to_string(); - let reconstruct_time_histo = RECONSTRUCT_TIME - .get_metric_with_label_values(&[&tenant_id, &timeline_id]) - .unwrap(); - let materialized_page_cache_hit_counter = MATERIALIZED_PAGE_CACHE_HIT + let get_reconstruct_data_time_histo = GET_RECONSTRUCT_DATA_TIME .get_metric_with_label_values(&[&tenant_id, &timeline_id]) .unwrap(); let flush_time_histo = @@ -794,14 +839,16 @@ impl TimelineMetrics { let evictions = EVICTIONS .get_metric_with_label_values(&[&tenant_id, &timeline_id]) .unwrap(); + let read_num_fs_layers = READ_NUM_FS_LAYERS + .get_metric_with_label_values(&[&tenant_id, &timeline_id]) + .unwrap(); let evictions_with_low_residence_duration = evictions_with_low_residence_duration_builder.build(&tenant_id, &timeline_id); TimelineMetrics { tenant_id, timeline_id, - reconstruct_time_histo, - materialized_page_cache_hit_counter, + get_reconstruct_data_time_histo, flush_time_histo, compact_time_histo, create_images_time_histo, @@ -819,6 +866,7 @@ impl TimelineMetrics { evictions_with_low_residence_duration: std::sync::RwLock::new( evictions_with_low_residence_duration, ), + read_num_fs_layers, } } } @@ -827,8 +875,7 @@ impl Drop for TimelineMetrics { fn drop(&mut self) { let tenant_id = &self.tenant_id; let timeline_id = &self.timeline_id; - let _ = RECONSTRUCT_TIME.remove_label_values(&[tenant_id, timeline_id]); - let _ = MATERIALIZED_PAGE_CACHE_HIT.remove_label_values(&[tenant_id, timeline_id]); + let _ = GET_RECONSTRUCT_DATA_TIME.remove_label_values(&[tenant_id, timeline_id]); let _ = LAST_RECORD_LSN.remove_label_values(&[tenant_id, timeline_id]); let _ = WAIT_LSN_TIME.remove_label_values(&[tenant_id, timeline_id]); let _ = RESIDENT_PHYSICAL_SIZE.remove_label_values(&[tenant_id, timeline_id]); @@ -836,6 +883,8 @@ impl Drop for TimelineMetrics { let _ = NUM_PERSISTENT_FILES_CREATED.remove_label_values(&[tenant_id, timeline_id]); let _ = PERSISTENT_BYTES_WRITTEN.remove_label_values(&[tenant_id, timeline_id]); let _ = EVICTIONS.remove_label_values(&[tenant_id, timeline_id]); + let _ = READ_NUM_FS_LAYERS.remove_label_values(&[tenant_id, timeline_id]); + self.evictions_with_low_residence_duration .write() .unwrap() @@ -1270,4 +1319,8 @@ pub fn preinitialize_metrics() { // Same as above for this metric, but, it's a Vec-type metric for which we don't know all the labels. BACKGROUND_LOOP_PERIOD_OVERRUN_COUNT.reset(); + + // Python tests need these. + MATERIALIZED_PAGE_CACHE_HIT_DIRECT.get(); + MATERIALIZED_PAGE_CACHE_HIT.get(); } diff --git a/pageserver/src/page_service.rs b/pageserver/src/page_service.rs index 9e9285a009..8728559d72 100644 --- a/pageserver/src/page_service.rs +++ b/pageserver/src/page_service.rs @@ -390,7 +390,9 @@ impl PageServerHandler { }; // Check that the timeline exists - let timeline = tenant.get_timeline(timeline_id, true)?; + let timeline = tenant + .get_timeline(timeline_id, true) + .map_err(|e| anyhow::anyhow!(e))?; // switch client to COPYBOTH pgb.write_message_noflush(&BeMessage::CopyBothResponse)?; @@ -902,7 +904,7 @@ where self.check_permission(Some(tenant_id))?; - let lsn = if params.len() == 3 { + let lsn = if params.len() >= 3 { Some( Lsn::from_str(params[2]) .with_context(|| format!("Failed to parse Lsn from {}", params[2]))?, @@ -911,10 +913,24 @@ where None }; - // Check that the timeline exists - self.handle_basebackup_request(pgb, tenant_id, timeline_id, lsn, None, false, ctx) - .await?; - pgb.write_message_noflush(&BeMessage::CommandComplete(b"SELECT 1"))?; + metrics::metric_vec_duration::observe_async_block_duration_by_result( + &*crate::metrics::BASEBACKUP_QUERY_TIME, + async move { + self.handle_basebackup_request( + pgb, + tenant_id, + timeline_id, + lsn, + None, + false, + ctx, + ) + .await?; + pgb.write_message_noflush(&BeMessage::CommandComplete(b"SELECT 1"))?; + anyhow::Ok(()) + }, + ) + .await?; } // return pair of prev_lsn and last_lsn else if query_string.starts_with("get_last_record_rlsn ") { @@ -1230,6 +1246,6 @@ async fn get_active_tenant_timeline( .map_err(GetActiveTimelineError::Tenant)?; let timeline = tenant .get_timeline(timeline_id, true) - .map_err(GetActiveTimelineError::Timeline)?; + .map_err(|e| GetActiveTimelineError::Timeline(anyhow::anyhow!(e)))?; Ok(timeline) } diff --git a/pageserver/src/pgdatadir_mapping.rs b/pageserver/src/pgdatadir_mapping.rs index 186209dfcf..a54cf9f91b 100644 --- a/pageserver/src/pgdatadir_mapping.rs +++ b/pageserver/src/pgdatadir_mapping.rs @@ -43,6 +43,16 @@ pub enum CalculateLogicalSizeError { Other(#[from] anyhow::Error), } +#[derive(Debug, thiserror::Error)] +pub enum RelationError { + #[error("Relation Already Exists")] + AlreadyExists, + #[error("invalid relnode")] + InvalidRelnode, + #[error(transparent)] + Other(#[from] anyhow::Error), +} + /// /// This impl provides all the functionality to store PostgreSQL relations, SLRUs, /// and other special kinds of files, in a versioned key-value store. The @@ -101,9 +111,9 @@ impl Timeline { ctx: &RequestContext, ) -> Result { if tag.relnode == 0 { - return Err(PageReconstructError::Other(anyhow::anyhow!( - "invalid relnode" - ))); + return Err(PageReconstructError::Other( + RelationError::InvalidRelnode.into(), + )); } let nblocks = self.get_rel_size(tag, lsn, latest, ctx).await?; @@ -148,9 +158,9 @@ impl Timeline { ctx: &RequestContext, ) -> Result { if tag.relnode == 0 { - return Err(PageReconstructError::Other(anyhow::anyhow!( - "invalid relnode" - ))); + return Err(PageReconstructError::Other( + RelationError::InvalidRelnode.into(), + )); } if let Some(nblocks) = self.get_cached_rel_size(&tag, lsn) { @@ -193,9 +203,9 @@ impl Timeline { ctx: &RequestContext, ) -> Result { if tag.relnode == 0 { - return Err(PageReconstructError::Other(anyhow::anyhow!( - "invalid relnode" - ))); + return Err(PageReconstructError::Other( + RelationError::InvalidRelnode.into(), + )); } // first try to lookup relation in cache @@ -699,6 +709,20 @@ impl<'a> DatadirModification<'a> { Ok(()) } + #[cfg(test)] + pub fn init_empty_test_timeline(&mut self) -> anyhow::Result<()> { + self.init_empty()?; + self.put_control_file(bytes::Bytes::from_static( + b"control_file contents do not matter", + )) + .context("put_control_file")?; + self.put_checkpoint(bytes::Bytes::from_static( + b"checkpoint_file contents do not matter", + )) + .context("put_checkpoint_file")?; + Ok(()) + } + /// Put a new page version that can be constructed from a WAL record /// /// NOTE: this will *not* implicitly extend the relation, if the page is beyond the @@ -710,7 +734,7 @@ impl<'a> DatadirModification<'a> { blknum: BlockNumber, rec: NeonWalRecord, ) -> anyhow::Result<()> { - anyhow::ensure!(rel.relnode != 0, "invalid relnode"); + anyhow::ensure!(rel.relnode != 0, RelationError::InvalidRelnode); self.put(rel_block_to_key(rel, blknum), Value::WalRecord(rec)); Ok(()) } @@ -737,7 +761,7 @@ impl<'a> DatadirModification<'a> { blknum: BlockNumber, img: Bytes, ) -> anyhow::Result<()> { - anyhow::ensure!(rel.relnode != 0, "invalid relnode"); + anyhow::ensure!(rel.relnode != 0, RelationError::InvalidRelnode); self.put(rel_block_to_key(rel, blknum), Value::Image(img)); Ok(()) } @@ -861,32 +885,38 @@ impl<'a> DatadirModification<'a> { rel: RelTag, nblocks: BlockNumber, ctx: &RequestContext, - ) -> anyhow::Result<()> { - anyhow::ensure!(rel.relnode != 0, "invalid relnode"); + ) -> Result<(), RelationError> { + if rel.relnode == 0 { + return Err(RelationError::InvalidRelnode); + } // It's possible that this is the first rel for this db in this // tablespace. Create the reldir entry for it if so. - let mut dbdir = DbDirectory::des(&self.get(DBDIR_KEY, ctx).await?)?; + let mut dbdir = DbDirectory::des(&self.get(DBDIR_KEY, ctx).await.context("read db")?) + .context("deserialize db")?; let rel_dir_key = rel_dir_to_key(rel.spcnode, rel.dbnode); let mut rel_dir = if dbdir.dbdirs.get(&(rel.spcnode, rel.dbnode)).is_none() { // Didn't exist. Update dbdir dbdir.dbdirs.insert((rel.spcnode, rel.dbnode), false); - let buf = DbDirectory::ser(&dbdir)?; + let buf = DbDirectory::ser(&dbdir).context("serialize db")?; self.put(DBDIR_KEY, Value::Image(buf.into())); // and create the RelDirectory RelDirectory::default() } else { // reldir already exists, fetch it - RelDirectory::des(&self.get(rel_dir_key, ctx).await?)? + RelDirectory::des(&self.get(rel_dir_key, ctx).await.context("read db")?) + .context("deserialize db")? }; // Add the new relation to the rel directory entry, and write it back if !rel_dir.rels.insert((rel.relnode, rel.forknum)) { - anyhow::bail!("rel {rel} already exists"); + return Err(RelationError::AlreadyExists); } self.put( rel_dir_key, - Value::Image(Bytes::from(RelDirectory::ser(&rel_dir)?)), + Value::Image(Bytes::from( + RelDirectory::ser(&rel_dir).context("serialize")?, + )), ); // Put size @@ -911,7 +941,7 @@ impl<'a> DatadirModification<'a> { nblocks: BlockNumber, ctx: &RequestContext, ) -> anyhow::Result<()> { - anyhow::ensure!(rel.relnode != 0, "invalid relnode"); + anyhow::ensure!(rel.relnode != 0, RelationError::InvalidRelnode); let last_lsn = self.tline.get_last_record_lsn(); if self.tline.get_rel_exists(rel, last_lsn, true, ctx).await? { let size_key = rel_size_to_key(rel); @@ -942,7 +972,7 @@ impl<'a> DatadirModification<'a> { nblocks: BlockNumber, ctx: &RequestContext, ) -> anyhow::Result<()> { - anyhow::ensure!(rel.relnode != 0, "invalid relnode"); + anyhow::ensure!(rel.relnode != 0, RelationError::InvalidRelnode); // Put size let size_key = rel_size_to_key(rel); @@ -963,7 +993,7 @@ impl<'a> DatadirModification<'a> { /// Drop a relation. pub async fn put_rel_drop(&mut self, rel: RelTag, ctx: &RequestContext) -> anyhow::Result<()> { - anyhow::ensure!(rel.relnode != 0, "invalid relnode"); + anyhow::ensure!(rel.relnode != 0, RelationError::InvalidRelnode); // Remove it from the directory entry let dir_key = rel_dir_to_key(rel.spcnode, rel.dbnode); @@ -1108,7 +1138,7 @@ impl<'a> DatadirModification<'a> { /// retains all the metadata, but data pages are flushed. That's again OK /// for bulk import, where you are just loading data pages and won't try to /// modify the same pages twice. - pub fn flush(&mut self) -> anyhow::Result<()> { + pub async fn flush(&mut self) -> anyhow::Result<()> { // Unless we have accumulated a decent amount of changes, it's not worth it // to scan through the pending_updates list. let pending_nblocks = self.pending_nblocks; @@ -1116,19 +1146,20 @@ impl<'a> DatadirModification<'a> { return Ok(()); } - let writer = self.tline.writer(); + let writer = self.tline.writer().await; // Flush relation and SLRU data blocks, keep metadata. - let mut result: anyhow::Result<()> = Ok(()); - self.pending_updates.retain(|&key, value| { - if result.is_ok() && (is_rel_block_key(key) || is_slru_block_key(key)) { - result = writer.put(key, self.lsn, value); - false + let mut retained_pending_updates = HashMap::new(); + for (key, value) in self.pending_updates.drain() { + if is_rel_block_key(key) || is_slru_block_key(key) { + // This bails out on first error without modifying pending_updates. + // That's Ok, cf this function's doc comment. + writer.put(key, self.lsn, &value).await?; } else { - true + retained_pending_updates.insert(key, value); } - }); - result?; + } + self.pending_updates.extend(retained_pending_updates); if pending_nblocks != 0 { writer.update_current_logical_size(pending_nblocks * i64::from(BLCKSZ)); @@ -1143,17 +1174,17 @@ impl<'a> DatadirModification<'a> { /// underlying timeline. /// All the modifications in this atomic update are stamped by the specified LSN. /// - pub fn commit(&mut self) -> anyhow::Result<()> { - let writer = self.tline.writer(); + pub async fn commit(&mut self) -> anyhow::Result<()> { + let writer = self.tline.writer().await; let lsn = self.lsn; let pending_nblocks = self.pending_nblocks; self.pending_nblocks = 0; for (key, value) in self.pending_updates.drain() { - writer.put(key, lsn, &value)?; + writer.put(key, lsn, &value).await?; } for key_range in self.pending_deletions.drain(..) { - writer.delete(key_range, lsn)?; + writer.delete(key_range, lsn).await?; } writer.finish_write(lsn); @@ -1593,20 +1624,6 @@ fn is_slru_block_key(key: Key) -> bool { && key.field6 != 0xffffffff // and not SlruSegSize } -#[cfg(test)] -pub fn create_test_timeline( - tenant: &crate::tenant::Tenant, - timeline_id: utils::id::TimelineId, - pg_version: u32, - ctx: &RequestContext, -) -> anyhow::Result> { - let tline = tenant.create_test_timeline(timeline_id, Lsn(8), pg_version, ctx)?; - let mut m = tline.begin_modification(Lsn(8)); - m.init_empty()?; - m.commit()?; - Ok(tline) -} - #[allow(clippy::bool_assert_comparison)] #[cfg(test)] mod tests { diff --git a/pageserver/src/task_mgr.rs b/pageserver/src/task_mgr.rs index 82aebc6c07..13db38d956 100644 --- a/pageserver/src/task_mgr.rs +++ b/pageserver/src/task_mgr.rs @@ -257,6 +257,9 @@ pub enum TaskKind { // task that handles attaching a tenant Attach, + // Used mostly for background deletion from s3 + TimelineDeletionWorker, + // task that handhes metrics collection MetricsCollection, @@ -476,27 +479,44 @@ pub async fn shutdown_tasks( && (timeline_id.is_none() || task_mut.timeline_id == timeline_id) { task.cancel.cancel(); - victim_tasks.push(Arc::clone(task)); + victim_tasks.push(( + Arc::clone(task), + task.kind, + task_mut.tenant_id, + task_mut.timeline_id, + )); } } } - for task in victim_tasks { + let log_all = kind.is_none() && tenant_id.is_none() && timeline_id.is_none(); + + for (task, task_kind, tenant_id, timeline_id) in victim_tasks { let join_handle = { let mut task_mut = task.mutable.lock().unwrap(); task_mut.join_handle.take() }; if let Some(mut join_handle) = join_handle { - let completed = tokio::select! { - _ = &mut join_handle => { true }, + if log_all { + if tenant_id.is_none() { + // there are quite few of these + info!(name = task.name, kind = ?task_kind, "stopping global task"); + } else { + // warn to catch these in tests; there shouldn't be any + warn!(name = task.name, tenant_id = ?tenant_id, timeline_id = ?timeline_id, kind = ?task_kind, "stopping left-over"); + } + } + let join_handle = tokio::select! { + biased; + _ = &mut join_handle => { None }, _ = tokio::time::sleep(std::time::Duration::from_secs(1)) => { // allow some time to elapse before logging to cut down the number of log // lines. info!("waiting for {} to shut down", task.name); - false + Some(join_handle) } }; - if !completed { + if let Some(join_handle) = join_handle { // we never handled this return value, but: // - we don't deschedule which would lead to is_cancelled // - panics are already logged (is_panicked) diff --git a/pageserver/src/tenant.rs b/pageserver/src/tenant.rs index 2827830f02..339291149f 100644 --- a/pageserver/src/tenant.rs +++ b/pageserver/src/tenant.rs @@ -11,15 +11,17 @@ //! parent timeline, and the last LSN that has been written to disk. //! -use anyhow::{bail, Context}; +use anyhow::{bail, ensure, Context}; use futures::FutureExt; use pageserver_api::models::TimelineState; use remote_storage::DownloadError; use remote_storage::GenericRemoteStorage; use storage_broker::BrokerClientChannel; use tokio::sync::watch; +use tokio::sync::OwnedMutexGuard; use tokio::task::JoinSet; use tracing::*; +use utils::completion; use utils::crashsafe::path_with_suffix_extension; use std::cmp::min; @@ -64,6 +66,7 @@ use crate::tenant::remote_timeline_client::PersistIndexPartWithDeletedFlagError; use crate::tenant::storage_layer::DeltaLayer; use crate::tenant::storage_layer::ImageLayer; use crate::tenant::storage_layer::Layer; +use crate::InitializationOrder; use crate::virtual_file::VirtualFile; use crate::walredo::PostgresRedoManager; @@ -83,6 +86,7 @@ pub mod block_io; pub mod disk_btree; pub(crate) mod ephemeral_file; pub mod layer_map; +pub mod manifest; pub mod metadata; mod par_fsync; @@ -182,18 +186,13 @@ struct TimelineUninitMark { } impl UninitializedTimeline<'_> { - /// Ensures timeline data is valid, loads it into pageserver's memory and removes - /// uninit mark file on success. + /// Finish timeline creation: insert it into the Tenant's timelines map and remove the + /// uninit mark file. /// /// This function launches the flush loop if not already done. /// /// The caller is responsible for activating the timeline (function `.activate()`). - fn initialize_with_lock( - mut self, - _ctx: &RequestContext, - timelines: &mut HashMap>, - load_layer_map: bool, - ) -> anyhow::Result> { + fn finish_creation(mut self) -> anyhow::Result> { let timeline_id = self.timeline_id; let tenant_id = self.owning_tenant.tenant_id; @@ -201,25 +200,19 @@ impl UninitializedTimeline<'_> { format!("No timeline for initalization found for {tenant_id}/{timeline_id}") })?; + // Check that the caller initialized disk_consistent_lsn let new_disk_consistent_lsn = new_timeline.get_disk_consistent_lsn(); - // TODO it would be good to ensure that, but apparently a lot of our testing is dependend on that at least - // ensure!(new_disk_consistent_lsn.is_valid(), - // "Timeline {tenant_id}/{timeline_id} has invalid disk_consistent_lsn and cannot be initialized"); + ensure!( + new_disk_consistent_lsn.is_valid(), + "new timeline {tenant_id}/{timeline_id} has invalid disk_consistent_lsn" + ); + let mut timelines = self.owning_tenant.timelines.lock().unwrap(); match timelines.entry(timeline_id) { Entry::Occupied(_) => anyhow::bail!( "Found freshly initialized timeline {tenant_id}/{timeline_id} in the tenant map" ), Entry::Vacant(v) => { - if load_layer_map { - new_timeline - .load_layer_map(new_disk_consistent_lsn) - .with_context(|| { - format!( - "Failed to load layermap for timeline {tenant_id}/{timeline_id}" - ) - })?; - } uninit_mark.remove_uninit_mark().with_context(|| { format!( "Failed to remove uninit mark file for timeline {tenant_id}/{timeline_id}" @@ -248,9 +241,10 @@ impl UninitializedTimeline<'_> { .await .context("Failed to import basebackup")?; + // Flush the new layer files to disk, before we make the timeline as available to + // the outside world. + // // Flush loop needs to be spawned in order to be able to flush. - // We want to run proper checkpoint before we mark timeline as available to outside world - // Thus spawning flush loop manually and skipping flush_loop setup in initialize_with_lock raw_timeline.maybe_spawn_flush_loop(); fail::fail_point!("before-checkpoint-new-timeline", |_| { @@ -262,11 +256,10 @@ impl UninitializedTimeline<'_> { .await .context("Failed to flush after basebackup import")?; - // Initialize without loading the layer map. We started with an empty layer map, and already - // updated it for the layers that we created during the import. - let mut timelines = self.owning_tenant.timelines.lock().unwrap(); - let tl = self.initialize_with_lock(ctx, &mut timelines, false)?; - tl.activate(broker_client, ctx); + // All the data has been imported. Insert the Timeline into the tenant's timelines + // map and remove the uninit mark file. + let tl = self.finish_creation()?; + tl.activate(broker_client, None, ctx); Ok(tl) } @@ -308,15 +301,6 @@ fn cleanup_timeline_directory(uninit_mark: TimelineUninitMark) { } impl TimelineUninitMark { - /// Useful for initializing timelines, existing on disk after the restart. - pub fn dummy() -> Self { - Self { - uninit_mark_deleted: true, - uninit_mark_path: PathBuf::new(), - timeline_path: PathBuf::new(), - } - } - fn new(uninit_mark_path: PathBuf, timeline_path: PathBuf) -> Self { Self { uninit_mark_deleted: false, @@ -437,16 +421,41 @@ remote: } } +#[derive(Debug, thiserror::Error, PartialEq, Eq)] +pub enum GetTimelineError { + #[error("Timeline {tenant_id}/{timeline_id} is not active, state: {state:?}")] + NotActive { + tenant_id: TenantId, + timeline_id: TimelineId, + state: TimelineState, + }, + #[error("Timeline {tenant_id}/{timeline_id} was not found")] + NotFound { + tenant_id: TenantId, + timeline_id: TimelineId, + }, +} + #[derive(Debug, thiserror::Error)] pub enum DeleteTimelineError { #[error("NotFound")] NotFound, + #[error("HasChildren")] - HasChildren, + HasChildren(Vec), + + #[error("Timeline deletion is already in progress")] + AlreadyInProgress, + #[error(transparent)] Other(#[from] anyhow::Error), } +pub enum SetStoppingError { + AlreadyStopping, + Broken, +} + struct RemoteStartupData { index_part: IndexPart, remote_metadata: TimelineMetadata, @@ -480,6 +489,26 @@ impl std::fmt::Display for WaitToBecomeActiveError { } } +pub(crate) enum ShutdownError { + AlreadyStopping, +} + +struct DeletionGuard(OwnedMutexGuard); + +impl DeletionGuard { + fn is_deleted(&self) -> bool { + *self.0 + } +} + +#[derive(thiserror::Error, Debug)] +pub enum CreateTimelineError { + #[error("a timeline with the given ID already exists")] + AlreadyExists, + #[error(transparent)] + Other(#[from] anyhow::Error), +} + impl Tenant { /// Yet another helper for timeline initialization. /// Contains the common part of `load_local_timeline` and `load_remote_timeline`. @@ -500,7 +529,8 @@ impl Tenant { local_metadata: Option, ancestor: Option>, first_save: bool, - ctx: &RequestContext, + init_order: Option<&InitializationOrder>, + _ctx: &RequestContext, ) -> anyhow::Result<()> { let tenant_id = self.tenant_id; @@ -511,52 +541,39 @@ impl Tenant { .context("merge_local_remote_metadata")? .to_owned(); - let timeline = { + let timeline = self.create_timeline_struct( + timeline_id, + up_to_date_metadata, + ancestor.clone(), + remote_client, + init_order, + )?; + let new_disk_consistent_lsn = timeline.get_disk_consistent_lsn(); + anyhow::ensure!( + new_disk_consistent_lsn.is_valid(), + "Timeline {tenant_id}/{timeline_id} has invalid disk_consistent_lsn" + ); + timeline + .load_layer_map(new_disk_consistent_lsn) + .await + .with_context(|| { + format!("Failed to load layermap for timeline {tenant_id}/{timeline_id}") + })?; + + { // avoiding holding it across awaits let mut timelines_accessor = self.timelines.lock().unwrap(); - if timelines_accessor.contains_key(&timeline_id) { - anyhow::bail!( - "Timeline {tenant_id}/{timeline_id} already exists in the tenant map" - ); - } - - let dummy_timeline = self.create_timeline_data( - timeline_id, - up_to_date_metadata, - ancestor.clone(), - remote_client, - )?; - - let timeline = UninitializedTimeline { - owning_tenant: self, - timeline_id, - raw_timeline: Some((dummy_timeline, TimelineUninitMark::dummy())), - }; - // Do not start walreceiver here. We do need loaded layer map for reconcile_with_remote - // But we shouldnt start walreceiver before we have all the data locally, because working walreceiver - // will ingest data which may require looking at the layers which are not yet available locally - match timeline.initialize_with_lock(ctx, &mut timelines_accessor, true) { - Ok(new_timeline) => new_timeline, - Err(e) => { - error!("Failed to initialize timeline {tenant_id}/{timeline_id}: {e:?}"); - // FIXME using None is a hack, it wont hurt, just ugly. - // Ideally initialize_with_lock error should return timeline in the error - // Or return ownership of itself completely so somethin like into_broken - // can be called directly on Uninitielized timeline - // also leades to redundant .clone - let broken_timeline = self - .create_timeline_data( - timeline_id, - up_to_date_metadata, - ancestor.clone(), - None, - ) - .with_context(|| { - format!("creating broken timeline data for {tenant_id}/{timeline_id}") - })?; - broken_timeline.set_state(TimelineState::Broken); - timelines_accessor.insert(timeline_id, broken_timeline); - return Err(e); + match timelines_accessor.entry(timeline_id) { + Entry::Occupied(_) => { + // The uninit mark file acts as a lock that prevents another task from + // initializing the timeline at the same time. + unreachable!( + "Timeline {tenant_id}/{timeline_id} already exists in the tenant map" + ); + } + Entry::Vacant(v) => { + v.insert(Arc::clone(&timeline)); + timeline.maybe_spawn_flush_loop(); } } }; @@ -580,7 +597,8 @@ impl Tenant { || timeline .layers .read() - .unwrap() + .await + .0 .iter_historic_layers() .next() .is_some(), @@ -645,16 +663,17 @@ impl Tenant { "attach tenant", false, async move { - let doit = async { - tenant_clone.attach(&ctx).await?; - tenant_clone.activate(broker_client, &ctx)?; - anyhow::Ok(()) - }; - match doit.await { - Ok(_) => {} + match tenant_clone.attach(&ctx).await { + Ok(()) => { + info!("attach finished, activating"); + tenant_clone.activate(broker_client, None, &ctx); + } Err(e) => { - tenant_clone.set_broken(e.to_string()); - error!("error attaching tenant: {:?}", e); + error!("attach failed, setting tenant state to Broken: {:?}", e); + tenant_clone.state.send_modify(|state| { + assert_eq!(*state, TenantState::Attaching, "the attach task owns the tenant state until activation is complete"); + *state = TenantState::broken_from_reason(e.to_string()); + }); } } Ok(()) @@ -671,6 +690,8 @@ impl Tenant { /// /// Background task that downloads all data for a tenant and brings it to Active state. /// + /// No background tasks are started as part of this routine. + /// async fn attach(self: &Arc, ctx: &RequestContext) -> anyhow::Result<()> { debug_assert_current_span_has_tenant_id(); @@ -746,7 +767,7 @@ impl Tenant { ); remote_index_and_client.insert(timeline_id, (index_part, client)); } - MaybeDeletedIndexPart::Deleted => { + MaybeDeletedIndexPart::Deleted(_) => { info!("timeline {} is deleted, skipping", timeline_id); continue; } @@ -845,6 +866,7 @@ impl Tenant { local_metadata, ancestor, true, + None, ctx, ) .await @@ -870,7 +892,6 @@ impl Tenant { )) } - /// /// Load a tenant that's available on local disk /// /// This is used at pageserver startup, to rebuild the in-memory @@ -880,15 +901,17 @@ impl Tenant { /// /// If the loading fails for some reason, the Tenant will go into Broken /// state. - /// - #[instrument(skip(conf, remote_storage, ctx), fields(tenant_id=%tenant_id))] + #[instrument(skip_all, fields(tenant_id=%tenant_id))] pub fn spawn_load( conf: &'static PageServerConf, tenant_id: TenantId, broker_client: storage_broker::BrokerClientChannel, remote_storage: Option, + init_order: Option, ctx: &RequestContext, ) -> Arc { + debug_assert_current_span_has_tenant_id(); + let tenant_conf = match Self::load_tenant_config(conf, tenant_id) { Ok(conf) => conf, Err(e) => { @@ -920,20 +943,27 @@ impl Tenant { "initial tenant load", false, async move { - let doit = async { - tenant_clone.load(&ctx).await?; - tenant_clone.activate(broker_client, &ctx)?; - anyhow::Ok(()) - }; - match doit.await { - Ok(()) => {} + let mut init_order = init_order; + + // take the completion because initial tenant loading will complete when all of + // these tasks complete. + let _completion = init_order.as_mut().and_then(|x| x.initial_tenant_load.take()); + + match tenant_clone.load(init_order.as_ref(), &ctx).await { + Ok(()) => { + debug!("load finished, activating"); + let background_jobs_can_start = init_order.as_ref().map(|x| &x.background_jobs_can_start); + tenant_clone.activate(broker_client, background_jobs_can_start, &ctx); + } Err(err) => { - tenant_clone.set_broken(err.to_string()); - error!("could not load tenant {tenant_id}: {err:?}"); + error!("load failed, setting tenant state to Broken: {err:?}"); + tenant_clone.state.send_modify(|state| { + assert_eq!(*state, TenantState::Loading, "the loading task owns the tenant state until activation is complete"); + *state = TenantState::broken_from_reason(err.to_string()); + }); } } - info!("initial load for tenant {tenant_id} finished!"); - Ok(()) + Ok(()) } .instrument({ let span = tracing::info_span!(parent: None, "load", tenant_id=%tenant_id); @@ -942,39 +972,19 @@ impl Tenant { }), ); - info!("spawned load into background"); - tenant } - /// - /// Background task to load in-memory data structures for this tenant, from - /// files on disk. Used at pageserver startup. - /// - async fn load(self: &Arc, ctx: &RequestContext) -> anyhow::Result<()> { - debug_assert_current_span_has_tenant_id(); - - info!("loading tenant task"); - - utils::failpoint_sleep_millis_async!("before-loading-tenant"); - - // TODO split this into two functions, scan and actual load - - // Load in-memory state to reflect the local files on disk - // - // Scan the directory, peek into the metadata file of each timeline, and - // collect a list of timelines and their ancestors. - let mut timelines_to_load: HashMap = HashMap::new(); + pub fn scan_and_sort_timelines_dir( + self: Arc, + ) -> anyhow::Result> { let timelines_dir = self.conf.timelines_path(&self.tenant_id); - for entry in std::fs::read_dir(&timelines_dir).with_context(|| { - format!( - "Failed to list timelines directory for tenant {}", - self.tenant_id - ) - })? { - let entry = entry.with_context(|| { - format!("cannot read timeline dir entry for {}", self.tenant_id) - })?; + let mut timelines_to_load: HashMap = HashMap::new(); + + for entry in + std::fs::read_dir(&timelines_dir).context("list timelines directory for tenant")? + { + let entry = entry.context("read timeline dir entry")?; let timeline_dir = entry.path(); if crate::is_temporary(&timeline_dir) { @@ -990,6 +1000,13 @@ impl Tenant { ); } } else if is_uninit_mark(&timeline_dir) { + if !timeline_dir.exists() { + warn!( + "Timeline dir entry become invalid: {}", + timeline_dir.display() + ); + continue; + } let timeline_uninit_mark_file = &timeline_dir; info!( "Found an uninit mark file {}, removing the timeline and its uninit mark", @@ -1013,6 +1030,13 @@ impl Tenant { error!("Failed to clean up uninit marked timeline: {e:?}"); } } else { + if !timeline_dir.exists() { + warn!( + "Timeline dir entry become invalid: {}", + timeline_dir.display() + ); + continue; + } let timeline_id = timeline_dir .file_name() .and_then(OsStr::to_str) @@ -1029,8 +1053,8 @@ impl Tenant { .timeline_uninit_mark_file_path(self.tenant_id, timeline_id); if timeline_uninit_mark_file.exists() { info!( - "Found an uninit mark file for timeline {}/{}, removing the timeline and its uninit mark", - self.tenant_id, timeline_id + %timeline_id, + "Found an uninit mark file, removing the timeline and its uninit mark", ); if let Err(e) = remove_timeline_and_uninit_mark(&timeline_dir, &timeline_uninit_mark_file) @@ -1059,17 +1083,50 @@ impl Tenant { // Sort the array of timeline IDs into tree-order, so that parent comes before // all its children. - let sorted_timelines = tree_sort_timelines(timelines_to_load)?; + tree_sort_timelines(timelines_to_load) + } + + /// + /// Background task to load in-memory data structures for this tenant, from + /// files on disk. Used at pageserver startup. + /// + /// No background tasks are started as part of this routine. + async fn load( + self: &Arc, + init_order: Option<&InitializationOrder>, + ctx: &RequestContext, + ) -> anyhow::Result<()> { + debug_assert_current_span_has_tenant_id(); + + debug!("loading tenant task"); + + utils::failpoint_sleep_millis_async!("before-loading-tenant"); + + // Load in-memory state to reflect the local files on disk + // + // Scan the directory, peek into the metadata file of each timeline, and + // collect a list of timelines and their ancestors. + let span = info_span!("blocking"); + let cloned = Arc::clone(self); + + let sorted_timelines: Vec<(_, _)> = tokio::task::spawn_blocking(move || { + let _g = span.entered(); + cloned.scan_and_sort_timelines_dir() + }) + .await + .context("load spawn_blocking") + .and_then(|res| res)?; + // FIXME original collect_timeline_files contained one more check: // 1. "Timeline has no ancestor and no layer files" for (timeline_id, local_metadata) in sorted_timelines { - self.load_local_timeline(timeline_id, local_metadata, ctx) + self.load_local_timeline(timeline_id, local_metadata, init_order, ctx) .await .with_context(|| format!("load local timeline {timeline_id}"))?; } - info!("Done"); + trace!("Done"); Ok(()) } @@ -1077,11 +1134,12 @@ impl Tenant { /// Subroutine of `load_tenant`, to load an individual timeline /// /// NB: The parent is assumed to be already loaded! - #[instrument(skip_all, fields(timeline_id))] + #[instrument(skip(self, local_metadata, init_order, ctx))] async fn load_local_timeline( - &self, + self: &Arc, timeline_id: TimelineId, local_metadata: TimelineMetadata, + init_order: Option<&InitializationOrder>, ctx: &RequestContext, ) -> anyhow::Result<()> { debug_assert_current_span_has_tenant_id(); @@ -1095,12 +1153,20 @@ impl Tenant { ) }); - let remote_startup_data = match &remote_client { + let ancestor = if let Some(ancestor_timeline_id) = local_metadata.ancestor_timeline() { + let ancestor_timeline = self.get_timeline(ancestor_timeline_id, false) + .with_context(|| anyhow::anyhow!("cannot find ancestor timeline {ancestor_timeline_id} for timeline {timeline_id}"))?; + Some(ancestor_timeline) + } else { + None + }; + + let (remote_startup_data, remote_client) = match remote_client { Some(remote_client) => match remote_client.download_index_file().await { Ok(index_part) => { let index_part = match index_part { MaybeDeletedIndexPart::IndexPart(index_part) => index_part, - MaybeDeletedIndexPart::Deleted => { + MaybeDeletedIndexPart::Deleted(index_part) => { // TODO: we won't reach here if remote storage gets de-configured after start of the deletion operation. // Example: // start deletion operation @@ -1111,37 +1177,63 @@ impl Tenant { // // We don't really anticipate remote storage to be de-configured, so, for now, this is fine. // Also, maybe we'll remove that option entirely in the future, see https://github.com/neondatabase/neon/issues/4099. - info!("is_deleted is set on remote, resuming removal of local data originally done by timeline deletion handler"); - std::fs::remove_dir_all( - self.conf.timeline_path(&timeline_id, &self.tenant_id), - ) - .context("remove_dir_all")?; + info!("is_deleted is set on remote, resuming removal of timeline data originally done by timeline deletion handler"); + + remote_client + .init_upload_queue_stopped_to_continue_deletion(&index_part)?; + + let timeline = self + .create_timeline_struct( + timeline_id, + &local_metadata, + ancestor, + Some(remote_client), + init_order, + ) + .context("create_timeline_struct")?; + + let guard = DeletionGuard( + Arc::clone(&timeline.delete_lock) + .try_lock_owned() + .expect("cannot happen because we're the only owner"), + ); + + // Note: here we even skip populating layer map. Timeline is essentially uninitialized. + // RemoteTimelineClient is the only functioning part. + timeline.set_state(TimelineState::Stopping); + // We meed to do this because when console retries delete request we shouldnt answer with 404 + // because 404 means successful deletion. + // FIXME consider TimelineState::Deleting. + let mut locked = self.timelines.lock().unwrap(); + locked.insert(timeline_id, Arc::clone(&timeline)); + + Tenant::schedule_delete_timeline( + Arc::clone(self), + timeline_id, + timeline, + guard, + ); return Ok(()); } }; let remote_metadata = index_part.parse_metadata().context("parse_metadata")?; - Some(RemoteStartupData { - index_part, - remote_metadata, - }) + ( + Some(RemoteStartupData { + index_part, + remote_metadata, + }), + Some(remote_client), + ) } Err(DownloadError::NotFound) => { info!("no index file was found on the remote"); - None + (None, Some(remote_client)) } Err(e) => return Err(anyhow::anyhow!(e)), }, - None => None, - }; - - let ancestor = if let Some(ancestor_timeline_id) = local_metadata.ancestor_timeline() { - let ancestor_timeline = self.get_timeline(ancestor_timeline_id, false) - .with_context(|| anyhow::anyhow!("cannot find ancestor timeline {ancestor_timeline_id} for timeline {timeline_id}"))?; - Some(ancestor_timeline) - } else { - None + None => (None, remote_client), }; self.timeline_init_and_sync( @@ -1151,6 +1243,7 @@ impl Tenant { Some(local_metadata), ancestor, false, + init_order, ctx, ) .await @@ -1166,19 +1259,21 @@ impl Tenant { &self, timeline_id: TimelineId, active_only: bool, - ) -> anyhow::Result> { + ) -> Result, GetTimelineError> { let timelines_accessor = self.timelines.lock().unwrap(); - let timeline = timelines_accessor.get(&timeline_id).with_context(|| { - format!("Timeline {}/{} was not found", self.tenant_id, timeline_id) - })?; + let timeline = timelines_accessor + .get(&timeline_id) + .ok_or(GetTimelineError::NotFound { + tenant_id: self.tenant_id, + timeline_id, + })?; if active_only && !timeline.is_active() { - anyhow::bail!( - "Timeline {}/{} is not active, state: {:?}", - self.tenant_id, + Err(GetTimelineError::NotActive { + tenant_id: self.tenant_id, timeline_id, - timeline.current_state() - ) + state: timeline.current_state(), + }) } else { Ok(Arc::clone(timeline)) } @@ -1198,6 +1293,18 @@ impl Tenant { /// This is used to create the initial 'main' timeline during bootstrapping, /// or when importing a new base backup. The caller is expected to load an /// initial image of the datadir to the new timeline after this. + /// + /// Until that happens, the on-disk state is invalid (disk_consistent_lsn=Lsn(0)) + /// and the timeline will fail to load at a restart. + /// + /// That's why we add an uninit mark file, and wrap it together witht the Timeline + /// in-memory object into UninitializedTimeline. + /// Once the caller is done setting up the timeline, they should call + /// `UninitializedTimeline::initialize_with_lock` to remove the uninit mark. + /// + /// For tests, use `DatadirModification::init_empty_test_timeline` + `commit` to setup the + /// minimum amount of keys required to get a writable timeline. + /// (Without it, `put` might fail due to `repartition` failing.) pub fn create_empty_timeline( &self, new_timeline_id: TimelineId, @@ -1215,6 +1322,8 @@ impl Tenant { drop(timelines); let new_metadata = TimelineMetadata::new( + // Initialize disk_consistent LSN to 0, The caller must import some data to + // make it valid, before calling finish_creation() Lsn(0), None, None, @@ -1223,11 +1332,11 @@ impl Tenant { initdb_lsn, pg_version, ); - self.prepare_timeline( + self.prepare_new_timeline( new_timeline_id, &new_metadata, timeline_uninit_mark, - true, + initdb_lsn, None, ) } @@ -1238,7 +1347,7 @@ impl Tenant { // This makes the various functions which anyhow::ensure! for Active state work in tests. // Our current tests don't need the background loops. #[cfg(test)] - pub fn create_test_timeline( + pub async fn create_test_timeline( &self, new_timeline_id: TimelineId, initdb_lsn: Lsn, @@ -1246,8 +1355,24 @@ impl Tenant { ctx: &RequestContext, ) -> anyhow::Result> { let uninit_tl = self.create_empty_timeline(new_timeline_id, initdb_lsn, pg_version, ctx)?; - let mut timelines = self.timelines.lock().unwrap(); - let tl = uninit_tl.initialize_with_lock(ctx, &mut timelines, true)?; + let tline = uninit_tl.raw_timeline().expect("we just created it"); + assert_eq!(tline.get_last_record_lsn(), Lsn(0)); + + // Setup minimum keys required for the timeline to be usable. + let mut modification = tline.begin_modification(initdb_lsn); + modification + .init_empty_test_timeline() + .context("init_empty_test_timeline")?; + modification + .commit() + .await + .context("commit init_empty_test_timeline modification")?; + + // Flush to disk so that uninit_tl's check for valid disk_consistent_lsn passes. + tline.maybe_spawn_flush_loop(); + tline.freeze_and_flush().await.context("freeze_and_flush")?; + + let tl = uninit_tl.finish_creation()?; // The non-test code would call tl.activate() here. tl.set_state(TimelineState::Active); Ok(tl) @@ -1258,8 +1383,7 @@ impl Tenant { /// Returns the new timeline ID and reference to its Timeline object. /// /// If the caller specified the timeline ID to use (`new_timeline_id`), and timeline with - /// the same timeline ID already exists, returns None. If `new_timeline_id` is not given, - /// a new unique ID is generated. + /// the same timeline ID already exists, returns CreateTimelineError::AlreadyExists. pub async fn create_timeline( &self, new_timeline_id: TimelineId, @@ -1268,11 +1392,12 @@ impl Tenant { pg_version: u32, broker_client: storage_broker::BrokerClientChannel, ctx: &RequestContext, - ) -> anyhow::Result>> { - anyhow::ensure!( - self.is_active(), - "Cannot create timelines on inactive tenant" - ); + ) -> Result, CreateTimelineError> { + if !self.is_active() { + return Err(CreateTimelineError::Other(anyhow::anyhow!( + "Cannot create timelines on inactive tenant" + ))); + } if let Ok(existing) = self.get_timeline(new_timeline_id, false) { debug!("timeline {new_timeline_id} already exists"); @@ -1292,7 +1417,7 @@ impl Tenant { .context("wait for timeline uploads to complete")?; } - return Ok(None); + return Err(CreateTimelineError::AlreadyExists); } let loaded_timeline = match ancestor_timeline_id { @@ -1307,12 +1432,12 @@ impl Tenant { let ancestor_ancestor_lsn = ancestor_timeline.get_ancestor_lsn(); if ancestor_ancestor_lsn > *lsn { // can we safely just branch from the ancestor instead? - bail!( + return Err(CreateTimelineError::Other(anyhow::anyhow!( "invalid start lsn {} for ancestor timeline {}: less than timeline ancestor lsn {}", lsn, ancestor_timeline_id, ancestor_ancestor_lsn, - ); + ))); } // Wait for the WAL to arrive and be processed on the parent branch up @@ -1333,7 +1458,7 @@ impl Tenant { } }; - loaded_timeline.activate(broker_client, ctx); + loaded_timeline.activate(broker_client, None, ctx); if let Some(remote_client) = loaded_timeline.remote_client.as_ref() { // Wait for the upload of the 'index_part.json` file to finish, so that when we return @@ -1346,7 +1471,7 @@ impl Tenant { })?; } - Ok(Some(loaded_timeline)) + Ok(loaded_timeline) } /// perform one garbage collection iteration, removing old data files from disk. @@ -1369,6 +1494,7 @@ impl Tenant { pitr: Duration, ctx: &RequestContext, ) -> anyhow::Result { + // there is a global allowed_error for this anyhow::ensure!( self.is_active(), "Cannot run GC iteration on inactive tenant" @@ -1396,7 +1522,13 @@ impl Tenant { let timelines = self.timelines.lock().unwrap(); let timelines_to_compact = timelines .iter() - .map(|(timeline_id, timeline)| (*timeline_id, timeline.clone())) + .filter_map(|(timeline_id, timeline)| { + if timeline.is_active() { + Some((*timeline_id, timeline.clone())) + } else { + None + } + }) .collect::>(); drop(timelines); timelines_to_compact @@ -1412,130 +1544,82 @@ impl Tenant { Ok(()) } - /// Flush all in-memory data to disk. + /// Flush all in-memory data to disk and remote storage, if any. /// /// Used at graceful shutdown. - /// - pub async fn freeze_and_flush(&self) -> anyhow::Result<()> { - // Scan through the hashmap and collect a list of all the timelines, - // while holding the lock. Then drop the lock and actually perform the - // flushing. We don't want to block everything else while the - // flushing is performed. - let timelines_to_flush = { + async fn freeze_and_flush_on_shutdown(&self) { + let mut js = tokio::task::JoinSet::new(); + + // execute on each timeline on the JoinSet, join after. + let per_timeline = |timeline_id: TimelineId, timeline: Arc| { + async move { + debug_assert_current_span_has_tenant_and_timeline_id(); + + match timeline.freeze_and_flush().await { + Ok(()) => {} + Err(e) => { + warn!("failed to freeze and flush: {e:#}"); + return; + } + } + + let res = if let Some(client) = timeline.remote_client.as_ref() { + // if we did not wait for completion here, it might be our shutdown process + // didn't wait for remote uploads to complete at all, as new tasks can forever + // be spawned. + // + // what is problematic is the shutting down of RemoteTimelineClient, because + // obviously it does not make sense to stop while we wait for it, but what + // about corner cases like s3 suddenly hanging up? + client.wait_completion().await + } else { + Ok(()) + }; + + if let Err(e) = res { + warn!("failed to await for frozen and flushed uploads: {e:#}"); + } + } + .instrument(tracing::info_span!("freeze_and_flush_on_shutdown", %timeline_id)) + }; + + { let timelines = self.timelines.lock().unwrap(); timelines .iter() - .map(|(_id, timeline)| Arc::clone(timeline)) - .collect::>() + .map(|(id, tl)| (*id, Arc::clone(tl))) + .for_each(|(timeline_id, timeline)| { + js.spawn(per_timeline(timeline_id, timeline)); + }) }; - for timeline in &timelines_to_flush { - timeline.freeze_and_flush().await?; - } - - Ok(()) - } - - /// Removes timeline-related in-memory data - pub async fn delete_timeline( - &self, - timeline_id: TimelineId, - _ctx: &RequestContext, - ) -> Result<(), DeleteTimelineError> { - timeline::debug_assert_current_span_has_tenant_and_timeline_id(); - - // Transition the timeline into TimelineState::Stopping. - // This should prevent new operations from starting. - let timeline = { - let mut timelines = self.timelines.lock().unwrap(); - - // Ensure that there are no child timelines **attached to that pageserver**, - // because detach removes files, which will break child branches - let children_exist = timelines - .iter() - .any(|(_, entry)| entry.get_ancestor_timeline_id() == Some(timeline_id)); - - if children_exist { - return Err(DeleteTimelineError::HasChildren); - } - - let timeline_entry = match timelines.entry(timeline_id) { - Entry::Occupied(e) => e, - Entry::Vacant(_) => return Err(DeleteTimelineError::NotFound), - }; - - let timeline = Arc::clone(timeline_entry.get()); - timeline.set_state(TimelineState::Stopping); - - drop(timelines); - timeline - }; - - // Now that the Timeline is in Stopping state, request all the related tasks to - // shut down. - // - // NB: If you call delete_timeline multiple times concurrently, they will - // all go through the motions here. Make sure the code here is idempotent, - // and don't error out if some of the shutdown tasks have already been - // completed! - - // Stop the walreceiver first. - debug!("waiting for wal receiver to shutdown"); - let maybe_started_walreceiver = { timeline.walreceiver.lock().unwrap().take() }; - if let Some(walreceiver) = maybe_started_walreceiver { - walreceiver.stop().await; - } - debug!("wal receiver shutdown confirmed"); - - // Prevent new uploads from starting. - if let Some(remote_client) = timeline.remote_client.as_ref() { - let res = remote_client.stop(); + while let Some(res) = js.join_next().await { match res { Ok(()) => {} - Err(e) => match e { - remote_timeline_client::StopError::QueueUninitialized => { - // This case shouldn't happen currently because the - // load and attach code bails out if _any_ of the timeline fails to fetch its IndexPart. - // That is, before we declare the Tenant as Active. - // But we only allow calls to delete_timeline on Active tenants. - return Err(DeleteTimelineError::Other(anyhow::anyhow!("upload queue is uninitialized, likely the timeline was in Broken state prior to this call because it failed to fetch IndexPart during load or attach, check the logs"))); - } - }, - } - } - - // Stop & wait for the remaining timeline tasks, including upload tasks. - // NB: This and other delete_timeline calls do not run as a task_mgr task, - // so, they are not affected by this shutdown_tasks() call. - info!("waiting for timeline tasks to shutdown"); - task_mgr::shutdown_tasks(None, Some(self.tenant_id), Some(timeline_id)).await; - - // Mark timeline as deleted in S3 so we won't pick it up next time - // during attach or pageserver restart. - // See comment in persist_index_part_with_deleted_flag. - if let Some(remote_client) = timeline.remote_client.as_ref() { - match remote_client.persist_index_part_with_deleted_flag().await { - // If we (now, or already) marked it successfully as deleted, we can proceed - Ok(()) | Err(PersistIndexPartWithDeletedFlagError::AlreadyDeleted(_)) => (), - // Bail out otherwise - Err(e @ PersistIndexPartWithDeletedFlagError::AlreadyInProgress(_)) - | Err(e @ PersistIndexPartWithDeletedFlagError::Other(_)) => { - return Err(DeleteTimelineError::Other(anyhow::anyhow!(e))); - } + Err(je) if je.is_cancelled() => unreachable!("no cancelling used"), + Err(je) if je.is_panic() => { /* logged already */ } + Err(je) => warn!("unexpected JoinError: {je:?}"), } } + } + /// Shuts down a timeline's tasks, removes its in-memory structures, and deletes its + /// data from both disk and s3. + async fn delete_timeline( + &self, + timeline_id: TimelineId, + timeline: Arc, + guard: DeletionGuard, + ) -> anyhow::Result<()> { { // Grab the layer_removal_cs lock, and actually perform the deletion. // - // This lock prevents multiple concurrent delete_timeline calls from - // stepping on each other's toes, while deleting the files. It also - // prevents GC or compaction from running at the same time. + // This lock prevents prevents GC or compaction from running at the same time. + // The GC task doesn't register itself with the timeline it's operating on, + // so it might still be running even though we called `shutdown_tasks`. // // Note that there are still other race conditions between - // GC, compaction and timeline deletion. GC task doesn't - // register itself properly with the timeline it's - // operating on. See + // GC, compaction and timeline deletion. See // https://github.com/neondatabase/neon/issues/2671 // // No timeout here, GC & Compaction should be responsive to the @@ -1547,7 +1631,9 @@ impl Tenant { // NB: storage_sync upload tasks that reference these layers have been cancelled // by the caller. - let local_timeline_directory = self.conf.timeline_path(&timeline_id, &self.tenant_id); + let local_timeline_directory = self + .conf + .timeline_path(&timeline.timeline_id, &self.tenant_id); fail::fail_point!("timeline-delete-before-rm", |_| { Err(anyhow::anyhow!("failpoint: timeline-delete-before-rm"))? @@ -1596,42 +1682,203 @@ impl Tenant { Err(anyhow::anyhow!("failpoint: timeline-delete-after-rm"))? }); - // Remove the timeline from the map. - let mut timelines = self.timelines.lock().unwrap(); - let children_exist = timelines - .iter() - .any(|(_, entry)| entry.get_ancestor_timeline_id() == Some(timeline_id)); - // XXX this can happen because `branch_timeline` doesn't check `TimelineState::Stopping`. - // We already deleted the layer files, so it's probably best to panic. - // (Ideally, above remove_dir_all is atomic so we don't see this timeline after a restart) - if children_exist { - panic!("Timeline grew children while we removed layer files"); + if let Some(remote_client) = &timeline.remote_client { + remote_client.delete_all().await.context("delete_all")? + }; + + // Have a failpoint that can use the `pause` failpoint action. + // We don't want to block the executor thread, hence, spawn_blocking + await. + if cfg!(feature = "testing") { + tokio::task::spawn_blocking({ + let current = tracing::Span::current(); + move || { + let _entered = current.entered(); + tracing::info!("at failpoint in_progress_delete"); + fail::fail_point!("in_progress_delete"); + } + }) + .await + .expect("spawn_blocking"); } - let removed_timeline = timelines.remove(&timeline_id); - if removed_timeline.is_none() { - // This can legitimately happen if there's a concurrent call to this function. - // T1 T2 - // lock - // unlock - // lock - // unlock - // remove files - // lock - // remove from map - // unlock - // return - // remove files - // lock - // remove from map observes empty map - // unlock - // return - debug!("concurrent call to this function won the race"); + + { + // Remove the timeline from the map. + let mut timelines = self.timelines.lock().unwrap(); + let children_exist = timelines + .iter() + .any(|(_, entry)| entry.get_ancestor_timeline_id() == Some(timeline_id)); + // XXX this can happen because `branch_timeline` doesn't check `TimelineState::Stopping`. + // We already deleted the layer files, so it's probably best to panic. + // (Ideally, above remove_dir_all is atomic so we don't see this timeline after a restart) + if children_exist { + panic!("Timeline grew children while we removed layer files"); + } + + timelines.remove(&timeline_id).expect( + "timeline that we were deleting was concurrently removed from 'timelines' map", + ); + + drop(timelines); } - drop(timelines); + + drop(guard); Ok(()) } + /// Removes timeline-related in-memory data and schedules removal from remote storage. + #[instrument(skip(self, _ctx))] + pub async fn prepare_and_schedule_delete_timeline( + self: Arc, + timeline_id: TimelineId, + _ctx: &RequestContext, + ) -> Result<(), DeleteTimelineError> { + timeline::debug_assert_current_span_has_tenant_and_timeline_id(); + + // Transition the timeline into TimelineState::Stopping. + // This should prevent new operations from starting. + // + // Also grab the Timeline's delete_lock to prevent another deletion from starting. + let timeline; + let delete_lock_guard; + { + let mut timelines = self.timelines.lock().unwrap(); + + // Ensure that there are no child timelines **attached to that pageserver**, + // because detach removes files, which will break child branches + let children: Vec = timelines + .iter() + .filter_map(|(id, entry)| { + if entry.get_ancestor_timeline_id() == Some(timeline_id) { + Some(*id) + } else { + None + } + }) + .collect(); + + if !children.is_empty() { + return Err(DeleteTimelineError::HasChildren(children)); + } + + let timeline_entry = match timelines.entry(timeline_id) { + Entry::Occupied(e) => e, + Entry::Vacant(_) => return Err(DeleteTimelineError::NotFound), + }; + + timeline = Arc::clone(timeline_entry.get()); + + // Prevent two tasks from trying to delete the timeline at the same time. + delete_lock_guard = DeletionGuard( + Arc::clone(&timeline.delete_lock) + .try_lock_owned() + .map_err(|_| DeleteTimelineError::AlreadyInProgress)?, + ); + + // If another task finished the deletion just before we acquired the lock, + // return success. + if delete_lock_guard.is_deleted() { + return Ok(()); + } + + timeline.set_state(TimelineState::Stopping); + + drop(timelines); + } + + // Now that the Timeline is in Stopping state, request all the related tasks to + // shut down. + // + // NB: If this fails half-way through, and is retried, the retry will go through + // all the same steps again. Make sure the code here is idempotent, and don't + // error out if some of the shutdown tasks have already been completed! + + // Stop the walreceiver first. + debug!("waiting for wal receiver to shutdown"); + let maybe_started_walreceiver = { timeline.walreceiver.lock().unwrap().take() }; + if let Some(walreceiver) = maybe_started_walreceiver { + walreceiver.stop().await; + } + debug!("wal receiver shutdown confirmed"); + + // Prevent new uploads from starting. + if let Some(remote_client) = timeline.remote_client.as_ref() { + let res = remote_client.stop(); + match res { + Ok(()) => {} + Err(e) => match e { + remote_timeline_client::StopError::QueueUninitialized => { + // This case shouldn't happen currently because the + // load and attach code bails out if _any_ of the timeline fails to fetch its IndexPart. + // That is, before we declare the Tenant as Active. + // But we only allow calls to delete_timeline on Active tenants. + return Err(DeleteTimelineError::Other(anyhow::anyhow!("upload queue is uninitialized, likely the timeline was in Broken state prior to this call because it failed to fetch IndexPart during load or attach, check the logs"))); + } + }, + } + } + + // Stop & wait for the remaining timeline tasks, including upload tasks. + // NB: This and other delete_timeline calls do not run as a task_mgr task, + // so, they are not affected by this shutdown_tasks() call. + info!("waiting for timeline tasks to shutdown"); + task_mgr::shutdown_tasks(None, Some(self.tenant_id), Some(timeline_id)).await; + + // Mark timeline as deleted in S3 so we won't pick it up next time + // during attach or pageserver restart. + // See comment in persist_index_part_with_deleted_flag. + if let Some(remote_client) = timeline.remote_client.as_ref() { + match remote_client.persist_index_part_with_deleted_flag().await { + // If we (now, or already) marked it successfully as deleted, we can proceed + Ok(()) | Err(PersistIndexPartWithDeletedFlagError::AlreadyDeleted(_)) => (), + // Bail out otherwise + // + // AlreadyInProgress shouldn't happen, because the 'delete_lock' prevents + // two tasks from performing the deletion at the same time. The first task + // that starts deletion should run it to completion. + Err(e @ PersistIndexPartWithDeletedFlagError::AlreadyInProgress(_)) + | Err(e @ PersistIndexPartWithDeletedFlagError::Other(_)) => { + return Err(DeleteTimelineError::Other(anyhow::anyhow!(e))); + } + } + } + self.schedule_delete_timeline(timeline_id, timeline, delete_lock_guard); + + Ok(()) + } + + fn schedule_delete_timeline( + self: Arc, + timeline_id: TimelineId, + timeline: Arc, + guard: DeletionGuard, + ) { + let tenant_id = self.tenant_id; + let timeline_clone = Arc::clone(&timeline); + + task_mgr::spawn( + task_mgr::BACKGROUND_RUNTIME.handle(), + TaskKind::TimelineDeletionWorker, + Some(self.tenant_id), + Some(timeline_id), + "timeline_delete", + false, + async move { + if let Err(err) = self.delete_timeline(timeline_id, timeline, guard).await { + error!("Error: {err:#}"); + timeline_clone.set_broken(err.to_string()) + }; + Ok(()) + } + .instrument({ + let span = + tracing::info_span!(parent: None, "delete_timeline", tenant_id=%tenant_id, timeline_id=%timeline_id); + span.follows_from(Span::current()); + span + }), + ); + } + pub fn current_state(&self) -> TenantState { self.state.borrow().clone() } @@ -1641,130 +1888,257 @@ impl Tenant { } /// Changes tenant status to active, unless shutdown was already requested. + /// + /// `background_jobs_can_start` is an optional barrier set to a value during pageserver startup + /// to delay background jobs. Background jobs can be started right away when None is given. fn activate( self: &Arc, broker_client: BrokerClientChannel, + background_jobs_can_start: Option<&completion::Barrier>, ctx: &RequestContext, - ) -> anyhow::Result<()> { + ) { debug_assert_current_span_has_tenant_id(); - let mut result = Ok(()); + let mut activating = false; self.state.send_modify(|current_state| { + use pageserver_api::models::ActivatingFrom; match &*current_state { - TenantState::Active => { - // activate() was called on an already Active tenant. Shouldn't happen. - result = Err(anyhow::anyhow!("Tenant is already active")); + TenantState::Activating(_) | TenantState::Active | TenantState::Broken { .. } | TenantState::Stopping => { + panic!("caller is responsible for calling activate() only on Loading / Attaching tenants, got {state:?}", state = current_state); } - TenantState::Broken { reason, .. } => { - // This shouldn't happen either - result = Err(anyhow::anyhow!( - "Could not activate tenant because it is in broken state due to: {reason}", - )); + TenantState::Loading => { + *current_state = TenantState::Activating(ActivatingFrom::Loading); } - TenantState::Stopping => { - // The tenant was detached, or system shutdown was requested, while we were - // loading or attaching the tenant. - info!("Tenant is already in Stopping state, skipping activation"); - } - TenantState::Loading | TenantState::Attaching => { - *current_state = TenantState::Active; - - debug!(tenant_id = %self.tenant_id, "Activating tenant"); - - let timelines_accessor = self.timelines.lock().unwrap(); - let not_broken_timelines = timelines_accessor - .values() - .filter(|timeline| timeline.current_state() != TimelineState::Broken); - - // Spawn gc and compaction loops. The loops will shut themselves - // down when they notice that the tenant is inactive. - tasks::start_background_loops(self); - - let mut activated_timelines = 0; - - for timeline in not_broken_timelines { - timeline.activate(broker_client.clone(), ctx); - activated_timelines += 1; - } - - let elapsed = self.loading_started_at.elapsed(); - let total_timelines = timelines_accessor.len(); - - // log a lot of stuff, because some tenants sometimes suffer from user-visible - // times to activate. see https://github.com/neondatabase/neon/issues/4025 - info!( - since_creation_millis = elapsed.as_millis(), - tenant_id = %self.tenant_id, - activated_timelines, - total_timelines, - post_state = <&'static str>::from(&*current_state), - "activation attempt finished" - ); + TenantState::Attaching => { + *current_state = TenantState::Activating(ActivatingFrom::Attaching); } } + debug!(tenant_id = %self.tenant_id, "Activating tenant"); + activating = true; + // Continue outside the closure. We need to grab timelines.lock() + // and we plan to turn it into a tokio::sync::Mutex in a future patch. }); - result + + if activating { + let timelines_accessor = self.timelines.lock().unwrap(); + let timelines_to_activate = timelines_accessor + .values() + .filter(|timeline| !(timeline.is_broken() || timeline.is_stopping())); + + // Spawn gc and compaction loops. The loops will shut themselves + // down when they notice that the tenant is inactive. + tasks::start_background_loops(self, background_jobs_can_start); + + let mut activated_timelines = 0; + + for timeline in timelines_to_activate { + timeline.activate(broker_client.clone(), background_jobs_can_start, ctx); + activated_timelines += 1; + } + + self.state.send_modify(move |current_state| { + assert!( + matches!(current_state, TenantState::Activating(_)), + "set_stopping and set_broken wait for us to leave Activating state", + ); + *current_state = TenantState::Active; + + let elapsed = self.loading_started_at.elapsed(); + let total_timelines = timelines_accessor.len(); + + // log a lot of stuff, because some tenants sometimes suffer from user-visible + // times to activate. see https://github.com/neondatabase/neon/issues/4025 + info!( + since_creation_millis = elapsed.as_millis(), + tenant_id = %self.tenant_id, + activated_timelines, + total_timelines, + post_state = <&'static str>::from(&*current_state), + "activation attempt finished" + ); + }); + } } - /// Change tenant status to Stopping, to mark that it is being shut down - pub fn set_stopping(&self) { - self.state.send_modify(|current_state| { - match current_state { - TenantState::Active | TenantState::Loading | TenantState::Attaching => { - *current_state = TenantState::Stopping; + /// Shutdown the tenant and join all of the spawned tasks. + /// + /// The method caters for all use-cases: + /// - pageserver shutdown (freeze_and_flush == true) + /// - detach + ignore (freeze_and_flush == false) + /// + /// This will attempt to shutdown even if tenant is broken. + pub(crate) async fn shutdown(&self, freeze_and_flush: bool) -> Result<(), ShutdownError> { + debug_assert_current_span_has_tenant_id(); + // Set tenant (and its timlines) to Stoppping state. + // + // Since we can only transition into Stopping state after activation is complete, + // run it in a JoinSet so all tenants have a chance to stop before we get SIGKILLed. + // + // Transitioning tenants to Stopping state has a couple of non-obvious side effects: + // 1. Lock out any new requests to the tenants. + // 2. Signal cancellation to WAL receivers (we wait on it below). + // 3. Signal cancellation for other tenant background loops. + // 4. ??? + // + // The waiting for the cancellation is not done uniformly. + // We certainly wait for WAL receivers to shut down. + // That is necessary so that no new data comes in before the freeze_and_flush. + // But the tenant background loops are joined-on in our caller. + // It's mesed up. + // we just ignore the failure to stop + match self.set_stopping().await { + Ok(()) => {} + Err(SetStoppingError::Broken) => { + // assume that this is acceptable + } + Err(SetStoppingError::AlreadyStopping) => return Err(ShutdownError::AlreadyStopping), + }; - // FIXME: If the tenant is still Loading or Attaching, new timelines - // might be created after this. That's harmless, as the Timelines - // won't be accessible to anyone, when the Tenant is in Stopping - // state. - let timelines_accessor = self.timelines.lock().unwrap(); - let not_broken_timelines = timelines_accessor - .values() - .filter(|timeline| timeline.current_state() != TimelineState::Broken); - for timeline in not_broken_timelines { - timeline.set_state(TimelineState::Stopping); - } - } - TenantState::Broken { reason, .. } => { - info!("Cannot set tenant to Stopping state, it is in Broken state due to: {reason}"); - } - TenantState::Stopping => { - // The tenant was detached, or system shutdown was requested, while we were - // loading or attaching the tenant. - info!("Tenant is already in Stopping state"); - } + if freeze_and_flush { + // walreceiver has already began to shutdown with TenantState::Stopping, but we need to + // await for them to stop. + task_mgr::shutdown_tasks( + Some(TaskKind::WalReceiverManager), + Some(self.tenant_id), + None, + ) + .await; + + // this will wait for uploads to complete; in the past, it was done outside tenant + // shutdown in pageserver::shutdown_pageserver. + self.freeze_and_flush_on_shutdown().await; + } + + // shutdown all tenant and timeline tasks: gc, compaction, page service + // No new tasks will be started for this tenant because it's in `Stopping` state. + // + // this will additionally shutdown and await all timeline tasks. + task_mgr::shutdown_tasks(None, Some(self.tenant_id), None).await; + + Ok(()) + } + + /// Change tenant status to Stopping, to mark that it is being shut down. + /// + /// This function waits for the tenant to become active if it isn't already, before transitioning it into Stopping state. + /// + /// This function is not cancel-safe! + async fn set_stopping(&self) -> Result<(), SetStoppingError> { + let mut rx = self.state.subscribe(); + + // cannot stop before we're done activating, so wait out until we're done activating + rx.wait_for(|state| match state { + TenantState::Activating(_) | TenantState::Loading | TenantState::Attaching => { + info!( + "waiting for {} to turn Active|Broken|Stopping", + <&'static str>::from(state) + ); + false + } + TenantState::Active | TenantState::Broken { .. } | TenantState::Stopping {} => true, + }) + .await + .expect("cannot drop self.state while on a &self method"); + + // we now know we're done activating, let's see whether this task is the winner to transition into Stopping + let mut err = None; + let stopping = self.state.send_if_modified(|current_state| match current_state { + TenantState::Activating(_) | TenantState::Loading | TenantState::Attaching => { + unreachable!("we ensured above that we're done with activation, and, there is no re-activation") + } + TenantState::Active => { + // FIXME: due to time-of-check vs time-of-use issues, it can happen that new timelines + // are created after the transition to Stopping. That's harmless, as the Timelines + // won't be accessible to anyone afterwards, because the Tenant is in Stopping state. + *current_state = TenantState::Stopping; + // Continue stopping outside the closure. We need to grab timelines.lock() + // and we plan to turn it into a tokio::sync::Mutex in a future patch. + true + } + TenantState::Broken { reason, .. } => { + info!( + "Cannot set tenant to Stopping state, it is in Broken state due to: {reason}" + ); + err = Some(SetStoppingError::Broken); + false + } + TenantState::Stopping => { + info!("Tenant is already in Stopping state"); + err = Some(SetStoppingError::AlreadyStopping); + false } }); + match (stopping, err) { + (true, None) => {} // continue + (false, Some(err)) => return Err(err), + (true, Some(_)) => unreachable!( + "send_if_modified closure must error out if not transitioning to Stopping" + ), + (false, None) => unreachable!( + "send_if_modified closure must return true if transitioning to Stopping" + ), + } + + let timelines_accessor = self.timelines.lock().unwrap(); + let not_broken_timelines = timelines_accessor + .values() + .filter(|timeline| !timeline.is_broken()); + for timeline in not_broken_timelines { + timeline.set_state(TimelineState::Stopping); + } + Ok(()) } - pub fn set_broken(&self, reason: String) { + /// Method for tenant::mgr to transition us into Broken state in case of a late failure in + /// `remove_tenant_from_memory` + /// + /// This function waits for the tenant to become active if it isn't already, before transitioning it into Stopping state. + /// + /// In tests, we also use this to set tenants to Broken state on purpose. + pub(crate) async fn set_broken(&self, reason: String) { + let mut rx = self.state.subscribe(); + + // The load & attach routines own the tenant state until it has reached `Active`. + // So, wait until it's done. + rx.wait_for(|state| match state { + TenantState::Activating(_) | TenantState::Loading | TenantState::Attaching => { + info!( + "waiting for {} to turn Active|Broken|Stopping", + <&'static str>::from(state) + ); + false + } + TenantState::Active | TenantState::Broken { .. } | TenantState::Stopping {} => true, + }) + .await + .expect("cannot drop self.state while on a &self method"); + + // we now know we're done activating, let's see whether this task is the winner to transition into Broken self.state.send_modify(|current_state| { match *current_state { + TenantState::Activating(_) | TenantState::Loading | TenantState::Attaching => { + unreachable!("we ensured above that we're done with activation, and, there is no re-activation") + } TenantState::Active => { - // Broken tenants can currently only used for fatal errors that happen - // while loading or attaching a tenant. A tenant that has already been - // activated should never be marked as broken. We cope with it the best - // we can, but it shouldn't happen. - warn!("Changing Active tenant to Broken state, reason: {}", reason); - *current_state = TenantState::broken_from_reason(reason); + if cfg!(feature = "testing") { + warn!("Changing Active tenant to Broken state, reason: {}", reason); + *current_state = TenantState::broken_from_reason(reason); + } else { + unreachable!("not allowed to call set_broken on Active tenants in non-testing builds") + } } TenantState::Broken { .. } => { - // This shouldn't happen either warn!("Tenant is already in Broken state"); } + // This is the only "expected" path, any other path is a bug. TenantState::Stopping => { - // This shouldn't happen either warn!( "Marking Stopping tenant as Broken state, reason: {}", reason ); *current_state = TenantState::broken_from_reason(reason); } - TenantState::Loading | TenantState::Attaching => { - info!("Setting tenant as Broken state, reason: {}", reason); - *current_state = TenantState::broken_from_reason(reason); - } - } + } }); } @@ -1777,7 +2151,7 @@ impl Tenant { loop { let current_state = receiver.borrow_and_update().clone(); match current_state { - TenantState::Loading | TenantState::Attaching => { + TenantState::Loading | TenantState::Attaching | TenantState::Activating(_) => { // in these states, there's a chance that we can reach ::Active receiver.changed().await.map_err( |_e: tokio::sync::watch::error::RecvError| { @@ -1943,12 +2317,18 @@ impl Tenant { } } - fn create_timeline_data( + /// Helper function to create a new Timeline struct. + /// + /// The returned Timeline is in Loading state. The caller is responsible for + /// initializing any on-disk state, and for inserting the Timeline to the 'timelines' + /// map. + fn create_timeline_struct( &self, new_timeline_id: TimelineId, new_metadata: &TimelineMetadata, ancestor: Option>, remote_client: Option, + init_order: Option<&InitializationOrder>, ) -> anyhow::Result> { if let Some(ancestor_timeline_id) = new_metadata.ancestor_timeline() { anyhow::ensure!( @@ -1957,6 +2337,9 @@ impl Tenant { ) } + let initial_logical_size_can_start = init_order.map(|x| &x.initial_logical_size_can_start); + let initial_logical_size_attempt = init_order.map(|x| &x.initial_logical_size_attempt); + let pg_version = new_metadata.pg_version(); Ok(Timeline::new( self.conf, @@ -1968,6 +2351,8 @@ impl Tenant { Arc::clone(&self.walredo_mgr), remote_client, pg_version, + initial_logical_size_can_start.cloned(), + initial_logical_size_attempt.cloned(), )) } @@ -2357,7 +2742,7 @@ impl Tenant { src_timeline: &Arc, dst_id: TimelineId, start_lsn: Option, - ctx: &RequestContext, + _ctx: &RequestContext, ) -> anyhow::Result> { let src_id = src_timeline.timeline_id; @@ -2442,17 +2827,15 @@ impl Tenant { src_timeline.pg_version, ); - let new_timeline = { - let mut timelines = self.timelines.lock().unwrap(); - self.prepare_timeline( - dst_id, - &metadata, - timeline_uninit_mark, - false, - Some(Arc::clone(src_timeline)), - )? - .initialize_with_lock(ctx, &mut timelines, true)? - }; + let uninitialized_timeline = self.prepare_new_timeline( + dst_id, + &metadata, + timeline_uninit_mark, + start_lsn + 1, + Some(Arc::clone(src_timeline)), + )?; + + let new_timeline = uninitialized_timeline.finish_creation()?; // Root timeline gets its layers during creation and uploads them along with the metadata. // A branch timeline though, when created, can get no writes for some time, hence won't get any layers created. @@ -2528,8 +2911,13 @@ impl Tenant { pgdata_lsn, pg_version, ); - let raw_timeline = - self.prepare_timeline(timeline_id, &new_metadata, timeline_uninit_mark, true, None)?; + let raw_timeline = self.prepare_new_timeline( + timeline_id, + &new_metadata, + timeline_uninit_mark, + pgdata_lsn, + None, + )?; let tenant_id = raw_timeline.owning_tenant.tenant_id; let unfinished_timeline = raw_timeline.raw_timeline()?; @@ -2545,10 +2933,10 @@ impl Tenant { format!("Failed to import pgdatadir for timeline {tenant_id}/{timeline_id}") })?; - // Flush the new layer files to disk, before we mark the timeline as available to + // Flush the new layer files to disk, before we make the timeline as available to // the outside world. // - // Thus spawn flush loop manually and skip flush_loop setup in initialize_with_lock + // Flush loop needs to be spawned in order to be able to flush. unfinished_timeline.maybe_spawn_flush_loop(); fail::fail_point!("before-checkpoint-new-timeline", |_| { @@ -2564,12 +2952,8 @@ impl Tenant { ) })?; - // Initialize the timeline without loading the layer map, because we already updated the layer - // map above, when we imported the datadir. - let timeline = { - let mut timelines = self.timelines.lock().unwrap(); - raw_timeline.initialize_with_lock(ctx, &mut timelines, false)? - }; + // All done! + let timeline = raw_timeline.finish_creation()?; info!( "created root timeline {} timeline.lsn {}", @@ -2580,14 +2964,18 @@ impl Tenant { Ok(timeline) } - /// Creates intermediate timeline structure and its files, without loading it into memory. - /// It's up to the caller to import the necesary data and import the timeline into memory. - fn prepare_timeline( + /// Creates intermediate timeline structure and its files. + /// + /// An empty layer map is initialized, and new data and WAL can be imported starting + /// at 'disk_consistent_lsn'. After any initial data has been imported, call + /// `finish_creation` to insert the Timeline into the timelines map and to remove the + /// uninit mark file. + fn prepare_new_timeline( &self, new_timeline_id: TimelineId, new_metadata: &TimelineMetadata, uninit_mark: TimelineUninitMark, - init_layers: bool, + start_lsn: Lsn, ancestor: Option>, ) -> anyhow::Result { let tenant_id = self.tenant_id; @@ -2605,33 +2993,27 @@ impl Tenant { None }; - match self.create_timeline_files( - &uninit_mark.timeline_path, - new_timeline_id, - new_metadata, - ancestor, - remote_client, - ) { - Ok(new_timeline) => { - if init_layers { - new_timeline.layers.write().unwrap().next_open_layer_at = - Some(new_timeline.initdb_lsn); - } - debug!( - "Successfully created initial files for timeline {tenant_id}/{new_timeline_id}" - ); - Ok(UninitializedTimeline { - owning_tenant: self, - timeline_id: new_timeline_id, - raw_timeline: Some((new_timeline, uninit_mark)), - }) - } - Err(e) => { - error!("Failed to create initial files for timeline {tenant_id}/{new_timeline_id}, cleaning up: {e:?}"); - cleanup_timeline_directory(uninit_mark); - Err(e) - } + let timeline_struct = self + .create_timeline_struct(new_timeline_id, new_metadata, ancestor, remote_client, None) + .context("Failed to create timeline data structure")?; + + timeline_struct.init_empty_layer_map(start_lsn); + + if let Err(e) = + self.create_timeline_files(&uninit_mark.timeline_path, new_timeline_id, new_metadata) + { + error!("Failed to create initial files for timeline {tenant_id}/{new_timeline_id}, cleaning up: {e:?}"); + cleanup_timeline_directory(uninit_mark); + return Err(e); } + + debug!("Successfully created initial files for timeline {tenant_id}/{new_timeline_id}"); + + Ok(UninitializedTimeline { + owning_tenant: self, + timeline_id: new_timeline_id, + raw_timeline: Some((timeline_struct, uninit_mark)), + }) } fn create_timeline_files( @@ -2639,13 +3021,8 @@ impl Tenant { timeline_path: &Path, new_timeline_id: TimelineId, new_metadata: &TimelineMetadata, - ancestor: Option>, - remote_client: Option, - ) -> anyhow::Result> { - let timeline_data = self - .create_timeline_data(new_timeline_id, new_metadata, ancestor, remote_client) - .context("Failed to create timeline data structure")?; - crashsafe::create_dir_all(timeline_path).context("Failed to create timeline directory")?; + ) -> anyhow::Result<()> { + crashsafe::create_dir(timeline_path).context("Failed to create timeline directory")?; fail::fail_point!("after-timeline-uninit-mark-creation", |_| { anyhow::bail!("failpoint after-timeline-uninit-mark-creation"); @@ -2659,8 +3036,7 @@ impl Tenant { true, ) .context("Failed to create timeline metadata")?; - - Ok(timeline_data) + Ok(()) } /// Attempts to create an uninit mark file for the timeline initialization. @@ -3044,9 +3420,8 @@ where #[cfg(test)] pub mod harness { use bytes::{Bytes, BytesMut}; - use once_cell::sync::Lazy; use once_cell::sync::OnceCell; - use std::sync::{Arc, RwLock, RwLockReadGuard, RwLockWriteGuard}; + use std::sync::Arc; use std::{fs, path::PathBuf}; use utils::logging; use utils::lsn::Lsn; @@ -3079,8 +3454,6 @@ pub mod harness { buf.freeze() } - static LOCK: Lazy> = Lazy::new(|| RwLock::new(())); - impl From for TenantConfOpt { fn from(tenant_conf: TenantConf) -> Self { Self { @@ -3102,37 +3475,21 @@ pub mod harness { evictions_low_residence_duration_metric_threshold: Some( tenant_conf.evictions_low_residence_duration_metric_threshold, ), + gc_feedback: Some(tenant_conf.gc_feedback), } } } - pub struct TenantHarness<'a> { + pub struct TenantHarness { pub conf: &'static PageServerConf, pub tenant_conf: TenantConf, pub tenant_id: TenantId, - - pub lock_guard: ( - Option>, - Option>, - ), } static LOG_HANDLE: OnceCell<()> = OnceCell::new(); - impl<'a> TenantHarness<'a> { + impl TenantHarness { pub fn create(test_name: &'static str) -> anyhow::Result { - Self::create_internal(test_name, false) - } - pub fn create_exclusive(test_name: &'static str) -> anyhow::Result { - Self::create_internal(test_name, true) - } - fn create_internal(test_name: &'static str, exclusive: bool) -> anyhow::Result { - let lock_guard = if exclusive { - (None, Some(LOCK.write().unwrap())) - } else { - (Some(LOCK.read().unwrap()), None) - }; - LOG_HANDLE.get_or_init(|| { logging::init( logging::LogFormat::Test, @@ -3168,7 +3525,6 @@ pub mod harness { conf, tenant_conf, tenant_id, - lock_guard, }) } @@ -3193,26 +3549,12 @@ pub mod harness { self.tenant_id, None, )); - // populate tenant with locally available timelines - let mut timelines_to_load = HashMap::new(); - for timeline_dir_entry in fs::read_dir(self.conf.timelines_path(&self.tenant_id)) - .expect("should be able to read timelines dir") - { - let timeline_dir_entry = timeline_dir_entry?; - let timeline_id: TimelineId = timeline_dir_entry - .path() - .file_name() - .unwrap() - .to_string_lossy() - .parse()?; - - let timeline_metadata = load_metadata(self.conf, timeline_id, self.tenant_id)?; - timelines_to_load.insert(timeline_id, timeline_metadata); - } tenant - .load(ctx) + .load(None, ctx) .instrument(info_span!("try_load", tenant_id=%self.tenant_id)) .await?; + + // TODO reuse Tenant::activate (needs broker) tenant.state.send_replace(TenantState::Active); for timeline in tenant.timelines.lock().unwrap().values() { timeline.set_state(TimelineState::Active); @@ -3274,15 +3616,21 @@ mod tests { #[tokio::test] async fn test_basic() -> anyhow::Result<()> { let (tenant, ctx) = TenantHarness::create("test_basic")?.load().await; - let tline = tenant.create_test_timeline(TIMELINE_ID, Lsn(0), DEFAULT_PG_VERSION, &ctx)?; + let tline = tenant + .create_test_timeline(TIMELINE_ID, Lsn(0x08), DEFAULT_PG_VERSION, &ctx) + .await?; - let writer = tline.writer(); - writer.put(*TEST_KEY, Lsn(0x10), &Value::Image(TEST_IMG("foo at 0x10")))?; + let writer = tline.writer().await; + writer + .put(*TEST_KEY, Lsn(0x10), &Value::Image(TEST_IMG("foo at 0x10"))) + .await?; writer.finish_write(Lsn(0x10)); drop(writer); - let writer = tline.writer(); - writer.put(*TEST_KEY, Lsn(0x20), &Value::Image(TEST_IMG("foo at 0x20")))?; + let writer = tline.writer().await; + writer + .put(*TEST_KEY, Lsn(0x20), &Value::Image(TEST_IMG("foo at 0x20"))) + .await?; writer.finish_write(Lsn(0x20)); drop(writer); @@ -3307,9 +3655,11 @@ mod tests { let (tenant, ctx) = TenantHarness::create("no_duplicate_timelines")? .load() .await; - let _ = tenant.create_test_timeline(TIMELINE_ID, Lsn(0), DEFAULT_PG_VERSION, &ctx)?; + let _ = tenant + .create_test_timeline(TIMELINE_ID, Lsn(0x10), DEFAULT_PG_VERSION, &ctx) + .await?; - match tenant.create_empty_timeline(TIMELINE_ID, Lsn(0), DEFAULT_PG_VERSION, &ctx) { + match tenant.create_empty_timeline(TIMELINE_ID, Lsn(0x10), DEFAULT_PG_VERSION, &ctx) { Ok(_) => panic!("duplicate timeline creation should fail"), Err(e) => assert_eq!( e.to_string(), @@ -3338,8 +3688,10 @@ mod tests { use std::str::from_utf8; let (tenant, ctx) = TenantHarness::create("test_branch")?.load().await; - let tline = tenant.create_test_timeline(TIMELINE_ID, Lsn(0), DEFAULT_PG_VERSION, &ctx)?; - let writer = tline.writer(); + let tline = tenant + .create_test_timeline(TIMELINE_ID, Lsn(0x10), DEFAULT_PG_VERSION, &ctx) + .await?; + let writer = tline.writer().await; #[allow(non_snake_case)] let TEST_KEY_A: Key = Key::from_hex("112222222233333333444444445500000001").unwrap(); @@ -3347,13 +3699,21 @@ mod tests { let TEST_KEY_B: Key = Key::from_hex("112222222233333333444444445500000002").unwrap(); // Insert a value on the timeline - writer.put(TEST_KEY_A, Lsn(0x20), &test_value("foo at 0x20"))?; - writer.put(TEST_KEY_B, Lsn(0x20), &test_value("foobar at 0x20"))?; + writer + .put(TEST_KEY_A, Lsn(0x20), &test_value("foo at 0x20")) + .await?; + writer + .put(TEST_KEY_B, Lsn(0x20), &test_value("foobar at 0x20")) + .await?; writer.finish_write(Lsn(0x20)); - writer.put(TEST_KEY_A, Lsn(0x30), &test_value("foo at 0x30"))?; + writer + .put(TEST_KEY_A, Lsn(0x30), &test_value("foo at 0x30")) + .await?; writer.finish_write(Lsn(0x30)); - writer.put(TEST_KEY_A, Lsn(0x40), &test_value("foo at 0x40"))?; + writer + .put(TEST_KEY_A, Lsn(0x40), &test_value("foo at 0x40")) + .await?; writer.finish_write(Lsn(0x40)); //assert_current_logical_size(&tline, Lsn(0x40)); @@ -3365,8 +3725,10 @@ mod tests { let newtline = tenant .get_timeline(NEW_TIMELINE_ID, true) .expect("Should have a local timeline"); - let new_writer = newtline.writer(); - new_writer.put(TEST_KEY_A, Lsn(0x40), &test_value("bar at 0x40"))?; + let new_writer = newtline.writer().await; + new_writer + .put(TEST_KEY_A, Lsn(0x40), &test_value("bar at 0x40")) + .await?; new_writer.finish_write(Lsn(0x40)); // Check page contents on both branches @@ -3392,38 +3754,46 @@ mod tests { let mut lsn = start_lsn; #[allow(non_snake_case)] { - let writer = tline.writer(); + let writer = tline.writer().await; // Create a relation on the timeline - writer.put( - *TEST_KEY, - lsn, - &Value::Image(TEST_IMG(&format!("foo at {}", lsn))), - )?; + writer + .put( + *TEST_KEY, + lsn, + &Value::Image(TEST_IMG(&format!("foo at {}", lsn))), + ) + .await?; writer.finish_write(lsn); lsn += 0x10; - writer.put( - *TEST_KEY, - lsn, - &Value::Image(TEST_IMG(&format!("foo at {}", lsn))), - )?; + writer + .put( + *TEST_KEY, + lsn, + &Value::Image(TEST_IMG(&format!("foo at {}", lsn))), + ) + .await?; writer.finish_write(lsn); lsn += 0x10; } tline.freeze_and_flush().await?; { - let writer = tline.writer(); - writer.put( - *TEST_KEY, - lsn, - &Value::Image(TEST_IMG(&format!("foo at {}", lsn))), - )?; + let writer = tline.writer().await; + writer + .put( + *TEST_KEY, + lsn, + &Value::Image(TEST_IMG(&format!("foo at {}", lsn))), + ) + .await?; writer.finish_write(lsn); lsn += 0x10; - writer.put( - *TEST_KEY, - lsn, - &Value::Image(TEST_IMG(&format!("foo at {}", lsn))), - )?; + writer + .put( + *TEST_KEY, + lsn, + &Value::Image(TEST_IMG(&format!("foo at {}", lsn))), + ) + .await?; writer.finish_write(lsn); } tline.freeze_and_flush().await @@ -3435,7 +3805,9 @@ mod tests { TenantHarness::create("test_prohibit_branch_creation_on_garbage_collected_data")? .load() .await; - let tline = tenant.create_test_timeline(TIMELINE_ID, Lsn(0), DEFAULT_PG_VERSION, &ctx)?; + let tline = tenant + .create_test_timeline(TIMELINE_ID, Lsn(0x10), DEFAULT_PG_VERSION, &ctx) + .await?; make_some_layers(tline.as_ref(), Lsn(0x20)).await?; // this removes layers before lsn 40 (50 minus 10), so there are two remaining layers, image and delta for 31-50 @@ -3472,8 +3844,9 @@ mod tests { .load() .await; - let tline = - tenant.create_test_timeline(TIMELINE_ID, Lsn(0x50), DEFAULT_PG_VERSION, &ctx)?; + let tline = tenant + .create_test_timeline(TIMELINE_ID, Lsn(0x50), DEFAULT_PG_VERSION, &ctx) + .await?; // try to branch at lsn 0x25, should fail because initdb lsn is 0x50 match tenant .branch_timeline_test(&tline, NEW_TIMELINE_ID, Some(Lsn(0x25)), &ctx) @@ -3522,7 +3895,9 @@ mod tests { TenantHarness::create("test_get_branchpoints_from_an_inactive_timeline")? .load() .await; - let tline = tenant.create_test_timeline(TIMELINE_ID, Lsn(0), DEFAULT_PG_VERSION, &ctx)?; + let tline = tenant + .create_test_timeline(TIMELINE_ID, Lsn(0x10), DEFAULT_PG_VERSION, &ctx) + .await?; make_some_layers(tline.as_ref(), Lsn(0x20)).await?; tenant @@ -3534,7 +3909,7 @@ mod tests { make_some_layers(newtline.as_ref(), Lsn(0x60)).await?; - tline.set_state(TimelineState::Broken); + tline.set_broken("test".to_owned()); tenant .gc_iteration(Some(TIMELINE_ID), 0x10, Duration::ZERO, &ctx) @@ -3570,7 +3945,9 @@ mod tests { TenantHarness::create("test_retain_data_in_parent_which_is_needed_for_child")? .load() .await; - let tline = tenant.create_test_timeline(TIMELINE_ID, Lsn(0), DEFAULT_PG_VERSION, &ctx)?; + let tline = tenant + .create_test_timeline(TIMELINE_ID, Lsn(0x10), DEFAULT_PG_VERSION, &ctx) + .await?; make_some_layers(tline.as_ref(), Lsn(0x20)).await?; tenant @@ -3593,7 +3970,9 @@ mod tests { TenantHarness::create("test_parent_keeps_data_forever_after_branching")? .load() .await; - let tline = tenant.create_test_timeline(TIMELINE_ID, Lsn(0), DEFAULT_PG_VERSION, &ctx)?; + let tline = tenant + .create_test_timeline(TIMELINE_ID, Lsn(0x10), DEFAULT_PG_VERSION, &ctx) + .await?; make_some_layers(tline.as_ref(), Lsn(0x20)).await?; tenant @@ -3625,8 +4004,9 @@ mod tests { let harness = TenantHarness::create(TEST_NAME)?; { let (tenant, ctx) = harness.load().await; - let tline = - tenant.create_test_timeline(TIMELINE_ID, Lsn(0x8000), DEFAULT_PG_VERSION, &ctx)?; + let tline = tenant + .create_test_timeline(TIMELINE_ID, Lsn(0x7000), DEFAULT_PG_VERSION, &ctx) + .await?; make_some_layers(tline.as_ref(), Lsn(0x8000)).await?; } @@ -3645,8 +4025,9 @@ mod tests { // create two timelines { let (tenant, ctx) = harness.load().await; - let tline = - tenant.create_test_timeline(TIMELINE_ID, Lsn(0), DEFAULT_PG_VERSION, &ctx)?; + let tline = tenant + .create_test_timeline(TIMELINE_ID, Lsn(0x10), DEFAULT_PG_VERSION, &ctx) + .await?; make_some_layers(tline.as_ref(), Lsn(0x20)).await?; @@ -3683,7 +4064,9 @@ mod tests { let harness = TenantHarness::create(TEST_NAME)?; let (tenant, ctx) = harness.load().await; - let tline = tenant.create_test_timeline(TIMELINE_ID, Lsn(0), DEFAULT_PG_VERSION, &ctx)?; + let tline = tenant + .create_test_timeline(TIMELINE_ID, Lsn(0x10), DEFAULT_PG_VERSION, &ctx) + .await?; drop(tline); drop(tenant); @@ -3697,9 +4080,13 @@ mod tests { std::fs::write(metadata_path, metadata_bytes)?; let err = harness.try_load(&ctx).await.err().expect("should fail"); - assert!(err - .to_string() - .starts_with("Failed to parse metadata bytes from path")); + // get all the stack with all .context, not tonly the last one + let message = format!("{err:#}"); + let expected = "Failed to parse metadata bytes from path"; + assert!( + message.contains(expected), + "message '{message}' expected to contain {expected}" + ); let mut found_error_message = false; let mut err_source = err.source(); @@ -3721,34 +4108,44 @@ mod tests { #[tokio::test] async fn test_images() -> anyhow::Result<()> { let (tenant, ctx) = TenantHarness::create("test_images")?.load().await; - let tline = tenant.create_test_timeline(TIMELINE_ID, Lsn(0), DEFAULT_PG_VERSION, &ctx)?; + let tline = tenant + .create_test_timeline(TIMELINE_ID, Lsn(0x08), DEFAULT_PG_VERSION, &ctx) + .await?; - let writer = tline.writer(); - writer.put(*TEST_KEY, Lsn(0x10), &Value::Image(TEST_IMG("foo at 0x10")))?; + let writer = tline.writer().await; + writer + .put(*TEST_KEY, Lsn(0x10), &Value::Image(TEST_IMG("foo at 0x10"))) + .await?; writer.finish_write(Lsn(0x10)); drop(writer); tline.freeze_and_flush().await?; tline.compact(&ctx).await?; - let writer = tline.writer(); - writer.put(*TEST_KEY, Lsn(0x20), &Value::Image(TEST_IMG("foo at 0x20")))?; + let writer = tline.writer().await; + writer + .put(*TEST_KEY, Lsn(0x20), &Value::Image(TEST_IMG("foo at 0x20"))) + .await?; writer.finish_write(Lsn(0x20)); drop(writer); tline.freeze_and_flush().await?; tline.compact(&ctx).await?; - let writer = tline.writer(); - writer.put(*TEST_KEY, Lsn(0x30), &Value::Image(TEST_IMG("foo at 0x30")))?; + let writer = tline.writer().await; + writer + .put(*TEST_KEY, Lsn(0x30), &Value::Image(TEST_IMG("foo at 0x30"))) + .await?; writer.finish_write(Lsn(0x30)); drop(writer); tline.freeze_and_flush().await?; tline.compact(&ctx).await?; - let writer = tline.writer(); - writer.put(*TEST_KEY, Lsn(0x40), &Value::Image(TEST_IMG("foo at 0x40")))?; + let writer = tline.writer().await; + writer + .put(*TEST_KEY, Lsn(0x40), &Value::Image(TEST_IMG("foo at 0x40"))) + .await?; writer.finish_write(Lsn(0x40)); drop(writer); @@ -3786,7 +4183,9 @@ mod tests { #[tokio::test] async fn test_bulk_insert() -> anyhow::Result<()> { let (tenant, ctx) = TenantHarness::create("test_bulk_insert")?.load().await; - let tline = tenant.create_test_timeline(TIMELINE_ID, Lsn(0), DEFAULT_PG_VERSION, &ctx)?; + let tline = tenant + .create_test_timeline(TIMELINE_ID, Lsn(0x08), DEFAULT_PG_VERSION, &ctx) + .await?; let mut lsn = Lsn(0x10); @@ -3797,12 +4196,14 @@ mod tests { for _ in 0..50 { for _ in 0..10000 { test_key.field6 = blknum; - let writer = tline.writer(); - writer.put( - test_key, - lsn, - &Value::Image(TEST_IMG(&format!("{} at {}", blknum, lsn))), - )?; + let writer = tline.writer().await; + writer + .put( + test_key, + lsn, + &Value::Image(TEST_IMG(&format!("{} at {}", blknum, lsn))), + ) + .await?; writer.finish_write(lsn); drop(writer); @@ -3828,7 +4229,9 @@ mod tests { #[tokio::test] async fn test_random_updates() -> anyhow::Result<()> { let (tenant, ctx) = TenantHarness::create("test_random_updates")?.load().await; - let tline = tenant.create_test_timeline(TIMELINE_ID, Lsn(0), DEFAULT_PG_VERSION, &ctx)?; + let tline = tenant + .create_test_timeline(TIMELINE_ID, Lsn(0x10), DEFAULT_PG_VERSION, &ctx) + .await?; const NUM_KEYS: usize = 1000; @@ -3840,17 +4243,19 @@ mod tests { // a read sees the latest page version. let mut updated = [Lsn(0); NUM_KEYS]; - let mut lsn = Lsn(0); + let mut lsn = Lsn(0x10); #[allow(clippy::needless_range_loop)] for blknum in 0..NUM_KEYS { lsn = Lsn(lsn.0 + 0x10); test_key.field6 = blknum as u32; - let writer = tline.writer(); - writer.put( - test_key, - lsn, - &Value::Image(TEST_IMG(&format!("{} at {}", blknum, lsn))), - )?; + let writer = tline.writer().await; + writer + .put( + test_key, + lsn, + &Value::Image(TEST_IMG(&format!("{} at {}", blknum, lsn))), + ) + .await?; writer.finish_write(lsn); updated[blknum] = lsn; drop(writer); @@ -3863,12 +4268,14 @@ mod tests { lsn = Lsn(lsn.0 + 0x10); let blknum = thread_rng().gen_range(0..NUM_KEYS); test_key.field6 = blknum as u32; - let writer = tline.writer(); - writer.put( - test_key, - lsn, - &Value::Image(TEST_IMG(&format!("{} at {}", blknum, lsn))), - )?; + let writer = tline.writer().await; + writer + .put( + test_key, + lsn, + &Value::Image(TEST_IMG(&format!("{} at {}", blknum, lsn))), + ) + .await?; writer.finish_write(lsn); drop(writer); updated[blknum] = lsn; @@ -3901,8 +4308,9 @@ mod tests { let (tenant, ctx) = TenantHarness::create("test_traverse_branches")? .load() .await; - let mut tline = - tenant.create_test_timeline(TIMELINE_ID, Lsn(0), DEFAULT_PG_VERSION, &ctx)?; + let mut tline = tenant + .create_test_timeline(TIMELINE_ID, Lsn(0x10), DEFAULT_PG_VERSION, &ctx) + .await?; const NUM_KEYS: usize = 1000; @@ -3914,17 +4322,19 @@ mod tests { // a read sees the latest page version. let mut updated = [Lsn(0); NUM_KEYS]; - let mut lsn = Lsn(0); + let mut lsn = Lsn(0x10); #[allow(clippy::needless_range_loop)] for blknum in 0..NUM_KEYS { lsn = Lsn(lsn.0 + 0x10); test_key.field6 = blknum as u32; - let writer = tline.writer(); - writer.put( - test_key, - lsn, - &Value::Image(TEST_IMG(&format!("{} at {}", blknum, lsn))), - )?; + let writer = tline.writer().await; + writer + .put( + test_key, + lsn, + &Value::Image(TEST_IMG(&format!("{} at {}", blknum, lsn))), + ) + .await?; writer.finish_write(lsn); updated[blknum] = lsn; drop(writer); @@ -3945,12 +4355,14 @@ mod tests { lsn = Lsn(lsn.0 + 0x10); let blknum = thread_rng().gen_range(0..NUM_KEYS); test_key.field6 = blknum as u32; - let writer = tline.writer(); - writer.put( - test_key, - lsn, - &Value::Image(TEST_IMG(&format!("{} at {}", blknum, lsn))), - )?; + let writer = tline.writer().await; + writer + .put( + test_key, + lsn, + &Value::Image(TEST_IMG(&format!("{} at {}", blknum, lsn))), + ) + .await?; println!("updating {} at {}", blknum, lsn); writer.finish_write(lsn); drop(writer); @@ -3984,8 +4396,9 @@ mod tests { let (tenant, ctx) = TenantHarness::create("test_traverse_ancestors")? .load() .await; - let mut tline = - tenant.create_test_timeline(TIMELINE_ID, Lsn(0), DEFAULT_PG_VERSION, &ctx)?; + let mut tline = tenant + .create_test_timeline(TIMELINE_ID, Lsn(0x10), DEFAULT_PG_VERSION, &ctx) + .await?; const NUM_KEYS: usize = 100; const NUM_TLINES: usize = 50; @@ -3994,7 +4407,7 @@ mod tests { // Track page mutation lsns across different timelines. let mut updated = [[Lsn(0); NUM_KEYS]; NUM_TLINES]; - let mut lsn = Lsn(0); + let mut lsn = Lsn(0x10); #[allow(clippy::needless_range_loop)] for idx in 0..NUM_TLINES { @@ -4010,12 +4423,14 @@ mod tests { lsn = Lsn(lsn.0 + 0x10); let blknum = thread_rng().gen_range(0..NUM_KEYS); test_key.field6 = blknum as u32; - let writer = tline.writer(); - writer.put( - test_key, - lsn, - &Value::Image(TEST_IMG(&format!("{} {} at {}", idx, blknum, lsn))), - )?; + let writer = tline.writer().await; + writer + .put( + test_key, + lsn, + &Value::Image(TEST_IMG(&format!("{} {} at {}", idx, blknum, lsn))), + ) + .await?; println!("updating [{}][{}] at {}", idx, blknum, lsn); writer.finish_write(lsn); drop(writer); @@ -4040,6 +4455,112 @@ mod tests { } Ok(()) } + + #[tokio::test] + async fn test_write_at_initdb_lsn_takes_optimization_code_path() -> anyhow::Result<()> { + let (tenant, ctx) = TenantHarness::create("test_empty_test_timeline_is_usable")? + .load() + .await; + + let initdb_lsn = Lsn(0x20); + let utline = + tenant.create_empty_timeline(TIMELINE_ID, initdb_lsn, DEFAULT_PG_VERSION, &ctx)?; + let tline = utline.raw_timeline().unwrap(); + + // Spawn flush loop now so that we can set the `expect_initdb_optimization` + tline.maybe_spawn_flush_loop(); + + // Make sure the timeline has the minimum set of required keys for operation. + // The only operation you can always do on an empty timeline is to `put` new data. + // Except if you `put` at `initdb_lsn`. + // In that case, there's an optimization to directly create image layers instead of delta layers. + // It uses `repartition()`, which assumes some keys to be present. + // Let's make sure the test timeline can handle that case. + { + let mut state = tline.flush_loop_state.lock().unwrap(); + assert_eq!( + timeline::FlushLoopState::Running { + expect_initdb_optimization: false, + initdb_optimization_count: 0, + }, + *state + ); + *state = timeline::FlushLoopState::Running { + expect_initdb_optimization: true, + initdb_optimization_count: 0, + }; + } + + // Make writes at the initdb_lsn. When we flush it below, it should be handled by the optimization. + // As explained above, the optimization requires some keys to be present. + // As per `create_empty_timeline` documentation, use init_empty to set them. + // This is what `create_test_timeline` does, by the way. + let mut modification = tline.begin_modification(initdb_lsn); + modification + .init_empty_test_timeline() + .context("init_empty_test_timeline")?; + modification + .commit() + .await + .context("commit init_empty_test_timeline modification")?; + + // Do the flush. The flush code will check the expectations that we set above. + tline.freeze_and_flush().await?; + + // assert freeze_and_flush exercised the initdb optimization + { + let state = tline.flush_loop_state.lock().unwrap(); + let + timeline::FlushLoopState::Running { + expect_initdb_optimization, + initdb_optimization_count, + } = *state else { + panic!("unexpected state: {:?}", *state); + }; + assert!(expect_initdb_optimization); + assert!(initdb_optimization_count > 0); + } + Ok(()) + } + + #[tokio::test] + async fn test_uninit_mark_crash() -> anyhow::Result<()> { + let name = "test_uninit_mark_crash"; + let harness = TenantHarness::create(name)?; + { + let (tenant, ctx) = harness.load().await; + let tline = + tenant.create_empty_timeline(TIMELINE_ID, Lsn(0), DEFAULT_PG_VERSION, &ctx)?; + // Keeps uninit mark in place + std::mem::forget(tline); + } + + let (tenant, _) = harness.load().await; + match tenant.get_timeline(TIMELINE_ID, false) { + Ok(_) => panic!("timeline should've been removed during load"), + Err(e) => { + assert_eq!( + e, + GetTimelineError::NotFound { + tenant_id: tenant.tenant_id, + timeline_id: TIMELINE_ID, + } + ) + } + } + + assert!(!harness + .conf + .timeline_path(&TIMELINE_ID, &tenant.tenant_id) + .exists()); + + assert!(!harness + .conf + .timeline_uninit_mark_file_path(tenant.tenant_id, TIMELINE_ID) + .exists()); + + Ok(()) + } } #[cfg(not(debug_assertions))] diff --git a/pageserver/src/tenant/config.rs b/pageserver/src/tenant/config.rs index 50de316bc4..ffe2c5eab6 100644 --- a/pageserver/src/tenant/config.rs +++ b/pageserver/src/tenant/config.rs @@ -38,8 +38,8 @@ pub mod defaults { pub const DEFAULT_GC_PERIOD: &str = "1 hr"; pub const DEFAULT_IMAGE_CREATION_THRESHOLD: usize = 3; pub const DEFAULT_PITR_INTERVAL: &str = "7 days"; - pub const DEFAULT_WALRECEIVER_CONNECT_TIMEOUT: &str = "2 seconds"; - pub const DEFAULT_WALRECEIVER_LAGGING_WAL_TIMEOUT: &str = "3 seconds"; + pub const DEFAULT_WALRECEIVER_CONNECT_TIMEOUT: &str = "10 seconds"; + pub const DEFAULT_WALRECEIVER_LAGGING_WAL_TIMEOUT: &str = "10 seconds"; pub const DEFAULT_MAX_WALRECEIVER_LSN_WAL_LAG: u64 = 10 * 1024 * 1024; pub const DEFAULT_EVICTIONS_LOW_RESIDENCE_DURATION_METRIC_THRESHOLD: &str = "24 hour"; } @@ -99,6 +99,7 @@ pub struct TenantConf { // See the corresponding metric's help string. #[serde(with = "humantime_serde")] pub evictions_low_residence_duration_metric_threshold: Duration, + pub gc_feedback: bool, } /// Same as TenantConf, but this struct preserves the information about @@ -175,6 +176,10 @@ pub struct TenantConfOpt { #[serde(with = "humantime_serde")] #[serde(default)] pub evictions_low_residence_duration_metric_threshold: Option, + + #[serde(skip_serializing_if = "Option::is_none")] + #[serde(default)] + pub gc_feedback: Option, } #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] @@ -242,6 +247,7 @@ impl TenantConfOpt { evictions_low_residence_duration_metric_threshold: self .evictions_low_residence_duration_metric_threshold .unwrap_or(global_conf.evictions_low_residence_duration_metric_threshold), + gc_feedback: self.gc_feedback.unwrap_or(global_conf.gc_feedback), } } } @@ -278,6 +284,7 @@ impl Default for TenantConf { DEFAULT_EVICTIONS_LOW_RESIDENCE_DURATION_METRIC_THRESHOLD, ) .expect("cannot parse default evictions_low_residence_duration_metric_threshold"), + gc_feedback: false, } } } @@ -372,6 +379,7 @@ impl TryFrom<&'_ models::TenantConfig> for TenantConfOpt { ))?, ); } + tenant_conf.gc_feedback = request_data.gc_feedback; Ok(tenant_conf) } diff --git a/pageserver/src/tenant/layer_map.rs b/pageserver/src/tenant/layer_map.rs index 8d06ccd565..dee02ac433 100644 --- a/pageserver/src/tenant/layer_map.rs +++ b/pageserver/src/tenant/layer_map.rs @@ -58,14 +58,16 @@ use std::sync::Arc; use utils::lsn::Lsn; use historic_layer_coverage::BufferedHistoricLayerCoverage; -pub use historic_layer_coverage::Replacement; +pub use historic_layer_coverage::LayerKey; use super::storage_layer::range_eq; +use super::storage_layer::PersistentLayerDesc; /// /// LayerMap tracks what layers exist on a timeline. /// -pub struct LayerMap { +#[derive(Default)] +pub struct LayerMap { // // 'open_layer' holds the current InMemoryLayer that is accepting new // records. If it is None, 'next_open_layer_at' will be set instead, indicating @@ -86,23 +88,11 @@ pub struct LayerMap { pub frozen_layers: VecDeque>, /// Index of the historic layers optimized for search - historic: BufferedHistoricLayerCoverage>, + historic: BufferedHistoricLayerCoverage>, /// L0 layers have key range Key::MIN..Key::MAX, and locating them using R-Tree search is very inefficient. /// So L0 layers are held in l0_delta_layers vector, in addition to the R-tree. - l0_delta_layers: Vec>, -} - -impl Default for LayerMap { - fn default() -> Self { - Self { - open_layer: None, - next_open_layer_at: None, - frozen_layers: VecDeque::default(), - l0_delta_layers: Vec::default(), - historic: BufferedHistoricLayerCoverage::default(), - } - } + l0_delta_layers: Vec>, } /// The primary update API for the layer map. @@ -110,23 +100,21 @@ impl Default for LayerMap { /// Batching historic layer insertions and removals is good for /// performance and this struct helps us do that correctly. #[must_use] -pub struct BatchedUpdates<'a, L: ?Sized + Layer> { +pub struct BatchedUpdates<'a> { // While we hold this exclusive reference to the layer map the type checker // will prevent us from accidentally reading any unflushed updates. - layer_map: &'a mut LayerMap, + layer_map: &'a mut LayerMap, } /// Provide ability to batch more updates while hiding the read /// API so we don't accidentally read without flushing. -impl BatchedUpdates<'_, L> -where - L: ?Sized + Layer, -{ +impl BatchedUpdates<'_> { /// /// Insert an on-disk layer. /// - pub fn insert_historic(&mut self, layer: Arc) { - self.layer_map.insert_historic_noflush(layer) + // TODO remove the `layer` argument when `mapping` is refactored out of `LayerMap` + pub fn insert_historic(&mut self, layer_desc: PersistentLayerDesc) { + self.layer_map.insert_historic_noflush(layer_desc) } /// @@ -134,28 +122,8 @@ where /// /// This should be called when the corresponding file on disk has been deleted. /// - pub fn remove_historic(&mut self, layer: Arc) { - self.layer_map.remove_historic_noflush(layer) - } - - /// Replaces existing layer iff it is the `expected`. - /// - /// If the expected layer has been removed it will not be inserted by this function. - /// - /// Returned `Replacement` describes succeeding in replacement or the reason why it could not - /// be done. - /// - /// TODO replacement can be done without buffering and rebuilding layer map updates. - /// One way to do that is to add a layer of indirection for returned values, so - /// that we can replace values only by updating a hashmap. - pub fn replace_historic( - &mut self, - expected: &Arc, - new: Arc, - ) -> anyhow::Result>> { - fail::fail_point!("layermap-replace-notfound", |_| Ok(Replacement::NotFound)); - - self.layer_map.replace_historic_noflush(expected, new) + pub fn remove_historic(&mut self, layer_desc: PersistentLayerDesc) { + self.layer_map.remove_historic_noflush(layer_desc) } // We will flush on drop anyway, but this method makes it @@ -171,25 +139,19 @@ where // than panic later or read without flushing. // // TODO maybe warn if flush hasn't explicitly been called -impl Drop for BatchedUpdates<'_, L> -where - L: ?Sized + Layer, -{ +impl Drop for BatchedUpdates<'_> { fn drop(&mut self) { self.layer_map.flush_updates(); } } /// Return value of LayerMap::search -pub struct SearchResult { - pub layer: Arc, +pub struct SearchResult { + pub layer: Arc, pub lsn_floor: Lsn, } -impl LayerMap -where - L: ?Sized + Layer, -{ +impl LayerMap { /// /// Find the latest layer (by lsn.end) that covers the given /// 'key', with lsn.start < 'end_lsn'. @@ -221,7 +183,7 @@ where /// NOTE: This only searches the 'historic' layers, *not* the /// 'open' and 'frozen' layers! /// - pub fn search(&self, key: Key, end_lsn: Lsn) -> Option> { + pub fn search(&self, key: Key, end_lsn: Lsn) -> Option { let version = self.historic.get().unwrap().get_version(end_lsn.0 - 1)?; let latest_delta = version.delta_coverage.query(key.to_i128()); let latest_image = version.image_coverage.query(key.to_i128()); @@ -264,7 +226,7 @@ where } /// Start a batch of updates, applied on drop - pub fn batch_update(&mut self) -> BatchedUpdates<'_, L> { + pub fn batch_update(&mut self) -> BatchedUpdates<'_> { BatchedUpdates { layer_map: self } } @@ -273,16 +235,18 @@ where /// /// Helper function for BatchedUpdates::insert_historic /// - pub(self) fn insert_historic_noflush(&mut self, layer: Arc) { + /// TODO(chi): remove L generic so that we do not need to pass layer object. + pub(self) fn insert_historic_noflush(&mut self, layer_desc: PersistentLayerDesc) { // TODO: See #3869, resulting #4088, attempted fix and repro #4094 - self.historic.insert( - historic_layer_coverage::LayerKey::from(&*layer), - Arc::clone(&layer), - ); - if Self::is_l0(&layer) { - self.l0_delta_layers.push(layer); + if Self::is_l0(&layer_desc) { + self.l0_delta_layers.push(layer_desc.clone().into()); } + + self.historic.insert( + historic_layer_coverage::LayerKey::from(&layer_desc), + layer_desc.into(), + ); } /// @@ -290,14 +254,15 @@ where /// /// Helper function for BatchedUpdates::remove_historic /// - pub fn remove_historic_noflush(&mut self, layer: Arc) { + pub fn remove_historic_noflush(&mut self, layer_desc: PersistentLayerDesc) { self.historic - .remove(historic_layer_coverage::LayerKey::from(&*layer)); - - if Self::is_l0(&layer) { + .remove(historic_layer_coverage::LayerKey::from(&layer_desc)); + let layer_key = layer_desc.key(); + if Self::is_l0(&layer_desc) { let len_before = self.l0_delta_layers.len(); - self.l0_delta_layers - .retain(|other| !Self::compare_arced_layers(other, &layer)); + let mut l0_delta_layers = std::mem::take(&mut self.l0_delta_layers); + l0_delta_layers.retain(|other| other.key() != layer_key); + self.l0_delta_layers = l0_delta_layers; // this assertion is related to use of Arc::ptr_eq in Self::compare_arced_layers, // there's a chance that the comparison fails at runtime due to it comparing (pointer, // vtable) pairs. @@ -309,55 +274,6 @@ where } } - pub(self) fn replace_historic_noflush( - &mut self, - expected: &Arc, - new: Arc, - ) -> anyhow::Result>> { - let key = historic_layer_coverage::LayerKey::from(&**expected); - let other = historic_layer_coverage::LayerKey::from(&*new); - - let expected_l0 = Self::is_l0(expected); - let new_l0 = Self::is_l0(&new); - - anyhow::ensure!( - key == other, - "expected and new must have equal LayerKeys: {key:?} != {other:?}" - ); - - anyhow::ensure!( - expected_l0 == new_l0, - "expected and new must both be l0 deltas or neither should be: {expected_l0} != {new_l0}" - ); - - let l0_index = if expected_l0 { - // find the index in case replace worked, we need to replace that as well - let pos = self - .l0_delta_layers - .iter() - .position(|slot| Self::compare_arced_layers(slot, expected)); - - if pos.is_none() { - return Ok(Replacement::NotFound); - } - pos - } else { - None - }; - - let replaced = self.historic.replace(&key, new.clone(), |existing| { - Self::compare_arced_layers(existing, expected) - }); - - if let Replacement::Replaced { .. } = &replaced { - if let Some(index) = l0_index { - self.l0_delta_layers[index] = new; - } - } - - Ok(replaced) - } - /// Helper function for BatchedUpdates::drop. pub(self) fn flush_updates(&mut self) { self.historic.rebuild(); @@ -383,7 +299,7 @@ where let start = key.start.to_i128(); let end = key.end.to_i128(); - let layer_covers = |layer: Option>| match layer { + let layer_covers = |layer: Option>| match layer { Some(layer) => layer.get_lsn_range().start >= lsn.start, None => false, }; @@ -403,7 +319,7 @@ where Ok(true) } - pub fn iter_historic_layers(&self) -> impl '_ + Iterator> { + pub fn iter_historic_layers(&self) -> impl '_ + Iterator> { self.historic.iter() } @@ -419,7 +335,7 @@ where &self, key_range: &Range, lsn: Lsn, - ) -> Result, Option>)>> { + ) -> Result, Option>)>> { let version = match self.historic.get().unwrap().get_version(lsn.0) { Some(v) => v, None => return Ok(vec![]), @@ -429,7 +345,7 @@ where let end = key_range.end.to_i128(); // Initialize loop variables - let mut coverage: Vec<(Range, Option>)> = vec![]; + let mut coverage: Vec<(Range, Option>)> = vec![]; let mut current_key = start; let mut current_val = version.image_coverage.query(start); @@ -448,7 +364,7 @@ where Ok(coverage) } - pub fn is_l0(layer: &L) -> bool { + pub fn is_l0(layer: &PersistentLayerDesc) -> bool { range_eq(&layer.get_key_range(), &(Key::MIN..Key::MAX)) } @@ -474,7 +390,7 @@ where /// TODO The optimal number should probably be slightly higher than 1, but to /// implement that we need to plumb a lot more context into this function /// than just the current partition_range. - pub fn is_reimage_worthy(layer: &L, partition_range: &Range) -> bool { + pub fn is_reimage_worthy(layer: &PersistentLayerDesc, partition_range: &Range) -> bool { // Case 1 if !Self::is_l0(layer) { return true; @@ -705,8 +621,8 @@ where } /// Return all L0 delta layers - pub fn get_level0_deltas(&self) -> Result>> { - Ok(self.l0_delta_layers.clone()) + pub fn get_level0_deltas(&self) -> Result>> { + Ok(self.l0_delta_layers.to_vec()) } /// debugging function to print out the contents of the layer map @@ -731,72 +647,51 @@ where println!("End dump LayerMap"); Ok(()) } - - /// Similar to `Arc::ptr_eq`, but only compares the object pointers, not vtables. - /// - /// Returns `true` if the two `Arc` point to the same layer, false otherwise. - #[inline(always)] - pub fn compare_arced_layers(left: &Arc, right: &Arc) -> bool { - // "dyn Trait" objects are "fat pointers" in that they have two components: - // - pointer to the object - // - pointer to the vtable - // - // rust does not provide a guarantee that these vtables are unique, but however - // `Arc::ptr_eq` as of writing (at least up to 1.67) uses a comparison where both the - // pointer and the vtable need to be equal. - // - // See: https://github.com/rust-lang/rust/issues/103763 - // - // A future version of rust will most likely use this form below, where we cast each - // pointer into a pointer to unit, which drops the inaccessible vtable pointer, making it - // not affect the comparison. - // - // See: https://github.com/rust-lang/rust/pull/106450 - let left = Arc::as_ptr(left) as *const (); - let right = Arc::as_ptr(right) as *const (); - - left == right - } } #[cfg(test)] mod tests { - use super::{LayerMap, Replacement}; - use crate::tenant::storage_layer::{Layer, LayerDescriptor, LayerFileName}; + use super::LayerMap; + use crate::tenant::storage_layer::{tests::LayerDescriptor, LayerFileName}; use std::str::FromStr; use std::sync::Arc; mod l0_delta_layers_updated { + use crate::tenant::{ + storage_layer::{PersistentLayer, PersistentLayerDesc}, + timeline::LayerFileManager, + }; + use super::*; #[test] fn for_full_range_delta() { // l0_delta_layers are used by compaction, and should observe all buffered updates l0_delta_layers_updated_scenario( - "000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000053423C21-0000000053424D69", - true - ) + "000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000053423C21-0000000053424D69", + true + ) } #[test] fn for_non_full_range_delta() { // has minimal uncovered areas compared to l0_delta_layers_updated_on_insert_replace_remove_for_full_range_delta l0_delta_layers_updated_scenario( - "000000000000000000000000000000000001-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFE__0000000053423C21-0000000053424D69", - // because not full range - false - ) + "000000000000000000000000000000000001-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFE__0000000053423C21-0000000053424D69", + // because not full range + false + ) } #[test] fn for_image() { l0_delta_layers_updated_scenario( - "000000000000000000000000000000000000-000000000000000000000000000000010000__0000000053424D69", - // code only checks if it is a full range layer, doesn't care about images, which must - // mean we should in practice never have full range images - false - ) + "000000000000000000000000000000000000-000000000000000000000000000000010000__0000000053424D69", + // code only checks if it is a full range layer, doesn't care about images, which must + // mean we should in practice never have full range images + false + ) } #[test] @@ -809,60 +704,67 @@ mod tests { let layer = LayerDescriptor::from(layer); // same skeletan construction; see scenario below - let not_found: Arc = Arc::new(layer.clone()); - let new_version: Arc = Arc::new(layer); + let not_found = Arc::new(layer.clone()); + let new_version = Arc::new(layer); - let mut map = LayerMap::default(); + // after the immutable storage state refactor, the replace operation + // will not use layer map any more. We keep it here for consistency in test cases + // and can remove it in the future. + let _map = LayerMap::default(); - let res = map.batch_update().replace_historic(¬_found, new_version); + let mut mapping = LayerFileManager::new(); - assert!(matches!(res, Ok(Replacement::NotFound)), "{res:?}"); + mapping + .replace_and_verify(not_found, new_version) + .unwrap_err(); } fn l0_delta_layers_updated_scenario(layer_name: &str, expected_l0: bool) { let name = LayerFileName::from_str(layer_name).unwrap(); let skeleton = LayerDescriptor::from(name); - let remote: Arc = Arc::new(skeleton.clone()); - let downloaded: Arc = Arc::new(skeleton); + let remote = Arc::new(skeleton.clone()); + let downloaded = Arc::new(skeleton); let mut map = LayerMap::default(); + let mut mapping = LayerFileManager::new(); // two disjoint Arcs in different lifecycle phases. even if it seems they must be the // same layer, we use LayerMap::compare_arced_layers as the identity of layers. - assert!(!LayerMap::compare_arced_layers(&remote, &downloaded)); + assert_eq!(remote.layer_desc(), downloaded.layer_desc()); let expected_in_counts = (1, usize::from(expected_l0)); - map.batch_update().insert_historic(remote.clone()); - assert_eq!(count_layer_in(&map, &remote), expected_in_counts); - - let replaced = map - .batch_update() - .replace_historic(&remote, downloaded.clone()) - .expect("name derived attributes are the same"); - assert!( - matches!(replaced, Replacement::Replaced { .. }), - "{replaced:?}" + map.batch_update() + .insert_historic(remote.layer_desc().clone()); + mapping.insert(remote.clone()); + assert_eq!( + count_layer_in(&map, remote.layer_desc()), + expected_in_counts ); - assert_eq!(count_layer_in(&map, &downloaded), expected_in_counts); - map.batch_update().remove_historic(downloaded.clone()); - assert_eq!(count_layer_in(&map, &downloaded), (0, 0)); + mapping + .replace_and_verify(remote, downloaded.clone()) + .expect("name derived attributes are the same"); + assert_eq!( + count_layer_in(&map, downloaded.layer_desc()), + expected_in_counts + ); + + map.batch_update() + .remove_historic(downloaded.layer_desc().clone()); + assert_eq!(count_layer_in(&map, downloaded.layer_desc()), (0, 0)); } - fn count_layer_in(map: &LayerMap, layer: &Arc) -> (usize, usize) { + fn count_layer_in(map: &LayerMap, layer: &PersistentLayerDesc) -> (usize, usize) { let historic = map .iter_historic_layers() - .filter(|x| LayerMap::compare_arced_layers(x, layer)) + .filter(|x| x.key() == layer.key()) .count(); let l0s = map .get_level0_deltas() .expect("why does this return a result"); - let l0 = l0s - .iter() - .filter(|x| LayerMap::compare_arced_layers(x, layer)) - .count(); + let l0 = l0s.iter().filter(|x| x.key() == layer.key()).count(); (historic, l0) } diff --git a/pageserver/src/tenant/layer_map/historic_layer_coverage.rs b/pageserver/src/tenant/layer_map/historic_layer_coverage.rs index b63c361314..0f51597027 100644 --- a/pageserver/src/tenant/layer_map/historic_layer_coverage.rs +++ b/pageserver/src/tenant/layer_map/historic_layer_coverage.rs @@ -3,6 +3,8 @@ use std::ops::Range; use tracing::info; +use crate::tenant::storage_layer::PersistentLayerDesc; + use super::layer_coverage::LayerCoverageTuple; /// Layers in this module are identified and indexed by this data. @@ -41,8 +43,8 @@ impl Ord for LayerKey { } } -impl<'a, L: crate::tenant::storage_layer::Layer + ?Sized> From<&'a L> for LayerKey { - fn from(layer: &'a L) -> Self { +impl From<&PersistentLayerDesc> for LayerKey { + fn from(layer: &PersistentLayerDesc) -> Self { let kr = layer.get_key_range(); let lr = layer.get_lsn_range(); LayerKey { @@ -204,6 +206,35 @@ fn test_off_by_one() { assert_eq!(version.image_coverage.query(5), None); } +/// White-box regression test, checking for incorrect removal of node at key.end +#[test] +fn test_regression() { + let mut map = HistoricLayerCoverage::::new(); + map.insert( + LayerKey { + key: 0..5, + lsn: 0..5, + is_image: false, + }, + "Layer 1".to_string(), + ); + map.insert( + LayerKey { + key: 0..5, + lsn: 1..2, + is_image: false, + }, + "Layer 2".to_string(), + ); + + // If an insertion operation improperly deletes the endpoint of a previous layer + // (which is more likely to happen with layers that collide on key.end), we will + // end up with an infinite layer, covering the entire keyspace. Here we assert + // that there's no layer at key 100 because we didn't insert any layer there. + let version = map.get_version(100).unwrap(); + assert_eq!(version.delta_coverage.query(100), None); +} + /// Cover edge cases where layers begin or end on the same key #[test] fn test_key_collision() { @@ -425,59 +456,6 @@ impl BufferedHistoricLayerCoverage { self.buffer.insert(layer_key, None); } - /// Replaces a previous layer with a new layer value. - /// - /// The replacement is conditional on: - /// - there is an existing `LayerKey` record - /// - there is no buffered removal for the given `LayerKey` - /// - the given closure returns true for the current `Value` - /// - /// The closure is used to compare the latest value (buffered insert, or existing layer) - /// against some expectation. This allows to use `Arc::ptr_eq` or similar which would be - /// inaccessible via `PartialEq` trait. - /// - /// Returns a `Replacement` value describing the outcome; only the case of - /// `Replacement::Replaced` modifies the map and requires a rebuild. - pub fn replace( - &mut self, - layer_key: &LayerKey, - new: Value, - check_expected: F, - ) -> Replacement - where - F: FnOnce(&Value) -> bool, - { - let (slot, in_buffered) = match self.buffer.get(layer_key) { - Some(inner @ Some(_)) => { - // we compare against the buffered version, because there will be a later - // rebuild before querying - (inner.as_ref(), true) - } - Some(None) => { - // buffer has removal for this key; it will not be equivalent by any check_expected. - return Replacement::RemovalBuffered; - } - None => { - // no pending modification for the key, check layers - (self.layers.get(layer_key), false) - } - }; - - match slot { - Some(existing) if !check_expected(existing) => { - // unfortunate clone here, but otherwise the nll borrowck grows the region of - // 'a to cover the whole function, and we could not mutate in the other - // Some(existing) branch - Replacement::Unexpected(existing.clone()) - } - None => Replacement::NotFound, - Some(_existing) => { - self.insert(layer_key.to_owned(), new); - Replacement::Replaced { in_buffered } - } - } - } - pub fn rebuild(&mut self) { // Find the first LSN that needs to be rebuilt let rebuild_since: u64 = match self.buffer.iter().next() { @@ -546,22 +524,6 @@ impl BufferedHistoricLayerCoverage { } } -/// Outcome of the replace operation. -#[derive(Debug)] -pub enum Replacement { - /// Previous value was replaced with the new value. - Replaced { - /// Replacement happened for a scheduled insert. - in_buffered: bool, - }, - /// Key was not found buffered updates or existing layers. - NotFound, - /// Key has been scheduled for removal, it was not replaced. - RemovalBuffered, - /// Previous value was rejected by the closure. - Unexpected(Value), -} - #[test] fn test_retroactive_regression_1() { let mut map = BufferedHistoricLayerCoverage::new(); @@ -670,139 +632,3 @@ fn test_retroactive_simple() { assert_eq!(version.image_coverage.query(8), Some("Image 4".to_string())); } } - -#[test] -fn test_retroactive_replacement() { - let mut map = BufferedHistoricLayerCoverage::new(); - - let keys = [ - LayerKey { - key: 0..5, - lsn: 100..101, - is_image: true, - }, - LayerKey { - key: 3..9, - lsn: 110..111, - is_image: true, - }, - LayerKey { - key: 4..6, - lsn: 120..121, - is_image: true, - }, - ]; - - let layers = [ - "Image 1".to_string(), - "Image 2".to_string(), - "Image 3".to_string(), - ]; - - for (key, layer) in keys.iter().zip(layers.iter()) { - map.insert(key.to_owned(), layer.to_owned()); - } - - // rebuild is not necessary here, because replace works for both buffered updates and existing - // layers. - - for (key, orig_layer) in keys.iter().zip(layers.iter()) { - let replacement = format!("Remote {orig_layer}"); - - // evict - let ret = map.replace(key, replacement.clone(), |l| l == orig_layer); - assert!( - matches!(ret, Replacement::Replaced { .. }), - "replace {orig_layer}: {ret:?}" - ); - map.rebuild(); - - let at = key.lsn.end + 1; - - let version = map.get().expect("rebuilt").get_version(at).unwrap(); - assert_eq!( - version.image_coverage.query(4).as_deref(), - Some(replacement.as_str()), - "query for 4 at version {at} after eviction", - ); - - // download - let ret = map.replace(key, orig_layer.clone(), |l| l == &replacement); - assert!( - matches!(ret, Replacement::Replaced { .. }), - "replace {orig_layer} back: {ret:?}" - ); - map.rebuild(); - let version = map.get().expect("rebuilt").get_version(at).unwrap(); - assert_eq!( - version.image_coverage.query(4).as_deref(), - Some(orig_layer.as_str()), - "query for 4 at version {at} after download", - ); - } -} - -#[test] -fn missing_key_is_not_inserted_with_replace() { - let mut map = BufferedHistoricLayerCoverage::new(); - let key = LayerKey { - key: 0..5, - lsn: 100..101, - is_image: true, - }; - - let ret = map.replace(&key, "should not replace", |_| true); - assert!(matches!(ret, Replacement::NotFound), "{ret:?}"); - map.rebuild(); - assert!(map - .get() - .expect("no changes to rebuild") - .get_version(102) - .is_none()); -} - -#[test] -fn replacing_buffered_insert_and_remove() { - let mut map = BufferedHistoricLayerCoverage::new(); - let key = LayerKey { - key: 0..5, - lsn: 100..101, - is_image: true, - }; - - map.insert(key.clone(), "Image 1"); - let ret = map.replace(&key, "Remote Image 1", |&l| l == "Image 1"); - assert!( - matches!(ret, Replacement::Replaced { in_buffered: true }), - "{ret:?}" - ); - map.rebuild(); - - assert_eq!( - map.get() - .expect("rebuilt") - .get_version(102) - .unwrap() - .image_coverage - .query(4), - Some("Remote Image 1") - ); - - map.remove(key.clone()); - let ret = map.replace(&key, "should not replace", |_| true); - assert!( - matches!(ret, Replacement::RemovalBuffered), - "cannot replace after scheduled remove: {ret:?}" - ); - - map.rebuild(); - - let ret = map.replace(&key, "should not replace", |_| true); - assert!( - matches!(ret, Replacement::NotFound), - "cannot replace after remove + rebuild: {ret:?}" - ); - - let at_version = map.get().expect("rebuilt").get_version(102); - assert!(at_version.is_none()); -} diff --git a/pageserver/src/tenant/layer_map/layer_coverage.rs b/pageserver/src/tenant/layer_map/layer_coverage.rs index 4e3b4516dc..47aace97a5 100644 --- a/pageserver/src/tenant/layer_map/layer_coverage.rs +++ b/pageserver/src/tenant/layer_map/layer_coverage.rs @@ -1,8 +1,8 @@ use std::ops::Range; -// TODO the `im` crate has 20x more downloads and also has -// persistent/immutable BTree. It also runs a bit faster but -// results are not the same on some tests. +// NOTE the `im` crate has 20x more downloads and also has +// persistent/immutable BTree. But it's bugged so rpds is a +// better choice https://github.com/neondatabase/neon/issues/3395 use rpds::RedBlackTreeMapSync; /// Data structure that can efficiently: @@ -10,19 +10,22 @@ use rpds::RedBlackTreeMapSync; /// - iterate the latest layers in a key range /// - insert layers in non-decreasing lsn.start order /// -/// The struct is parameterized over Value for easier -/// testing, but in practice it's some sort of layer. +/// For a detailed explanation and justification of this approach, see: +/// https://neon.tech/blog/persistent-structures-in-neons-wal-indexing +/// +/// NOTE The struct is parameterized over Value for easier +/// testing, but in practice it's some sort of layer. pub struct LayerCoverage { /// For every change in coverage (as we sweep the key space) /// we store (lsn.end, value). /// - /// We use an immutable/persistent tree so that we can keep historic - /// versions of this coverage without cloning the whole thing and - /// incurring quadratic memory cost. See HistoricLayerCoverage. + /// NOTE We use an immutable/persistent tree so that we can keep historic + /// versions of this coverage without cloning the whole thing and + /// incurring quadratic memory cost. See HistoricLayerCoverage. /// - /// We use the Sync version of the map because we want Self to - /// be Sync. Using nonsync might be faster, if we can work with - /// that. + /// NOTE We use the Sync version of the map because we want Self to + /// be Sync. Using nonsync might be faster, if we can work with + /// that. nodes: RedBlackTreeMapSync>, } @@ -41,6 +44,13 @@ impl LayerCoverage { /// Helper function to subdivide the key range without changing any values /// + /// This operation has no semantic effect by itself. It only helps us pin in + /// place the part of the coverage we don't want to change when inserting. + /// + /// As an analogy, think of a polygon. If you add a vertex along one of the + /// segments, the polygon is still the same, but it behaves differently when + /// we move or delete one of the other points. + /// /// Complexity: O(log N) fn add_node(&mut self, key: i128) { let value = match self.nodes.range(..=key).last() { @@ -74,7 +84,7 @@ impl LayerCoverage { let mut to_update = Vec::new(); let mut to_remove = Vec::new(); let mut prev_covered = false; - for (k, node) in self.nodes.range(key.clone()) { + for (k, node) in self.nodes.range(key) { let needs_cover = match node { None => true, Some((h, _)) => h < &lsn.end, @@ -87,9 +97,8 @@ impl LayerCoverage { } prev_covered = needs_cover; } - if !prev_covered { - to_remove.push(key.end); - } + // TODO check if the nodes inserted at key.start and key.end are safe + // to remove. It's fine to keep them but they could be redundant. for k in to_update { self.nodes.insert_mut(k, Some((lsn.end, value.clone()))); } diff --git a/pageserver/src/tenant/manifest.rs b/pageserver/src/tenant/manifest.rs new file mode 100644 index 0000000000..745437dfbd --- /dev/null +++ b/pageserver/src/tenant/manifest.rs @@ -0,0 +1,325 @@ +//! This module contains the encoding and decoding of the local manifest file. +//! +//! MANIFEST is a write-ahead log which is stored locally to each timeline. It +//! records the state of the storage engine. It contains a snapshot of the +//! state and all operations proceeding that snapshot. The file begins with a +//! header recording MANIFEST version number. After that, it contains a snapshot. +//! The snapshot is followed by a list of operations. Each operation is a list +//! of records. Each record is either an addition or a removal of a layer. +//! +//! With MANIFEST, we can: +//! +//! 1. recover state quickly by reading the file, potentially boosting the +//! startup speed. +//! 2. ensure all operations are atomic and avoid corruption, solving issues +//! like redundant image layer and preparing us for future compaction +//! strategies. +//! +//! There is also a format for storing all layer files on S3, called +//! `index_part.json`. Compared with index_part, MANIFEST is an WAL which +//! records all operations as logs, and therefore we can easily replay the +//! operations when recovering from crash, while ensuring those operations +//! are atomic upon restart. +//! +//! Currently, this is not used in the system. Future refactors will ensure +//! the storage state will be recorded in this file, and the system can be +//! recovered from this file. This is tracked in +//! https://github.com/neondatabase/neon/issues/4418 + +use std::io::{self, Read, Write}; + +use crate::virtual_file::VirtualFile; +use anyhow::Result; +use bytes::{Buf, BufMut, Bytes, BytesMut}; +use crc32c::crc32c; +use serde::{Deserialize, Serialize}; +use tracing::log::warn; +use utils::lsn::Lsn; + +use super::storage_layer::PersistentLayerDesc; + +pub struct Manifest { + file: VirtualFile, +} + +#[derive(Clone, Serialize, Deserialize, PartialEq, Eq, Debug)] +pub struct Snapshot { + pub layers: Vec, +} + +/// serde by default encode this in tagged enum, and therefore it will be something +/// like `{ "AddLayer": { ... } }`. +#[derive(Clone, Serialize, Deserialize, PartialEq, Eq, Debug)] +pub enum Record { + AddLayer(PersistentLayerDesc), + RemoveLayer(PersistentLayerDesc), +} + +/// `echo neon.manifest | sha1sum` and take the leading 8 bytes. +const MANIFEST_MAGIC_NUMBER: u64 = 0xf5c44592b806109c; +const MANIFEST_VERSION: u64 = 1; + +#[derive(Clone, Serialize, Deserialize, PartialEq, Eq, Debug)] +pub struct ManifestHeader { + magic_number: u64, + version: u64, +} + +const MANIFEST_HEADER_LEN: usize = 16; + +impl ManifestHeader { + fn encode(&self) -> BytesMut { + let mut buf = BytesMut::with_capacity(MANIFEST_HEADER_LEN); + buf.put_u64(self.magic_number); + buf.put_u64(self.version); + buf + } + + fn decode(mut buf: &[u8]) -> Self { + assert!(buf.len() == MANIFEST_HEADER_LEN, "invalid header"); + Self { + magic_number: buf.get_u64(), + version: buf.get_u64(), + } + } +} + +#[derive(Clone, Serialize, Deserialize, PartialEq, Eq, Debug)] +pub enum Operation { + /// A snapshot of the current state. + /// + /// Lsn field represents the LSN that is persisted to disk for this snapshot. + Snapshot(Snapshot, Lsn), + /// An atomic operation that changes the state. + /// + /// Lsn field represents the LSN that is persisted to disk after the operation is done. + /// This will only change when new L0 is flushed to the disk. + Operation(Vec, Lsn), +} + +struct RecordHeader { + size: u32, + checksum: u32, +} + +const RECORD_HEADER_LEN: usize = 8; + +impl RecordHeader { + fn encode(&self) -> BytesMut { + let mut buf = BytesMut::with_capacity(RECORD_HEADER_LEN); + buf.put_u32(self.size); + buf.put_u32(self.checksum); + buf + } + + fn decode(mut buf: &[u8]) -> Self { + assert!(buf.len() == RECORD_HEADER_LEN, "invalid header"); + Self { + size: buf.get_u32(), + checksum: buf.get_u32(), + } + } +} + +#[derive(Debug, thiserror::Error)] +pub enum ManifestLoadError { + #[error("manifest header is corrupted")] + CorruptedManifestHeader, + #[error("unsupported manifest version: got {0}, expected {1}")] + UnsupportedVersion(u64, u64), + #[error("error when decoding record: {0}")] + DecodeRecord(serde_json::Error), + #[error("I/O error: {0}")] + Io(io::Error), +} + +#[must_use = "Should check if the manifest is partially corrupted"] +pub struct ManifestPartiallyCorrupted(bool); + +impl Manifest { + /// Create a new manifest by writing the manifest header and a snapshot record to the given file. + pub fn init(file: VirtualFile, snapshot: Snapshot, lsn: Lsn) -> Result { + let mut manifest = Self { file }; + manifest.append_manifest_header(ManifestHeader { + magic_number: MANIFEST_MAGIC_NUMBER, + version: MANIFEST_VERSION, + })?; + manifest.append_operation(Operation::Snapshot(snapshot, lsn))?; + Ok(manifest) + } + + /// Load a manifest. Returns the manifest and a list of operations. If the manifest is corrupted, + /// the bool flag will be set to true and the user is responsible to reconstruct a new manifest and + /// backup the current one. + pub fn load( + mut file: VirtualFile, + ) -> Result<(Self, Vec, ManifestPartiallyCorrupted), ManifestLoadError> { + let mut buf = vec![]; + file.read_to_end(&mut buf).map_err(ManifestLoadError::Io)?; + + // Read manifest header + let mut buf = Bytes::from(buf); + if buf.remaining() < MANIFEST_HEADER_LEN { + return Err(ManifestLoadError::CorruptedManifestHeader); + } + let header = ManifestHeader::decode(&buf[..MANIFEST_HEADER_LEN]); + buf.advance(MANIFEST_HEADER_LEN); + if header.version != MANIFEST_VERSION { + return Err(ManifestLoadError::UnsupportedVersion( + header.version, + MANIFEST_VERSION, + )); + } + + // Read operations + let mut operations = Vec::new(); + let corrupted = loop { + if buf.remaining() == 0 { + break false; + } + if buf.remaining() < RECORD_HEADER_LEN { + warn!("incomplete header when decoding manifest, could be corrupted"); + break true; + } + let RecordHeader { size, checksum } = RecordHeader::decode(&buf[..RECORD_HEADER_LEN]); + let size = size as usize; + buf.advance(RECORD_HEADER_LEN); + if buf.remaining() < size { + warn!("incomplete data when decoding manifest, could be corrupted"); + break true; + } + let data = &buf[..size]; + if crc32c(data) != checksum { + warn!("checksum mismatch when decoding manifest, could be corrupted"); + break true; + } + // if the following decode fails, we cannot use the manifest or safely ignore any record. + operations.push(serde_json::from_slice(data).map_err(ManifestLoadError::DecodeRecord)?); + buf.advance(size); + }; + Ok(( + Self { file }, + operations, + ManifestPartiallyCorrupted(corrupted), + )) + } + + fn append_data(&mut self, data: &[u8]) -> Result<()> { + if data.len() >= u32::MAX as usize { + panic!("data too large"); + } + let header = RecordHeader { + size: data.len() as u32, + checksum: crc32c(data), + }; + let header = header.encode(); + self.file.write_all(&header)?; + self.file.write_all(data)?; + self.file.sync_all()?; + Ok(()) + } + + fn append_manifest_header(&mut self, header: ManifestHeader) -> Result<()> { + let encoded = header.encode(); + self.file.write_all(&encoded)?; + Ok(()) + } + + /// Add an operation to the manifest. The operation will be appended to the end of the file, + /// and the file will fsync. + pub fn append_operation(&mut self, operation: Operation) -> Result<()> { + let encoded = Vec::from(serde_json::to_string(&operation)?); + self.append_data(&encoded) + } +} + +#[cfg(test)] +mod tests { + use std::fs::OpenOptions; + + use crate::repository::Key; + + use super::*; + + #[test] + fn test_read_manifest() { + let testdir = crate::config::PageServerConf::test_repo_dir("test_read_manifest"); + std::fs::create_dir_all(&testdir).unwrap(); + let file = VirtualFile::create(&testdir.join("MANIFEST")).unwrap(); + let layer1 = PersistentLayerDesc::new_test(Key::from_i128(0)..Key::from_i128(233)); + let layer2 = PersistentLayerDesc::new_test(Key::from_i128(233)..Key::from_i128(2333)); + let layer3 = PersistentLayerDesc::new_test(Key::from_i128(2333)..Key::from_i128(23333)); + let layer4 = PersistentLayerDesc::new_test(Key::from_i128(23333)..Key::from_i128(233333)); + + // Write a manifest with a snapshot and some operations + let snapshot = Snapshot { + layers: vec![layer1, layer2], + }; + let mut manifest = Manifest::init(file, snapshot.clone(), Lsn::from(0)).unwrap(); + manifest + .append_operation(Operation::Operation( + vec![Record::AddLayer(layer3.clone())], + Lsn::from(1), + )) + .unwrap(); + drop(manifest); + + // Open the second time and write + let file = VirtualFile::open_with_options( + &testdir.join("MANIFEST"), + OpenOptions::new() + .read(true) + .write(true) + .create_new(false) + .truncate(false), + ) + .unwrap(); + let (mut manifest, operations, corrupted) = Manifest::load(file).unwrap(); + assert!(!corrupted.0); + assert_eq!(operations.len(), 2); + assert_eq!( + &operations[0], + &Operation::Snapshot(snapshot.clone(), Lsn::from(0)) + ); + assert_eq!( + &operations[1], + &Operation::Operation(vec![Record::AddLayer(layer3.clone())], Lsn::from(1)) + ); + manifest + .append_operation(Operation::Operation( + vec![ + Record::RemoveLayer(layer3.clone()), + Record::AddLayer(layer4.clone()), + ], + Lsn::from(2), + )) + .unwrap(); + drop(manifest); + + // Open the third time and verify + let file = VirtualFile::open_with_options( + &testdir.join("MANIFEST"), + OpenOptions::new() + .read(true) + .write(true) + .create_new(false) + .truncate(false), + ) + .unwrap(); + let (_manifest, operations, corrupted) = Manifest::load(file).unwrap(); + assert!(!corrupted.0); + assert_eq!(operations.len(), 3); + assert_eq!(&operations[0], &Operation::Snapshot(snapshot, Lsn::from(0))); + assert_eq!( + &operations[1], + &Operation::Operation(vec![Record::AddLayer(layer3.clone())], Lsn::from(1)) + ); + assert_eq!( + &operations[2], + &Operation::Operation( + vec![Record::RemoveLayer(layer3), Record::AddLayer(layer4)], + Lsn::from(2) + ) + ); + } +} diff --git a/pageserver/src/tenant/mgr.rs b/pageserver/src/tenant/mgr.rs index a50cb59deb..c54b8b8629 100644 --- a/pageserver/src/tenant/mgr.rs +++ b/pageserver/src/tenant/mgr.rs @@ -11,6 +11,7 @@ use tokio::fs; use anyhow::Context; use once_cell::sync::Lazy; use tokio::sync::RwLock; +use tokio::task::JoinSet; use tracing::*; use remote_storage::GenericRemoteStorage; @@ -21,7 +22,7 @@ use crate::context::{DownloadBehavior, RequestContext}; use crate::task_mgr::{self, TaskKind}; use crate::tenant::config::TenantConfOpt; use crate::tenant::{create_tenant_files, CreateTenantFilesMode, Tenant, TenantState}; -use crate::IGNORED_TENANT_FILE_NAME; +use crate::{InitializationOrder, IGNORED_TENANT_FILE_NAME}; use utils::fs_ext::PathExt; use utils::id::{TenantId, TimelineId}; @@ -64,6 +65,7 @@ pub async fn init_tenant_mgr( conf: &'static PageServerConf, broker_client: storage_broker::BrokerClientChannel, remote_storage: Option, + init_order: InitializationOrder, ) -> anyhow::Result<()> { // Scan local filesystem for attached tenants let tenants_dir = conf.tenants_path(); @@ -120,6 +122,7 @@ pub async fn init_tenant_mgr( &tenant_dir_path, broker_client.clone(), remote_storage.clone(), + Some(init_order.clone()), &ctx, ) { Ok(tenant) => { @@ -155,6 +158,7 @@ pub fn schedule_local_tenant_processing( tenant_path: &Path, broker_client: storage_broker::BrokerClientChannel, remote_storage: Option, + init_order: Option, ctx: &RequestContext, ) -> anyhow::Result> { anyhow::ensure!( @@ -208,7 +212,14 @@ pub fn schedule_local_tenant_processing( } else { info!("tenant {tenant_id} is assumed to be loadable, starting load operation"); // Start loading the tenant into memory. It will initially be in Loading state. - Tenant::spawn_load(conf, tenant_id, broker_client, remote_storage, ctx) + Tenant::spawn_load( + conf, + tenant_id, + broker_client, + remote_storage, + init_order, + ctx, + ) }; Ok(tenant) } @@ -223,6 +234,7 @@ pub fn schedule_local_tenant_processing( /// That could be easily misinterpreted by control plane, the consumer of the /// management API. For example, it could attach the tenant on a different pageserver. /// We would then be in split-brain once this pageserver restarts. +#[instrument] pub async fn shutdown_all_tenants() { // Prevent new tenants from being created. let tenants_to_shut_down = { @@ -239,39 +251,51 @@ pub async fn shutdown_all_tenants() { tenants_clone } TenantsMap::ShuttingDown(_) => { + // TODO: it is possible that detach and shutdown happen at the same time. as a + // result, during shutdown we do not wait for detach. error!("already shutting down, this function isn't supposed to be called more than once"); return; } } }; - let mut tenants_to_freeze_and_flush = Vec::with_capacity(tenants_to_shut_down.len()); - for (_, tenant) in tenants_to_shut_down { - if tenant.is_active() { - // updates tenant state, forbidding new GC and compaction iterations from starting - tenant.set_stopping(); - tenants_to_freeze_and_flush.push(tenant); + let mut join_set = JoinSet::new(); + for (tenant_id, tenant) in tenants_to_shut_down { + join_set.spawn( + async move { + let freeze_and_flush = true; + + match tenant.shutdown(freeze_and_flush).await { + Ok(()) => debug!("tenant successfully stopped"), + Err(super::ShutdownError::AlreadyStopping) => { + warn!("tenant was already shutting down") + } + } + } + .instrument(info_span!("shutdown", %tenant_id)), + ); + } + + let mut panicked = 0; + + while let Some(res) = join_set.join_next().await { + match res { + Ok(()) => {} + Err(join_error) if join_error.is_cancelled() => { + unreachable!("we are not cancelling any of the futures"); + } + Err(join_error) if join_error.is_panic() => { + // cannot really do anything, as this panic is likely a bug + panicked += 1; + } + Err(join_error) => { + warn!("unknown kind of JoinError: {join_error}"); + } } } - // Shut down all existing walreceiver connections and stop accepting the new ones. - task_mgr::shutdown_tasks(Some(TaskKind::WalReceiverManager), None, None).await; - - // Ok, no background tasks running anymore. Flush any remaining data in - // memory to disk. - // - // We assume that any incoming connections that might request pages from - // the tenant have already been terminated by the caller, so there - // should be no more activity in any of the repositories. - // - // On error, log it but continue with the shutdown for other tenants. - for tenant in tenants_to_freeze_and_flush { - let tenant_id = tenant.tenant_id(); - debug!("shutdown tenant {tenant_id}"); - - if let Err(err) = tenant.freeze_and_flush().await { - error!("Could not checkpoint tenant {tenant_id} during shutdown: {err:?}"); - } + if panicked > 0 { + warn!(panicked, "observed panicks while shutting down tenants"); } } @@ -302,7 +326,9 @@ pub async fn create_tenant( }); let created_tenant = - schedule_local_tenant_processing(conf, &tenant_directory, broker_client, remote_storage, ctx)?; + schedule_local_tenant_processing(conf, &tenant_directory, broker_client, remote_storage, None, ctx)?; + // TODO: tenant object & its background loops remain, untracked in tenant map, if we fail here. + // See https://github.com/neondatabase/neon/issues/4233 let created_tenant = scopeguard::guard(created_tenant, |tenant| { // As we might have removed the directory, the tenant should directly go into the broken state. tenant.set_broken("failed to create".into()); @@ -393,7 +419,9 @@ pub async fn delete_timeline( ctx: &RequestContext, ) -> Result<(), DeleteTimelineError> { let tenant = get_tenant(tenant_id, true).await?; - tenant.delete_timeline(timeline_id, ctx).await?; + tenant + .prepare_and_schedule_delete_timeline(timeline_id, ctx) + .await?; Ok(()) } @@ -458,7 +486,7 @@ pub async fn load_tenant( .with_context(|| format!("Failed to remove tenant ignore mark {tenant_ignore_mark:?} during tenant loading"))?; } - let new_tenant = schedule_local_tenant_processing(conf, &tenant_path, broker_client, remote_storage, ctx) + let new_tenant = schedule_local_tenant_processing(conf, &tenant_path, broker_client, remote_storage, None, ctx) .with_context(|| { format!("Failed to schedule tenant processing in path {tenant_path:?}") })?; @@ -540,8 +568,9 @@ pub async fn attach_tenant( "create_tenant_files should have created the attach marker file" ); - let attached_tenant = - schedule_local_tenant_processing(conf, &tenant_dir, broker_client, Some(remote_storage), ctx)?; + let attached_tenant = schedule_local_tenant_processing(conf, &tenant_dir, broker_client, Some(remote_storage), None, ctx)?; + // TODO: tenant object & its background loops remain, untracked in tenant map, if we fail here. + // See https://github.com/neondatabase/neon/issues/4233 let attached_tenant = scopeguard::guard(attached_tenant, |tenant| { // As we might have removed the directory, the tenant should directly go into the broken state. tenant.set_broken("failed to attach".into()); @@ -623,25 +652,26 @@ where // The exclusive lock here ensures we don't miss the tenant state updates before trying another removal. // tenant-wde cleanup operations may take some time (removing the entire tenant directory), we want to // avoid holding the lock for the entire process. - { - let tenants_accessor = TENANTS.write().await; - match tenants_accessor.get(&tenant_id) { - Some(tenant) => match tenant.current_state() { - TenantState::Attaching - | TenantState::Loading - | TenantState::Broken { .. } - | TenantState::Active => tenant.set_stopping(), - TenantState::Stopping => return Err(TenantStateError::IsStopping(tenant_id)), - }, - None => return Err(TenantStateError::NotFound(tenant_id)), + let tenant = { + TENANTS + .write() + .await + .get(&tenant_id) + .cloned() + .ok_or(TenantStateError::NotFound(tenant_id))? + }; + + let freeze_and_flush = false; + + // shutdown is sure to transition tenant to stopping, and wait for all tasks to complete, so + // that we can continue safely to cleanup. + match tenant.shutdown(freeze_and_flush).await { + Ok(()) => {} + Err(super::ShutdownError::AlreadyStopping) => { + return Err(TenantStateError::IsStopping(tenant_id)) } } - // shutdown all tenant and timeline tasks: gc, compaction, page service) - // No new tasks will be started for this tenant because it's in `Stopping` state. - // Hence, once we're done here, the `tenant_cleanup` callback can mutate tenant on-disk state freely. - task_mgr::shutdown_tasks(None, Some(tenant_id), None).await; - match tenant_cleanup .await .with_context(|| format!("Failed to run cleanup for tenant {tenant_id}")) @@ -657,7 +687,7 @@ where let tenants_accessor = TENANTS.read().await; match tenants_accessor.get(&tenant_id) { Some(tenant) => { - tenant.set_broken(e.to_string()); + tenant.set_broken(e.to_string()).await; } None => { warn!("Tenant {tenant_id} got removed from memory"); @@ -685,7 +715,7 @@ pub async fn immediate_gc( .get(&tenant_id) .map(Arc::clone) .with_context(|| format!("tenant {tenant_id}")) - .map_err(ApiError::NotFound)?; + .map_err(|e| ApiError::NotFound(e.into()))?; let gc_horizon = gc_req.gc_horizon.unwrap_or_else(|| tenant.get_gc_horizon()); // Use tenant's pitr setting @@ -723,7 +753,6 @@ pub async fn immediate_gc( Ok(wait_task_done) } -#[cfg(feature = "testing")] pub async fn immediate_compact( tenant_id: TenantId, timeline_id: TimelineId, @@ -735,11 +764,11 @@ pub async fn immediate_compact( .get(&tenant_id) .map(Arc::clone) .with_context(|| format!("tenant {tenant_id}")) - .map_err(ApiError::NotFound)?; + .map_err(|e| ApiError::NotFound(e.into()))?; let timeline = tenant .get_timeline(timeline_id, true) - .map_err(ApiError::NotFound)?; + .map_err(|e| ApiError::NotFound(e.into()))?; // Run in task_mgr to avoid race with tenant_detach operation let ctx = ctx.detached_child(TaskKind::Compaction, DownloadBehavior::Download); diff --git a/pageserver/src/tenant/par_fsync.rs b/pageserver/src/tenant/par_fsync.rs index 0b0217ab58..3cbcfe8774 100644 --- a/pageserver/src/tenant/par_fsync.rs +++ b/pageserver/src/tenant/par_fsync.rs @@ -19,14 +19,8 @@ fn parallel_worker(paths: &[PathBuf], next_path_idx: &AtomicUsize) -> io::Result Ok(()) } -pub fn par_fsync(paths: &[PathBuf]) -> io::Result<()> { - const PARALLEL_PATH_THRESHOLD: usize = 1; - if paths.len() <= PARALLEL_PATH_THRESHOLD { - for path in paths { - fsync_path(path)?; - } - return Ok(()); - } +fn fsync_in_thread_pool(paths: &[PathBuf]) -> io::Result<()> { + // TODO: remove this function in favor of `par_fsync_async` once we asyncify everything. /// Use at most this number of threads. /// Increasing this limit will @@ -36,11 +30,11 @@ pub fn par_fsync(paths: &[PathBuf]) -> io::Result<()> { let num_threads = paths.len().min(MAX_NUM_THREADS); let next_path_idx = AtomicUsize::new(0); - crossbeam_utils::thread::scope(|s| -> io::Result<()> { + std::thread::scope(|s| -> io::Result<()> { let mut handles = vec![]; // Spawn `num_threads - 1`, as the current thread is also a worker. for _ in 1..num_threads { - handles.push(s.spawn(|_| parallel_worker(paths, &next_path_idx))); + handles.push(s.spawn(|| parallel_worker(paths, &next_path_idx))); } parallel_worker(paths, &next_path_idx)?; @@ -51,5 +45,41 @@ pub fn par_fsync(paths: &[PathBuf]) -> io::Result<()> { Ok(()) }) - .unwrap() +} + +/// Parallel fsync all files. Can be used in non-async context as it is using rayon thread pool. +pub fn par_fsync(paths: &[PathBuf]) -> io::Result<()> { + if paths.len() == 1 { + fsync_path(&paths[0])?; + return Ok(()); + } + + fsync_in_thread_pool(paths) +} + +/// Parallel fsync asynchronously. If number of files are less than PARALLEL_PATH_THRESHOLD, fsync is done in the current +/// execution thread. Otherwise, we will spawn_blocking and run it in tokio. +pub async fn par_fsync_async(paths: &[PathBuf]) -> io::Result<()> { + const MAX_CONCURRENT_FSYNC: usize = 64; + let mut next = paths.iter().peekable(); + let mut js = tokio::task::JoinSet::new(); + loop { + while js.len() < MAX_CONCURRENT_FSYNC && next.peek().is_some() { + let next = next.next().expect("just peeked"); + let next = next.to_owned(); + js.spawn_blocking(move || fsync_path(&next)); + } + + // now the joinset has been filled up, wait for next to complete + if let Some(res) = js.join_next().await { + res??; + } else { + // last item had already completed + assert!( + next.peek().is_none(), + "joinset emptied, we shouldn't have more work" + ); + return Ok(()); + } + } } diff --git a/pageserver/src/tenant/remote_timeline_client.rs b/pageserver/src/tenant/remote_timeline_client.rs index c4640307d0..190512f48f 100644 --- a/pageserver/src/tenant/remote_timeline_client.rs +++ b/pageserver/src/tenant/remote_timeline_client.rs @@ -210,13 +210,15 @@ use chrono::{NaiveDateTime, Utc}; pub use download::{is_temp_download_file, list_remote_timelines}; use scopeguard::ScopeGuard; +use std::collections::{HashMap, VecDeque}; +use std::path::Path; use std::sync::atomic::{AtomicU32, Ordering}; use std::sync::{Arc, Mutex}; -use remote_storage::{DownloadError, GenericRemoteStorage}; +use remote_storage::{DownloadError, GenericRemoteStorage, RemotePath}; use std::ops::DerefMut; use tokio::runtime::Runtime; -use tracing::{debug, error, info, warn}; +use tracing::{debug, error, info, instrument, warn}; use tracing::{info_span, Instrument}; use utils::lsn::Lsn; @@ -225,7 +227,9 @@ use crate::metrics::{ RemoteTimelineClientMetricsCallTrackSize, REMOTE_ONDEMAND_DOWNLOADED_BYTES, REMOTE_ONDEMAND_DOWNLOADED_LAYERS, }; +use crate::tenant::debug_assert_current_span_has_tenant_and_timeline_id; use crate::tenant::remote_timeline_client::index::LayerFileMetadata; +use crate::tenant::upload_queue::Delete; use crate::{ config::PageServerConf, task_mgr, @@ -259,7 +263,7 @@ const FAILED_UPLOAD_WARN_THRESHOLD: u32 = 3; pub enum MaybeDeletedIndexPart { IndexPart(IndexPart), - Deleted, + Deleted(IndexPart), } /// Errors that can arise when calling [`RemoteTimelineClient::stop`]. @@ -361,11 +365,42 @@ impl RemoteTimelineClient { Ok(()) } + /// Initialize the queue in stopped state. Used in startup path + /// to continue deletion operation interrupted by pageserver crash or restart. + pub fn init_upload_queue_stopped_to_continue_deletion( + &self, + index_part: &IndexPart, + ) -> anyhow::Result<()> { + // FIXME: consider newtype for DeletedIndexPart. + let deleted_at = index_part.deleted_at.ok_or(anyhow::anyhow!( + "bug: it is responsibility of the caller to provide index part from MaybeDeletedIndexPart::Deleted" + ))?; + + { + let mut upload_queue = self.upload_queue.lock().unwrap(); + upload_queue.initialize_with_current_remote_index_part(index_part)?; + self.update_remote_physical_size_gauge(Some(index_part)); + } + // also locks upload queue, without dropping the guard above it will be a deadlock + self.stop().expect("initialized line above"); + + let mut upload_queue = self.upload_queue.lock().unwrap(); + + upload_queue + .stopped_mut() + .expect("stopped above") + .deleted_at = SetDeletedFlagProgress::Successful(deleted_at); + + Ok(()) + } + pub fn last_uploaded_consistent_lsn(&self) -> Option { match &*self.upload_queue.lock().unwrap() { UploadQueue::Uninitialized => None, UploadQueue::Initialized(q) => Some(q.last_uploaded_consistent_lsn), - UploadQueue::Stopped(q) => Some(q.last_uploaded_consistent_lsn), + UploadQueue::Stopped(q) => { + Some(q.upload_queue_for_deletion.last_uploaded_consistent_lsn) + } } } @@ -420,7 +455,7 @@ impl RemoteTimelineClient { .await?; if index_part.deleted_at.is_some() { - Ok(MaybeDeletedIndexPart::Deleted) + Ok(MaybeDeletedIndexPart::Deleted(index_part)) } else { Ok(MaybeDeletedIndexPart::IndexPart(index_part)) } @@ -622,7 +657,11 @@ impl RemoteTimelineClient { // schedule the actual deletions for name in names { - let op = UploadOp::Delete(RemoteOpFileKind::Layer, name.clone()); + let op = UploadOp::Delete(Delete { + file_kind: RemoteOpFileKind::Layer, + layer_file_name: name.clone(), + scheduled_from_timeline_delete: false, + }); self.calls_unfinished_metric_begin(&op); upload_queue.queued_operations.push_back(op); info!("scheduled layer file deletion {}", name.file_name()); @@ -639,18 +678,11 @@ impl RemoteTimelineClient { /// Wait for all previously scheduled uploads/deletions to complete /// pub async fn wait_completion(self: &Arc) -> anyhow::Result<()> { - let (sender, mut receiver) = tokio::sync::watch::channel(()); - let barrier_op = UploadOp::Barrier(sender); - - { + let mut receiver = { let mut guard = self.upload_queue.lock().unwrap(); let upload_queue = guard.initialized_mut()?; - upload_queue.queued_operations.push_back(barrier_op); - // Don't count this kind of operation! - - // Launch the task immediately, if possible - self.launch_queued_tasks(upload_queue); - } + self.schedule_barrier(upload_queue) + }; if receiver.changed().await.is_err() { anyhow::bail!("wait_completion aborted because upload queue was stopped"); @@ -658,6 +690,22 @@ impl RemoteTimelineClient { Ok(()) } + fn schedule_barrier( + self: &Arc, + upload_queue: &mut UploadQueueInitialized, + ) -> tokio::sync::watch::Receiver<()> { + let (sender, receiver) = tokio::sync::watch::channel(()); + let barrier_op = UploadOp::Barrier(sender); + + upload_queue.queued_operations.push_back(barrier_op); + // Don't count this kind of operation! + + // Launch the task immediately, if possible + self.launch_queued_tasks(upload_queue); + + receiver + } + /// Set the deleted_at field in the remote index file. /// /// This fails if the upload queue has not been `stop()`ed. @@ -665,6 +713,7 @@ impl RemoteTimelineClient { /// The caller is responsible for calling `stop()` AND for waiting /// for any ongoing upload tasks to finish after `stop()` has succeeded. /// Check method [`RemoteTimelineClient::stop`] for details. + #[instrument(skip_all)] pub(crate) async fn persist_index_part_with_deleted_flag( self: &Arc, ) -> Result<(), PersistIndexPartWithDeletedFlagError> { @@ -674,15 +723,7 @@ impl RemoteTimelineClient { // We must be in stopped state because otherwise // we can have inprogress index part upload that can overwrite the file // with missing is_deleted flag that we going to set below - let stopped = match &mut *locked { - UploadQueue::Uninitialized => { - return Err(anyhow::anyhow!("is not Stopped but Uninitialized").into()) - } - UploadQueue::Initialized(_) => { - return Err(anyhow::anyhow!("is not Stopped but Initialized").into()) - } - UploadQueue::Stopped(stopped) => stopped, - }; + let stopped = locked.stopped_mut()?; match stopped.deleted_at { SetDeletedFlagProgress::NotRunning => (), // proceed @@ -696,48 +737,34 @@ impl RemoteTimelineClient { let deleted_at = Utc::now().naive_utc(); stopped.deleted_at = SetDeletedFlagProgress::InProgress(deleted_at); - let mut index_part = IndexPart::new( - stopped.latest_files.clone(), - stopped.last_uploaded_consistent_lsn, - stopped - .latest_metadata - .to_bytes() - .context("serialize metadata")?, - ); + let mut index_part = IndexPart::try_from(&stopped.upload_queue_for_deletion) + .context("IndexPart serialize")?; index_part.deleted_at = Some(deleted_at); index_part }; let undo_deleted_at = scopeguard::guard(Arc::clone(self), |self_clone| { let mut locked = self_clone.upload_queue.lock().unwrap(); - let stopped = match &mut *locked { - UploadQueue::Uninitialized | UploadQueue::Initialized(_) => unreachable!( - "there's no way out of Stopping, and we checked it's Stopping above: {:?}", - locked.as_str(), - ), - UploadQueue::Stopped(stopped) => stopped, - }; + let stopped = locked + .stopped_mut() + .expect("there's no way out of Stopping, and we checked it's Stopping above"); stopped.deleted_at = SetDeletedFlagProgress::NotRunning; }); // Have a failpoint that can use the `pause` failpoint action. // We don't want to block the executor thread, hence, spawn_blocking + await. - #[cfg(feature = "testing")] - tokio::task::spawn_blocking({ - let current = tracing::Span::current(); - move || { - let _entered = current.entered(); - tracing::info!( - "at failpoint persist_index_part_with_deleted_flag_after_set_before_upload_pause" - ); - fail::fail_point!( - "persist_index_part_with_deleted_flag_after_set_before_upload_pause" - ); - } - }) - .await - .expect("spawn_blocking"); - + if cfg!(feature = "testing") { + tokio::task::spawn_blocking({ + let current = tracing::Span::current(); + move || { + let _entered = current.entered(); + tracing::info!("at failpoint persist_deleted_index_part"); + fail::fail_point!("persist_deleted_index_part"); + } + }) + .await + .expect("spawn_blocking"); + } upload::upload_index_part( self.conf, &self.storage_impl, @@ -751,13 +778,10 @@ impl RemoteTimelineClient { ScopeGuard::into_inner(undo_deleted_at); { let mut locked = self.upload_queue.lock().unwrap(); - let stopped = match &mut *locked { - UploadQueue::Uninitialized | UploadQueue::Initialized(_) => unreachable!( - "there's no way out of Stopping, and we checked it's Stopping above: {:?}", - locked.as_str(), - ), - UploadQueue::Stopped(stopped) => stopped, - }; + + let stopped = locked + .stopped_mut() + .expect("there's no way out of Stopping, and we checked it's Stopping above"); stopped.deleted_at = SetDeletedFlagProgress::Successful( index_part_with_deleted_at .deleted_at @@ -768,6 +792,90 @@ impl RemoteTimelineClient { Ok(()) } + /// Prerequisites: UploadQueue should be in stopped state and deleted_at should be successfuly set. + /// The function deletes layer files one by one, then lists the prefix to see if we leaked something + /// deletes leaked files if any and proceeds with deletion of index file at the end. + pub(crate) async fn delete_all(self: &Arc) -> anyhow::Result<()> { + debug_assert_current_span_has_tenant_and_timeline_id(); + + let (mut receiver, deletions_queued) = { + let mut deletions_queued = 0; + + let mut locked = self.upload_queue.lock().unwrap(); + let stopped = locked.stopped_mut()?; + + if !matches!(stopped.deleted_at, SetDeletedFlagProgress::Successful(_)) { + anyhow::bail!("deleted_at is not set") + } + + debug_assert!(stopped.upload_queue_for_deletion.no_pending_work()); + + stopped + .upload_queue_for_deletion + .queued_operations + .reserve(stopped.upload_queue_for_deletion.latest_files.len()); + + // schedule the actual deletions + for name in stopped.upload_queue_for_deletion.latest_files.keys() { + let op = UploadOp::Delete(Delete { + file_kind: RemoteOpFileKind::Layer, + layer_file_name: name.clone(), + scheduled_from_timeline_delete: true, + }); + self.calls_unfinished_metric_begin(&op); + stopped + .upload_queue_for_deletion + .queued_operations + .push_back(op); + + info!("scheduled layer file deletion {}", name.file_name()); + deletions_queued += 1; + } + + self.launch_queued_tasks(&mut stopped.upload_queue_for_deletion); + + ( + self.schedule_barrier(&mut stopped.upload_queue_for_deletion), + deletions_queued, + ) + }; + + receiver.changed().await?; + + // Do not delete index part yet, it is needed for possible retry. If we remove it first + // and retry will arrive to different pageserver there wont be any traces of it on remote storage + let timeline_path = self.conf.timeline_path(&self.timeline_id, &self.tenant_id); + let timeline_storage_path = self.conf.remote_path(&timeline_path)?; + + let remaining = self + .storage_impl + .list_prefixes(Some(&timeline_storage_path)) + .await?; + + let remaining: Vec = remaining + .into_iter() + .filter(|p| p.object_name() != Some(IndexPart::FILE_NAME)) + .collect(); + + if !remaining.is_empty() { + warn!( + "Found {} files not bound to index_file.json, proceeding with their deletion", + remaining.len() + ); + warn!("About to remove {} files", remaining.len()); + self.storage_impl.delete_objects(&remaining).await?; + } + + let index_file_path = timeline_storage_path.join(Path::new(IndexPart::FILE_NAME)); + + debug!("deleting index part"); + self.storage_impl.delete(&index_file_path).await?; + + info!(deletions_queued, "done deleting, including index_part.json"); + + Ok(()) + } + /// /// Pick next tasks from the queue, and start as many of them as possible without violating /// the ordering constraints. @@ -786,7 +894,7 @@ impl RemoteTimelineClient { // have finished. upload_queue.inprogress_tasks.is_empty() } - UploadOp::Delete(_, _) => { + UploadOp::Delete(_) => { // Wait for preceding uploads to finish. Concurrent deletions are OK, though. upload_queue.num_inprogress_deletions == upload_queue.inprogress_tasks.len() } @@ -817,7 +925,7 @@ impl RemoteTimelineClient { UploadOp::UploadMetadata(_, _) => { upload_queue.num_inprogress_metadata_uploads += 1; } - UploadOp::Delete(_, _) => { + UploadOp::Delete(_) => { upload_queue.num_inprogress_deletions += 1; } UploadOp::Barrier(sender) => { @@ -891,7 +999,6 @@ impl RemoteTimelineClient { unreachable!("we never launch an upload task if the queue is uninitialized, and once it is initialized, we never go back") } } - self.calls_unfinished_metric_end(&task.op); return; } @@ -937,16 +1044,16 @@ impl RemoteTimelineClient { } res } - UploadOp::Delete(metric_file_kind, ref layer_file_name) => { + UploadOp::Delete(delete) => { let path = &self .conf .timeline_path(&self.timeline_id, &self.tenant_id) - .join(layer_file_name.file_name()); + .join(delete.layer_file_name.file_name()); delete::delete_layer(self.conf, &self.storage_impl, path) .measure_remote_op( self.tenant_id, self.timeline_id, - *metric_file_kind, + delete.file_kind, RemoteOpKind::Delete, Arc::clone(&self.metrics), ) @@ -1012,11 +1119,24 @@ impl RemoteTimelineClient { let mut upload_queue_guard = self.upload_queue.lock().unwrap(); let upload_queue = match upload_queue_guard.deref_mut() { UploadQueue::Uninitialized => panic!("callers are responsible for ensuring this is only called on an initialized queue"), - UploadQueue::Stopped(_) => { + UploadQueue::Stopped(stopped) => { + // Special care is needed for deletions, if it was an earlier deletion (not scheduled from deletion) + // then stop() took care of it so we just return. + // For deletions that come from delete_all we still want to maintain metrics, launch following tasks, etc. + match &task.op { + UploadOp::Delete(delete) if delete.scheduled_from_timeline_delete => Some(&mut stopped.upload_queue_for_deletion), + _ => None + } + }, + UploadQueue::Initialized(qi) => { Some(qi) } + }; + + let upload_queue = match upload_queue { + Some(upload_queue) => upload_queue, + None => { info!("another concurrent task already stopped the queue"); return; - }, // nothing to do - UploadQueue::Initialized(qi) => { qi } + } }; upload_queue.inprogress_tasks.remove(&task.task_id); @@ -1029,7 +1149,7 @@ impl RemoteTimelineClient { upload_queue.num_inprogress_metadata_uploads -= 1; upload_queue.last_uploaded_consistent_lsn = lsn; // XXX monotonicity check? } - UploadOp::Delete(_, _) => { + UploadOp::Delete(_) => { upload_queue.num_inprogress_deletions -= 1; } UploadOp::Barrier(_) => unreachable!(), @@ -1063,8 +1183,8 @@ impl RemoteTimelineClient { reason: "metadata uploads are tiny", }, ), - UploadOp::Delete(file_kind, _) => ( - *file_kind, + UploadOp::Delete(delete) => ( + delete.file_kind, RemoteOpKind::Delete, DontTrackSize { reason: "should we track deletes? positive or negative sign?", @@ -1111,32 +1231,36 @@ impl RemoteTimelineClient { info!("another concurrent task already shut down the queue"); Ok(()) } - UploadQueue::Initialized(UploadQueueInitialized { - latest_files, - latest_metadata, - last_uploaded_consistent_lsn, - .. - }) => { + UploadQueue::Initialized(initialized) => { info!("shutting down upload queue"); // Replace the queue with the Stopped state, taking ownership of the old // Initialized queue. We will do some checks on it, and then drop it. let qi = { - // take or clone what we need - let latest_files = std::mem::take(latest_files); - let last_uploaded_consistent_lsn = *last_uploaded_consistent_lsn; - // this could be Copy - let latest_metadata = latest_metadata.clone(); - - let stopped = UploadQueueStopped { - latest_files, - last_uploaded_consistent_lsn, - latest_metadata, - deleted_at: SetDeletedFlagProgress::NotRunning, + // Here we preserve working version of the upload queue for possible use during deletions. + // In-place replace of Initialized to Stopped can be done with the help of https://github.com/Sgeo/take_mut + // but for this use case it doesnt really makes sense to bring unsafe code only for this usage point. + // Deletion is not really perf sensitive so there shouldnt be any problems with cloning a fraction of it. + let upload_queue_for_deletion = UploadQueueInitialized { + task_counter: 0, + latest_files: initialized.latest_files.clone(), + latest_files_changes_since_metadata_upload_scheduled: 0, + latest_metadata: initialized.latest_metadata.clone(), + last_uploaded_consistent_lsn: initialized.last_uploaded_consistent_lsn, + num_inprogress_layer_uploads: 0, + num_inprogress_metadata_uploads: 0, + num_inprogress_deletions: 0, + inprogress_tasks: HashMap::default(), + queued_operations: VecDeque::default(), }; - let upload_queue = - std::mem::replace(&mut *guard, UploadQueue::Stopped(stopped)); + let upload_queue = std::mem::replace( + &mut *guard, + UploadQueue::Stopped(UploadQueueStopped { + upload_queue_for_deletion, + deleted_at: SetDeletedFlagProgress::NotRunning, + }), + ); if let UploadQueue::Initialized(qi) = upload_queue { qi } else { @@ -1144,8 +1268,6 @@ impl RemoteTimelineClient { } }; - assert!(qi.latest_files.is_empty(), "do not use this anymore"); - // consistency check assert_eq!( qi.num_inprogress_layer_uploads @@ -1243,7 +1365,7 @@ mod tests { struct TestSetup { runtime: &'static tokio::runtime::Runtime, entered_runtime: EnterGuard<'static>, - harness: TenantHarness<'static>, + harness: TenantHarness, tenant: Arc, tenant_ctx: RequestContext, remote_fs_dir: PathBuf, @@ -1264,7 +1386,12 @@ mod tests { let harness = TenantHarness::create(test_name)?; let (tenant, ctx) = runtime.block_on(harness.load()); // create an empty timeline directory - let _ = tenant.create_test_timeline(TIMELINE_ID, Lsn(0), DEFAULT_PG_VERSION, &ctx)?; + let _ = runtime.block_on(tenant.create_test_timeline( + TIMELINE_ID, + Lsn(8), + DEFAULT_PG_VERSION, + &ctx, + ))?; let remote_fs_dir = harness.conf.workdir.join("remote_fs"); std::fs::create_dir_all(remote_fs_dir)?; @@ -1408,7 +1535,7 @@ mod tests { // Download back the index.json, and check that the list of files is correct let index_part = match runtime.block_on(client.download_index_file())? { MaybeDeletedIndexPart::IndexPart(index_part) => index_part, - MaybeDeletedIndexPart::Deleted => panic!("unexpectedly got deleted index part"), + MaybeDeletedIndexPart::Deleted(_) => panic!("unexpectedly got deleted index part"), }; assert_file_list( diff --git a/pageserver/src/tenant/remote_timeline_client/index.rs b/pageserver/src/tenant/remote_timeline_client/index.rs index 7a06e57a6b..c3f6dcadec 100644 --- a/pageserver/src/tenant/remote_timeline_client/index.rs +++ b/pageserver/src/tenant/remote_timeline_client/index.rs @@ -7,9 +7,11 @@ use std::collections::{HashMap, HashSet}; use chrono::NaiveDateTime; use serde::{Deserialize, Serialize}; use serde_with::{serde_as, DisplayFromStr}; +use utils::bin_ser::SerializeError; use crate::tenant::metadata::TimelineMetadata; use crate::tenant::storage_layer::LayerFileName; +use crate::tenant::upload_queue::UploadQueueInitialized; use utils::lsn::Lsn; @@ -115,6 +117,21 @@ impl IndexPart { } } +impl TryFrom<&UploadQueueInitialized> for IndexPart { + type Error = SerializeError; + + fn try_from(upload_queue: &UploadQueueInitialized) -> Result { + let disk_consistent_lsn = upload_queue.latest_metadata.disk_consistent_lsn(); + let metadata_bytes = upload_queue.latest_metadata.to_bytes()?; + + Ok(Self::new( + upload_queue.latest_files.clone(), + disk_consistent_lsn, + metadata_bytes, + )) + } +} + /// Serialized form of [`LayerFileMetadata`]. #[derive(Debug, PartialEq, Eq, Clone, Serialize, Deserialize, Default)] pub struct IndexLayerMetadata { diff --git a/pageserver/src/tenant/storage_layer.rs b/pageserver/src/tenant/storage_layer.rs index 3ca8e28c16..7bc513b3a1 100644 --- a/pageserver/src/tenant/storage_layer.rs +++ b/pageserver/src/tenant/storage_layer.rs @@ -4,6 +4,7 @@ pub mod delta_layer; mod filename; mod image_layer; mod inmemory_layer; +mod layer_desc; mod remote_layer; use crate::config::PageServerConf; @@ -37,6 +38,7 @@ pub use delta_layer::{DeltaLayer, DeltaLayerWriter}; pub use filename::{DeltaFileName, ImageFileName, LayerFileName}; pub use image_layer::{ImageLayer, ImageLayerWriter}; pub use inmemory_layer::InMemoryLayer; +pub use layer_desc::{PersistentLayerDesc, PersistentLayerKey}; pub use remote_layer::RemoteLayer; use super::layer_map::BatchedUpdates; @@ -174,13 +176,10 @@ impl LayerAccessStats { /// Create an empty stats object and record a [`LayerLoad`] event with the given residence status. /// /// See [`record_residence_event`] for why you need to do this while holding the layer map lock. - pub(crate) fn for_loading_layer( - layer_map_lock_held_witness: &BatchedUpdates<'_, L>, + pub(crate) fn for_loading_layer( + layer_map_lock_held_witness: &BatchedUpdates<'_>, status: LayerResidenceStatus, - ) -> Self - where - L: ?Sized + Layer, - { + ) -> Self { let new = LayerAccessStats(Mutex::new(LayerAccessStatsLocked::default())); new.record_residence_event( layer_map_lock_held_witness, @@ -195,14 +194,11 @@ impl LayerAccessStats { /// The `new_status` is not recorded in `self`. /// /// See [`record_residence_event`] for why you need to do this while holding the layer map lock. - pub(crate) fn clone_for_residence_change( + pub(crate) fn clone_for_residence_change( &self, - layer_map_lock_held_witness: &BatchedUpdates<'_, L>, + layer_map_lock_held_witness: &BatchedUpdates<'_>, new_status: LayerResidenceStatus, - ) -> LayerAccessStats - where - L: ?Sized + Layer, - { + ) -> LayerAccessStats { let clone = { let inner = self.0.lock().unwrap(); inner.clone() @@ -230,14 +226,12 @@ impl LayerAccessStats { /// - Compact: Grab layer map lock, add the new L1 to layer map and remove the L0s, release layer map lock. /// - Eviction: observes the new L1 layer whose only activity timestamp is the LayerCreate event. /// - pub(crate) fn record_residence_event( + pub(crate) fn record_residence_event( &self, - _layer_map_lock_held_witness: &BatchedUpdates<'_, L>, + _layer_map_lock_held_witness: &BatchedUpdates<'_>, status: LayerResidenceStatus, reason: LayerResidenceEventReason, - ) where - L: ?Sized + Layer, - { + ) { let mut locked = self.0.lock().unwrap(); locked.iter_mut().for_each(|inner| { inner @@ -387,10 +381,10 @@ pub trait Layer: std::fmt::Debug + Send + Sync { } /// Returned by [`Layer::iter`] -pub type LayerIter<'i> = Box> + 'i>; +pub type LayerIter<'i> = Box> + 'i + Send>; /// Returned by [`Layer::key_iter`] -pub type LayerKeyIter<'i> = Box + 'i>; +pub type LayerKeyIter<'i> = Box + 'i + Send>; /// A Layer contains all data in a "rectangle" consisting of a range of keys and /// range of LSNs. @@ -406,14 +400,23 @@ pub type LayerKeyIter<'i> = Box + 'i>; /// An image layer is a snapshot of all the data in a key-range, at a single /// LSN. pub trait PersistentLayer: Layer { - fn get_tenant_id(&self) -> TenantId; + /// Get the layer descriptor. + fn layer_desc(&self) -> &PersistentLayerDesc; + + fn get_tenant_id(&self) -> TenantId { + self.layer_desc().tenant_id + } /// Identify the timeline this layer belongs to - fn get_timeline_id(&self) -> TimelineId; + fn get_timeline_id(&self) -> TimelineId { + self.layer_desc().timeline_id + } /// File name used for this layer, both in the pageserver's local filesystem /// state as well as in the remote storage. - fn filename(&self) -> LayerFileName; + fn filename(&self) -> LayerFileName { + self.layer_desc().filename() + } // Path to the layer file in the local filesystem. // `None` for `RemoteLayer`. @@ -443,7 +446,9 @@ pub trait PersistentLayer: Layer { /// /// Should not change over the lifetime of the layer object because /// current_physical_size is computed as the som of this value. - fn file_size(&self) -> u64; + fn file_size(&self) -> u64 { + self.layer_desc().file_size + } fn info(&self, reset: LayerAccessStatsReset) -> HistoricLayerInfo; @@ -460,80 +465,125 @@ pub fn downcast_remote_layer( } } -/// Holds metadata about a layer without any content. Used mostly for testing. -/// -/// To use filenames as fixtures, parse them as [`LayerFileName`] then convert from that to a -/// LayerDescriptor. -#[derive(Clone, Debug)] -pub struct LayerDescriptor { - pub key: Range, - pub lsn: Range, - pub is_incremental: bool, - pub short_id: String, -} +pub mod tests { + use super::*; -impl Layer for LayerDescriptor { - fn get_key_range(&self) -> Range { - self.key.clone() + /// Holds metadata about a layer without any content. Used mostly for testing. + /// + /// To use filenames as fixtures, parse them as [`LayerFileName`] then convert from that to a + /// LayerDescriptor. + #[derive(Clone, Debug)] + pub struct LayerDescriptor { + base: PersistentLayerDesc, } - fn get_lsn_range(&self) -> Range { - self.lsn.clone() - } - - fn is_incremental(&self) -> bool { - self.is_incremental - } - - fn get_value_reconstruct_data( - &self, - _key: Key, - _lsn_range: Range, - _reconstruct_data: &mut ValueReconstructState, - _ctx: &RequestContext, - ) -> Result { - todo!("This method shouldn't be part of the Layer trait") - } - - fn short_id(&self) -> String { - self.short_id.clone() - } - - fn dump(&self, _verbose: bool, _ctx: &RequestContext) -> Result<()> { - todo!() - } -} - -impl From for LayerDescriptor { - fn from(value: DeltaFileName) -> Self { - let short_id = value.to_string(); - LayerDescriptor { - key: value.key_range, - lsn: value.lsn_range, - is_incremental: true, - short_id, + impl From for LayerDescriptor { + fn from(base: PersistentLayerDesc) -> Self { + Self { base } } } -} -impl From for LayerDescriptor { - fn from(value: ImageFileName) -> Self { - let short_id = value.to_string(); - let lsn = value.lsn_as_range(); - LayerDescriptor { - key: value.key_range, - lsn, - is_incremental: false, - short_id, + impl Layer for LayerDescriptor { + fn get_value_reconstruct_data( + &self, + _key: Key, + _lsn_range: Range, + _reconstruct_data: &mut ValueReconstructState, + _ctx: &RequestContext, + ) -> Result { + todo!("This method shouldn't be part of the Layer trait") + } + + fn dump(&self, _verbose: bool, _ctx: &RequestContext) -> Result<()> { + todo!() + } + + /// Boilerplate to implement the Layer trait, always use layer_desc for persistent layers. + fn get_key_range(&self) -> Range { + self.layer_desc().key_range.clone() + } + + /// Boilerplate to implement the Layer trait, always use layer_desc for persistent layers. + fn get_lsn_range(&self) -> Range { + self.layer_desc().lsn_range.clone() + } + + /// Boilerplate to implement the Layer trait, always use layer_desc for persistent layers. + fn is_incremental(&self) -> bool { + self.layer_desc().is_incremental + } + + /// Boilerplate to implement the Layer trait, always use layer_desc for persistent layers. + fn short_id(&self) -> String { + self.layer_desc().short_id() } } -} -impl From for LayerDescriptor { - fn from(value: LayerFileName) -> Self { - match value { - LayerFileName::Delta(d) => Self::from(d), - LayerFileName::Image(i) => Self::from(i), + impl PersistentLayer for LayerDescriptor { + fn layer_desc(&self) -> &PersistentLayerDesc { + &self.base + } + + fn local_path(&self) -> Option { + unimplemented!() + } + + fn iter(&self, _: &RequestContext) -> Result> { + unimplemented!() + } + + fn key_iter(&self, _: &RequestContext) -> Result> { + unimplemented!() + } + + fn delete_resident_layer_file(&self) -> Result<()> { + unimplemented!() + } + + fn info(&self, _: LayerAccessStatsReset) -> HistoricLayerInfo { + unimplemented!() + } + + fn access_stats(&self) -> &LayerAccessStats { + unimplemented!() + } + } + + impl From for LayerDescriptor { + fn from(value: DeltaFileName) -> Self { + LayerDescriptor { + base: PersistentLayerDesc::new_delta( + TenantId::from_array([0; 16]), + TimelineId::from_array([0; 16]), + value.key_range, + value.lsn_range, + 233, + ), + } + } + } + + impl From for LayerDescriptor { + fn from(value: ImageFileName) -> Self { + LayerDescriptor { + base: PersistentLayerDesc::new_img( + TenantId::from_array([0; 16]), + TimelineId::from_array([0; 16]), + value.key_range, + value.lsn, + false, + 233, + ), + } + } + } + + impl From for LayerDescriptor { + fn from(value: LayerFileName) -> Self { + match value { + LayerFileName::Delta(d) => Self::from(d), + LayerFileName::Image(i) => Self::from(i), + } } } } diff --git a/pageserver/src/tenant/storage_layer/delta_layer.rs b/pageserver/src/tenant/storage_layer/delta_layer.rs index 63b8e57bb0..6e14663121 100644 --- a/pageserver/src/tenant/storage_layer/delta_layer.rs +++ b/pageserver/src/tenant/storage_layer/delta_layer.rs @@ -37,6 +37,7 @@ use crate::virtual_file::VirtualFile; use crate::{walrecord, TEMP_FILE_SUFFIX}; use crate::{DELTA_FILE_MAGIC, STORAGE_FORMAT_VERSION}; use anyhow::{bail, ensure, Context, Result}; +use once_cell::sync::OnceCell; use pageserver_api::models::{HistoricLayerInfo, LayerAccessKind}; use rand::{distributions::Alphanumeric, Rng}; use serde::{Deserialize, Serialize}; @@ -46,7 +47,6 @@ use std::io::{Seek, SeekFrom}; use std::ops::Range; use std::os::unix::fs::FileExt; use std::path::{Path, PathBuf}; -use std::sync::{RwLock, RwLockReadGuard, RwLockWriteGuard}; use tracing::*; use utils::{ @@ -56,8 +56,8 @@ use utils::{ }; use super::{ - DeltaFileName, Layer, LayerAccessStats, LayerAccessStatsReset, LayerFileName, LayerIter, - LayerKeyIter, PathOrConf, + DeltaFileName, Layer, LayerAccessStats, LayerAccessStatsReset, LayerIter, LayerKeyIter, + PathOrConf, PersistentLayerDesc, }; /// @@ -89,10 +89,10 @@ impl From<&DeltaLayer> for Summary { magic: DELTA_FILE_MAGIC, format_version: STORAGE_FORMAT_VERSION, - tenant_id: layer.tenant_id, - timeline_id: layer.timeline_id, - key_range: layer.key_range.clone(), - lsn_range: layer.lsn_range.clone(), + tenant_id: layer.desc.tenant_id, + timeline_id: layer.desc.timeline_id, + key_range: layer.desc.key_range.clone(), + lsn_range: layer.desc.lsn_range.clone(), index_start_blk: 0, index_root_blk: 0, @@ -180,16 +180,11 @@ impl DeltaKey { pub struct DeltaLayer { path_or_conf: PathOrConf, - pub tenant_id: TenantId, - pub timeline_id: TimelineId, - pub key_range: Range, - pub lsn_range: Range, - - pub file_size: u64, + pub desc: PersistentLayerDesc, access_stats: LayerAccessStats, - inner: RwLock, + inner: OnceCell, } impl std::fmt::Debug for DeltaLayer { @@ -197,30 +192,26 @@ impl std::fmt::Debug for DeltaLayer { use super::RangeDisplayDebug; f.debug_struct("DeltaLayer") - .field("key_range", &RangeDisplayDebug(&self.key_range)) - .field("lsn_range", &self.lsn_range) - .field("file_size", &self.file_size) + .field("key_range", &RangeDisplayDebug(&self.desc.key_range)) + .field("lsn_range", &self.desc.lsn_range) + .field("file_size", &self.desc.file_size) .field("inner", &self.inner) .finish() } } pub struct DeltaLayerInner { - /// If false, the fields below have not been loaded into memory yet. - loaded: bool, - // values copied from summary index_start_blk: u32, index_root_blk: u32, - /// Reader object for reading blocks from the file. (None if not loaded yet) - file: Option>, + /// Reader object for reading blocks from the file. + file: FileBlockReader, } impl std::fmt::Debug for DeltaLayerInner { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.debug_struct("DeltaLayerInner") - .field("loaded", &self.loaded) .field("index_start_blk", &self.index_start_blk) .field("index_root_blk", &self.index_root_blk) .finish() @@ -228,30 +219,16 @@ impl std::fmt::Debug for DeltaLayerInner { } impl Layer for DeltaLayer { - fn get_key_range(&self) -> Range { - self.key_range.clone() - } - - fn get_lsn_range(&self) -> Range { - self.lsn_range.clone() - } - fn is_incremental(&self) -> bool { - true - } - - fn short_id(&self) -> String { - self.filename().file_name() - } /// debugging function to print out the contents of the layer fn dump(&self, verbose: bool, ctx: &RequestContext) -> Result<()> { println!( "----- delta layer for ten {} tli {} keys {}-{} lsn {}-{} ----", - self.tenant_id, - self.timeline_id, - self.key_range.start, - self.key_range.end, - self.lsn_range.start, - self.lsn_range.end + self.desc.tenant_id, + self.desc.timeline_id, + self.desc.key_range.start, + self.desc.key_range.end, + self.desc.lsn_range.start, + self.desc.lsn_range.end ); if !verbose { @@ -265,7 +242,7 @@ impl Layer for DeltaLayer { inner.index_start_blk, inner.index_root_blk ); - let file = inner.file.as_ref().unwrap(); + let file = &inner.file; let tree_reader = DiskBtreeReader::<_, DELTA_KEY_SIZE>::new( inner.index_start_blk, inner.index_root_blk, @@ -324,17 +301,17 @@ impl Layer for DeltaLayer { reconstruct_state: &mut ValueReconstructState, ctx: &RequestContext, ) -> anyhow::Result { - ensure!(lsn_range.start >= self.lsn_range.start); + ensure!(lsn_range.start >= self.desc.lsn_range.start); let mut need_image = true; - ensure!(self.key_range.contains(&key)); + ensure!(self.desc.key_range.contains(&key)); { // Open the file and lock the metadata in memory let inner = self.load(LayerAccessKind::GetValueReconstructData, ctx)?; // Scan the page versions backwards, starting from `lsn`. - let file = inner.file.as_ref().unwrap(); + let file = &inner.file; let tree_reader = DiskBtreeReader::<_, DELTA_KEY_SIZE>::new( inner.index_start_blk, inner.index_root_blk, @@ -402,19 +379,31 @@ impl Layer for DeltaLayer { Ok(ValueReconstructResult::Complete) } } + + /// Boilerplate to implement the Layer trait, always use layer_desc for persistent layers. + fn get_key_range(&self) -> Range { + self.layer_desc().key_range.clone() + } + + /// Boilerplate to implement the Layer trait, always use layer_desc for persistent layers. + fn get_lsn_range(&self) -> Range { + self.layer_desc().lsn_range.clone() + } + + /// Boilerplate to implement the Layer trait, always use layer_desc for persistent layers. + fn is_incremental(&self) -> bool { + self.layer_desc().is_incremental + } + + /// Boilerplate to implement the Layer trait, always use layer_desc for persistent layers. + fn short_id(&self) -> String { + self.layer_desc().short_id() + } } impl PersistentLayer for DeltaLayer { - fn get_tenant_id(&self) -> TenantId { - self.tenant_id - } - - fn get_timeline_id(&self) -> TimelineId { - self.timeline_id - } - - fn filename(&self) -> LayerFileName { - self.layer_name().into() + fn layer_desc(&self) -> &PersistentLayerDesc { + &self.desc } fn local_path(&self) -> Option { @@ -444,10 +433,6 @@ impl PersistentLayer for DeltaLayer { Ok(()) } - fn file_size(&self) -> u64 { - self.file_size - } - fn info(&self, reset: LayerAccessStatsReset) -> HistoricLayerInfo { let layer_file_name = self.filename().file_name(); let lsn_range = self.get_lsn_range(); @@ -456,7 +441,7 @@ impl PersistentLayer for DeltaLayer { HistoricLayerInfo::Delta { layer_file_name, - layer_file_size: self.file_size, + layer_file_size: self.desc.file_size, lsn_start: lsn_range.start, lsn_end: lsn_range.end, remote: false, @@ -511,51 +496,22 @@ impl DeltaLayer { /// Open the underlying file and read the metadata into memory, if it's /// not loaded already. /// - fn load( - &self, - access_kind: LayerAccessKind, - ctx: &RequestContext, - ) -> Result> { + fn load(&self, access_kind: LayerAccessKind, ctx: &RequestContext) -> Result<&DeltaLayerInner> { self.access_stats .record_access(access_kind, ctx.task_kind()); - loop { - // Quick exit if already loaded - let inner = self.inner.read().unwrap(); - if inner.loaded { - return Ok(inner); - } - - // Need to open the file and load the metadata. Upgrade our lock to - // a write lock. (Or rather, release and re-lock in write mode.) - drop(inner); - let inner = self.inner.write().unwrap(); - if !inner.loaded { - self.load_inner(inner).with_context(|| { - format!("Failed to load delta layer {}", self.path().display()) - })?; - } else { - // Another thread loaded it while we were not holding the lock. - } - - // We now have the file open and loaded. There's no function to do - // that in the std library RwLock, so we have to release and re-lock - // in read mode. (To be precise, the lock guard was moved in the - // above call to `load_inner`, so it's already been released). And - // while we do that, another thread could unload again, so we have - // to re-check and retry if that happens. - } + // Quick exit if already loaded + self.inner + .get_or_try_init(|| self.load_inner()) + .with_context(|| format!("Failed to load delta layer {}", self.path().display())) } - fn load_inner(&self, mut inner: RwLockWriteGuard) -> Result<()> { + fn load_inner(&self) -> Result { let path = self.path(); - // Open the file if it's not open already. - if inner.file.is_none() { - let file = VirtualFile::open(&path) - .with_context(|| format!("Failed to open file '{}'", path.display()))?; - inner.file = Some(FileBlockReader::new(file)); - } - let file = inner.file.as_mut().unwrap(); + let file = VirtualFile::open(&path) + .with_context(|| format!("Failed to open file '{}'", path.display()))?; + let file = FileBlockReader::new(file); + let summary_blk = file.read_blk(0)?; let actual_summary = Summary::des_prefix(summary_blk.as_ref())?; @@ -582,13 +538,13 @@ impl DeltaLayer { } } - inner.index_start_blk = actual_summary.index_start_blk; - inner.index_root_blk = actual_summary.index_root_blk; - debug!("loaded from {}", &path.display()); - inner.loaded = true; - Ok(()) + Ok(DeltaLayerInner { + file, + index_start_blk: actual_summary.index_start_blk, + index_root_blk: actual_summary.index_root_blk, + }) } /// Create a DeltaLayer struct representing an existing file on disk. @@ -602,18 +558,15 @@ impl DeltaLayer { ) -> DeltaLayer { DeltaLayer { path_or_conf: PathOrConf::Conf(conf), - timeline_id, - tenant_id, - key_range: filename.key_range.clone(), - lsn_range: filename.lsn_range.clone(), - file_size, + desc: PersistentLayerDesc::new_delta( + tenant_id, + timeline_id, + filename.key_range.clone(), + filename.lsn_range.clone(), + file_size, + ), access_stats, - inner: RwLock::new(DeltaLayerInner { - loaded: false, - file: None, - index_start_blk: 0, - index_root_blk: 0, - }), + inner: once_cell::sync::OnceCell::new(), } } @@ -632,34 +585,27 @@ impl DeltaLayer { Ok(DeltaLayer { path_or_conf: PathOrConf::Path(path.to_path_buf()), - timeline_id: summary.timeline_id, - tenant_id: summary.tenant_id, - key_range: summary.key_range, - lsn_range: summary.lsn_range, - file_size: metadata.len(), + desc: PersistentLayerDesc::new_delta( + summary.tenant_id, + summary.timeline_id, + summary.key_range, + summary.lsn_range, + metadata.len(), + ), access_stats: LayerAccessStats::empty_will_record_residence_event_later(), - inner: RwLock::new(DeltaLayerInner { - loaded: false, - file: None, - index_start_blk: 0, - index_root_blk: 0, - }), + inner: once_cell::sync::OnceCell::new(), }) } fn layer_name(&self) -> DeltaFileName { - DeltaFileName { - key_range: self.key_range.clone(), - lsn_range: self.lsn_range.clone(), - } + self.desc.delta_file_name() } - /// Path to the layer file in pageserver workdir. pub fn path(&self) -> PathBuf { Self::path_for( &self.path_or_conf, - self.timeline_id, - self.tenant_id, + self.desc.timeline_id, + self.desc.tenant_id, &self.layer_name(), ) } @@ -803,18 +749,15 @@ impl DeltaLayerWriterInner { // set inner.file here. The first read will have to re-open it. let layer = DeltaLayer { path_or_conf: PathOrConf::Conf(self.conf), - tenant_id: self.tenant_id, - timeline_id: self.timeline_id, - key_range: self.key_start..key_end, - lsn_range: self.lsn_range.clone(), - file_size: metadata.len(), + desc: PersistentLayerDesc::new_delta( + self.tenant_id, + self.timeline_id, + self.key_start..key_end, + self.lsn_range.clone(), + metadata.len(), + ), access_stats: LayerAccessStats::empty_will_record_residence_event_later(), - inner: RwLock::new(DeltaLayerInner { - loaded: false, - file: None, - index_start_blk, - index_root_blk, - }), + inner: once_cell::sync::OnceCell::new(), }; // fsync the file @@ -949,13 +892,13 @@ struct DeltaValueIter<'a> { reader: BlockCursor>, } -struct Adapter<'a>(RwLockReadGuard<'a, DeltaLayerInner>); +struct Adapter<'a>(&'a DeltaLayerInner); impl<'a> BlockReader for Adapter<'a> { type BlockLease = PageReadGuard<'static>; fn read_blk(&self, blknum: u32) -> Result { - self.0.file.as_ref().unwrap().read_blk(blknum) + self.0.file.read_blk(blknum) } } @@ -968,8 +911,8 @@ impl<'a> Iterator for DeltaValueIter<'a> { } impl<'a> DeltaValueIter<'a> { - fn new(inner: RwLockReadGuard<'a, DeltaLayerInner>) -> Result { - let file = inner.file.as_ref().unwrap(); + fn new(inner: &'a DeltaLayerInner) -> Result { + let file = &inner.file; let tree_reader = DiskBtreeReader::<_, DELTA_KEY_SIZE>::new( inner.index_start_blk, inner.index_root_blk, @@ -1042,8 +985,8 @@ impl Iterator for DeltaKeyIter { } impl<'a> DeltaKeyIter { - fn new(inner: RwLockReadGuard<'a, DeltaLayerInner>) -> Result { - let file = inner.file.as_ref().unwrap(); + fn new(inner: &'a DeltaLayerInner) -> Result { + let file = &inner.file; let tree_reader = DiskBtreeReader::<_, DELTA_KEY_SIZE>::new( inner.index_start_blk, inner.index_root_blk, @@ -1083,3 +1026,21 @@ impl<'a> DeltaKeyIter { Ok(iter) } } + +#[cfg(test)] +mod test { + use super::DeltaKeyIter; + use super::DeltaLayer; + use super::DeltaValueIter; + + // We will soon need the iters to be send in the compaction code. + // Cf https://github.com/neondatabase/neon/pull/4462#issuecomment-1587398883 + // Cf https://github.com/neondatabase/neon/issues/4471 + #[test] + fn is_send() { + fn assert_send() {} + assert_send::(); + assert_send::(); + assert_send::(); + } +} diff --git a/pageserver/src/tenant/storage_layer/filename.rs b/pageserver/src/tenant/storage_layer/filename.rs index e2112fc388..5dcd54689e 100644 --- a/pageserver/src/tenant/storage_layer/filename.rs +++ b/pageserver/src/tenant/storage_layer/filename.rs @@ -9,6 +9,8 @@ use std::str::FromStr; use utils::lsn::Lsn; +use super::PersistentLayerDesc; + // Note: Timeline::load_layer_map() relies on this sort order #[derive(PartialEq, Eq, Clone, Hash)] pub struct DeltaFileName { @@ -153,7 +155,7 @@ impl Ord for ImageFileName { impl ImageFileName { pub fn lsn_as_range(&self) -> Range { // Saves from having to copypaste this all over - self.lsn..(self.lsn + 1) + PersistentLayerDesc::image_layer_lsn_range(self.lsn) } } diff --git a/pageserver/src/tenant/storage_layer/image_layer.rs b/pageserver/src/tenant/storage_layer/image_layer.rs index a5dd16fae2..07a16a7de2 100644 --- a/pageserver/src/tenant/storage_layer/image_layer.rs +++ b/pageserver/src/tenant/storage_layer/image_layer.rs @@ -52,8 +52,8 @@ use utils::{ lsn::Lsn, }; -use super::filename::{ImageFileName, LayerFileName}; -use super::{Layer, LayerAccessStatsReset, LayerIter, PathOrConf}; +use super::filename::ImageFileName; +use super::{Layer, LayerAccessStatsReset, LayerIter, PathOrConf, PersistentLayerDesc}; /// /// Header stored in the beginning of the file @@ -84,9 +84,9 @@ impl From<&ImageLayer> for Summary { Self { magic: IMAGE_FILE_MAGIC, format_version: STORAGE_FORMAT_VERSION, - tenant_id: layer.tenant_id, - timeline_id: layer.timeline_id, - key_range: layer.key_range.clone(), + tenant_id: layer.desc.tenant_id, + timeline_id: layer.desc.timeline_id, + key_range: layer.desc.key_range.clone(), lsn: layer.lsn, index_start_blk: 0, @@ -104,12 +104,9 @@ impl From<&ImageLayer> for Summary { /// and it needs to be loaded before using it in queries. pub struct ImageLayer { path_or_conf: PathOrConf, - pub tenant_id: TenantId, - pub timeline_id: TimelineId, - pub key_range: Range, - pub file_size: u64, - // This entry contains an image of all pages as of this LSN + pub desc: PersistentLayerDesc, + // This entry contains an image of all pages as of this LSN, should be the same as desc.lsn pub lsn: Lsn, access_stats: LayerAccessStats, @@ -122,8 +119,8 @@ impl std::fmt::Debug for ImageLayer { use super::RangeDisplayDebug; f.debug_struct("ImageLayer") - .field("key_range", &RangeDisplayDebug(&self.key_range)) - .field("file_size", &self.file_size) + .field("key_range", &RangeDisplayDebug(&self.desc.key_range)) + .field("file_size", &self.desc.file_size) .field("lsn", &self.lsn) .field("inner", &self.inner) .finish() @@ -153,27 +150,15 @@ impl std::fmt::Debug for ImageLayerInner { } impl Layer for ImageLayer { - fn get_key_range(&self) -> Range { - self.key_range.clone() - } - - fn get_lsn_range(&self) -> Range { - // End-bound is exclusive - self.lsn..(self.lsn + 1) - } - fn is_incremental(&self) -> bool { - false - } - - fn short_id(&self) -> String { - self.filename().file_name() - } - /// debugging function to print out the contents of the layer fn dump(&self, verbose: bool, ctx: &RequestContext) -> Result<()> { println!( "----- image layer for ten {} tli {} key {}-{} at {} ----", - self.tenant_id, self.timeline_id, self.key_range.start, self.key_range.end, self.lsn + self.desc.tenant_id, + self.desc.timeline_id, + self.desc.key_range.start, + self.desc.key_range.end, + self.lsn ); if !verbose { @@ -203,7 +188,7 @@ impl Layer for ImageLayer { reconstruct_state: &mut ValueReconstructState, ctx: &RequestContext, ) -> anyhow::Result { - assert!(self.key_range.contains(&key)); + assert!(self.desc.key_range.contains(&key)); assert!(lsn_range.start >= self.lsn); assert!(lsn_range.end >= self.lsn); @@ -230,24 +215,37 @@ impl Layer for ImageLayer { Ok(ValueReconstructResult::Missing) } } + + /// Boilerplate to implement the Layer trait, always use layer_desc for persistent layers. + fn get_key_range(&self) -> Range { + self.layer_desc().key_range.clone() + } + + /// Boilerplate to implement the Layer trait, always use layer_desc for persistent layers. + fn get_lsn_range(&self) -> Range { + self.layer_desc().lsn_range.clone() + } + + /// Boilerplate to implement the Layer trait, always use layer_desc for persistent layers. + fn is_incremental(&self) -> bool { + self.layer_desc().is_incremental + } + + /// Boilerplate to implement the Layer trait, always use layer_desc for persistent layers. + fn short_id(&self) -> String { + self.layer_desc().short_id() + } } impl PersistentLayer for ImageLayer { - fn filename(&self) -> LayerFileName { - self.layer_name().into() + fn layer_desc(&self) -> &PersistentLayerDesc { + &self.desc } fn local_path(&self) -> Option { Some(self.path()) } - fn get_tenant_id(&self) -> TenantId { - self.tenant_id - } - - fn get_timeline_id(&self) -> TimelineId { - self.timeline_id - } fn iter(&self, _ctx: &RequestContext) -> Result> { unimplemented!(); } @@ -258,17 +256,13 @@ impl PersistentLayer for ImageLayer { Ok(()) } - fn file_size(&self) -> u64 { - self.file_size - } - fn info(&self, reset: LayerAccessStatsReset) -> HistoricLayerInfo { let layer_file_name = self.filename().file_name(); let lsn_range = self.get_lsn_range(); HistoricLayerInfo::Image { layer_file_name, - layer_file_size: self.file_size, + layer_file_size: self.desc.file_size, lsn_start: lsn_range.start, remote: false, access_stats: self.access_stats.as_api_model(reset), @@ -405,11 +399,15 @@ impl ImageLayer { ) -> ImageLayer { ImageLayer { path_or_conf: PathOrConf::Conf(conf), - timeline_id, - tenant_id, - key_range: filename.key_range.clone(), + desc: PersistentLayerDesc::new_img( + tenant_id, + timeline_id, + filename.key_range.clone(), + filename.lsn, + false, + file_size, + ), // Now we assume image layer ALWAYS covers the full range. This may change in the future. lsn: filename.lsn, - file_size, access_stats, inner: RwLock::new(ImageLayerInner { loaded: false, @@ -433,11 +431,15 @@ impl ImageLayer { .context("get file metadata to determine size")?; Ok(ImageLayer { path_or_conf: PathOrConf::Path(path.to_path_buf()), - timeline_id: summary.timeline_id, - tenant_id: summary.tenant_id, - key_range: summary.key_range, + desc: PersistentLayerDesc::new_img( + summary.tenant_id, + summary.timeline_id, + summary.key_range, + summary.lsn, + false, + metadata.len(), + ), // Now we assume image layer ALWAYS covers the full range. This may change in the future. lsn: summary.lsn, - file_size: metadata.len(), access_stats: LayerAccessStats::empty_will_record_residence_event_later(), inner: RwLock::new(ImageLayerInner { file: None, @@ -449,18 +451,15 @@ impl ImageLayer { } fn layer_name(&self) -> ImageFileName { - ImageFileName { - key_range: self.key_range.clone(), - lsn: self.lsn, - } + self.desc.image_file_name() } /// Path to the layer file in pageserver workdir. pub fn path(&self) -> PathBuf { Self::path_for( &self.path_or_conf, - self.timeline_id, - self.tenant_id, + self.desc.timeline_id, + self.desc.tenant_id, &self.layer_name(), ) } @@ -484,6 +483,7 @@ struct ImageLayerWriterInner { tenant_id: TenantId, key_range: Range, lsn: Lsn, + is_incremental: bool, blob_writer: WriteBlobWriter, tree: DiskBtreeBuilder, @@ -499,6 +499,7 @@ impl ImageLayerWriterInner { tenant_id: TenantId, key_range: &Range, lsn: Lsn, + is_incremental: bool, ) -> anyhow::Result { // Create the file initially with a temporary filename. // We'll atomically rename it to the final name when we're done. @@ -533,6 +534,7 @@ impl ImageLayerWriterInner { lsn, tree: tree_builder, blob_writer, + is_incremental, }; Ok(writer) @@ -588,16 +590,22 @@ impl ImageLayerWriterInner { .metadata() .context("get metadata to determine file size")?; + let desc = PersistentLayerDesc::new_img( + self.tenant_id, + self.timeline_id, + self.key_range.clone(), + self.lsn, + self.is_incremental, // for now, image layer ALWAYS covers the full range + metadata.len(), + ); + // Note: Because we open the file in write-only mode, we cannot // reuse the same VirtualFile for reading later. That's why we don't // set inner.file here. The first read will have to re-open it. let layer = ImageLayer { path_or_conf: PathOrConf::Conf(self.conf), - timeline_id: self.timeline_id, - tenant_id: self.tenant_id, - key_range: self.key_range.clone(), + desc, lsn: self.lsn, - file_size: metadata.len(), access_stats: LayerAccessStats::empty_will_record_residence_event_later(), inner: RwLock::new(ImageLayerInner { loaded: false, @@ -667,6 +675,7 @@ impl ImageLayerWriter { tenant_id: TenantId, key_range: &Range, lsn: Lsn, + is_incremental: bool, ) -> anyhow::Result { Ok(Self { inner: Some(ImageLayerWriterInner::new( @@ -675,6 +684,7 @@ impl ImageLayerWriter { tenant_id, key_range, lsn, + is_incremental, )?), }) } diff --git a/pageserver/src/tenant/storage_layer/inmemory_layer.rs b/pageserver/src/tenant/storage_layer/inmemory_layer.rs index c453683fea..78bcfdafc0 100644 --- a/pageserver/src/tenant/storage_layer/inmemory_layer.rs +++ b/pageserver/src/tenant/storage_layer/inmemory_layer.rs @@ -304,7 +304,7 @@ impl InMemoryLayer { Ok(()) } - pub fn put_tombstone(&self, _key_range: Range, _lsn: Lsn) -> Result<()> { + pub async fn put_tombstone(&self, _key_range: Range, _lsn: Lsn) -> Result<()> { // TODO: Currently, we just leak the storage for any deleted keys Ok(()) diff --git a/pageserver/src/tenant/storage_layer/layer_desc.rs b/pageserver/src/tenant/storage_layer/layer_desc.rs new file mode 100644 index 0000000000..5ed548909e --- /dev/null +++ b/pageserver/src/tenant/storage_layer/layer_desc.rs @@ -0,0 +1,191 @@ +use anyhow::Result; +use std::ops::Range; +use utils::{ + id::{TenantId, TimelineId}, + lsn::Lsn, +}; + +use crate::{context::RequestContext, repository::Key}; + +use super::{DeltaFileName, ImageFileName, LayerFileName}; + +use serde::{Deserialize, Serialize}; + +/// A unique identifier of a persistent layer. This is different from `LayerDescriptor`, which is only used in the +/// benchmarks. This struct contains all necessary information to find the image / delta layer. It also provides +/// a unified way to generate layer information like file name. +#[derive(Debug, PartialEq, Eq, Clone, Serialize, Deserialize)] +pub struct PersistentLayerDesc { + pub tenant_id: TenantId, + pub timeline_id: TimelineId, + pub key_range: Range, + /// For image layer, this is `[lsn, lsn+1)`. + pub lsn_range: Range, + /// Whether this is a delta layer. + pub is_delta: bool, + /// Whether this layer only contains page images for part of the keys in the range. In the current implementation, this should + /// always be equal to `is_delta`. If we land the partial image layer PR someday, image layer could also be + /// incremental. + pub is_incremental: bool, + /// File size + pub file_size: u64, +} + +/// A unique identifier of a persistent layer within the context of one timeline. +#[derive(Debug, PartialEq, Eq, Clone, Hash)] +pub struct PersistentLayerKey { + pub key_range: Range, + pub lsn_range: Range, + pub is_delta: bool, +} + +impl PersistentLayerDesc { + pub fn key(&self) -> PersistentLayerKey { + PersistentLayerKey { + key_range: self.key_range.clone(), + lsn_range: self.lsn_range.clone(), + is_delta: self.is_delta, + } + } + + pub fn short_id(&self) -> String { + self.filename().file_name() + } + + #[cfg(test)] + pub fn new_test(key_range: Range) -> Self { + Self { + tenant_id: TenantId::generate(), + timeline_id: TimelineId::generate(), + key_range, + lsn_range: Lsn(0)..Lsn(1), + is_delta: false, + is_incremental: false, + file_size: 0, + } + } + + pub fn new_img( + tenant_id: TenantId, + timeline_id: TimelineId, + key_range: Range, + lsn: Lsn, + is_incremental: bool, + file_size: u64, + ) -> Self { + Self { + tenant_id, + timeline_id, + key_range, + lsn_range: Self::image_layer_lsn_range(lsn), + is_delta: false, + is_incremental, + file_size, + } + } + + pub fn new_delta( + tenant_id: TenantId, + timeline_id: TimelineId, + key_range: Range, + lsn_range: Range, + file_size: u64, + ) -> Self { + Self { + tenant_id, + timeline_id, + key_range, + lsn_range, + is_delta: true, + is_incremental: true, + file_size, + } + } + + /// Get the LSN that the image layer covers. + pub fn image_layer_lsn(&self) -> Lsn { + assert!(!self.is_delta); + assert!(self.lsn_range.start + 1 == self.lsn_range.end); + self.lsn_range.start + } + + /// Get the LSN range corresponding to a single image layer LSN. + pub fn image_layer_lsn_range(lsn: Lsn) -> Range { + lsn..(lsn + 1) + } + + /// Get a delta file name for this layer. + /// + /// Panic: if this is not a delta layer. + pub fn delta_file_name(&self) -> DeltaFileName { + assert!(self.is_delta); + DeltaFileName { + key_range: self.key_range.clone(), + lsn_range: self.lsn_range.clone(), + } + } + + /// Get a delta file name for this layer. + /// + /// Panic: if this is not an image layer, or the lsn range is invalid + pub fn image_file_name(&self) -> ImageFileName { + assert!(!self.is_delta); + assert!(self.lsn_range.start + 1 == self.lsn_range.end); + ImageFileName { + key_range: self.key_range.clone(), + lsn: self.lsn_range.start, + } + } + + pub fn filename(&self) -> LayerFileName { + if self.is_delta { + self.delta_file_name().into() + } else { + self.image_file_name().into() + } + } + + // TODO: remove this in the future once we refactor timeline APIs. + + pub fn get_lsn_range(&self) -> Range { + self.lsn_range.clone() + } + + pub fn get_key_range(&self) -> Range { + self.key_range.clone() + } + + pub fn get_timeline_id(&self) -> TimelineId { + self.timeline_id + } + + pub fn get_tenant_id(&self) -> TenantId { + self.tenant_id + } + + pub fn is_incremental(&self) -> bool { + self.is_incremental + } + + pub fn is_delta(&self) -> bool { + self.is_delta + } + + pub fn dump(&self, _verbose: bool, _ctx: &RequestContext) -> Result<()> { + println!( + "----- layer for ten {} tli {} keys {}-{} lsn {}-{} ----", + self.tenant_id, + self.timeline_id, + self.key_range.start, + self.key_range.end, + self.lsn_range.start, + self.lsn_range.end + ); + + Ok(()) + } + + pub fn file_size(&self) -> u64 { + self.file_size + } +} diff --git a/pageserver/src/tenant/storage_layer/remote_layer.rs b/pageserver/src/tenant/storage_layer/remote_layer.rs index 2106587ab2..9d423ed815 100644 --- a/pageserver/src/tenant/storage_layer/remote_layer.rs +++ b/pageserver/src/tenant/storage_layer/remote_layer.rs @@ -18,11 +18,10 @@ use utils::{ lsn::Lsn, }; -use super::filename::{DeltaFileName, ImageFileName, LayerFileName}; -use super::image_layer::ImageLayer; +use super::filename::{DeltaFileName, ImageFileName}; use super::{ - DeltaLayer, LayerAccessStats, LayerAccessStatsReset, LayerIter, LayerKeyIter, - LayerResidenceStatus, PersistentLayer, + DeltaLayer, ImageLayer, LayerAccessStats, LayerAccessStatsReset, LayerIter, LayerKeyIter, + LayerResidenceStatus, PersistentLayer, PersistentLayerDesc, }; /// RemoteLayer is a not yet downloaded [`ImageLayer`] or @@ -34,19 +33,10 @@ use super::{ /// /// See: [`crate::context::RequestContext`] for authorization to download pub struct RemoteLayer { - tenantid: TenantId, - timelineid: TimelineId, - key_range: Range, - lsn_range: Range, - - pub file_name: LayerFileName, + pub desc: PersistentLayerDesc, pub layer_metadata: LayerFileMetadata, - is_delta: bool, - - is_incremental: bool, - access_stats: LayerAccessStats, pub(crate) ongoing_download: Arc, @@ -66,22 +56,14 @@ pub struct RemoteLayer { impl std::fmt::Debug for RemoteLayer { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.debug_struct("RemoteLayer") - .field("file_name", &self.file_name) + .field("file_name", &self.desc.filename()) .field("layer_metadata", &self.layer_metadata) - .field("is_incremental", &self.is_incremental) + .field("is_incremental", &self.desc.is_incremental) .finish() } } impl Layer for RemoteLayer { - fn get_key_range(&self) -> Range { - self.key_range.clone() - } - - fn get_lsn_range(&self) -> Range { - self.lsn_range.clone() - } - fn get_value_reconstruct_data( &self, _key: Key, @@ -95,53 +77,45 @@ impl Layer for RemoteLayer { ); } - fn is_incremental(&self) -> bool { - self.is_incremental - } - /// debugging function to print out the contents of the layer fn dump(&self, _verbose: bool, _ctx: &RequestContext) -> Result<()> { println!( "----- remote layer for ten {} tli {} keys {}-{} lsn {}-{} ----", - self.tenantid, - self.timelineid, - self.key_range.start, - self.key_range.end, - self.lsn_range.start, - self.lsn_range.end + self.desc.tenant_id, + self.desc.timeline_id, + self.desc.key_range.start, + self.desc.key_range.end, + self.desc.lsn_range.start, + self.desc.lsn_range.end ); Ok(()) } + /// Boilerplate to implement the Layer trait, always use layer_desc for persistent layers. + fn get_key_range(&self) -> Range { + self.layer_desc().key_range.clone() + } + + /// Boilerplate to implement the Layer trait, always use layer_desc for persistent layers. + fn get_lsn_range(&self) -> Range { + self.layer_desc().lsn_range.clone() + } + + /// Boilerplate to implement the Layer trait, always use layer_desc for persistent layers. + fn is_incremental(&self) -> bool { + self.layer_desc().is_incremental + } + + /// Boilerplate to implement the Layer trait, always use layer_desc for persistent layers. fn short_id(&self) -> String { - self.filename().file_name() + self.layer_desc().short_id() } } impl PersistentLayer for RemoteLayer { - fn get_tenant_id(&self) -> TenantId { - self.tenantid - } - - fn get_timeline_id(&self) -> TimelineId { - self.timelineid - } - - fn filename(&self) -> LayerFileName { - if self.is_delta { - DeltaFileName { - key_range: self.key_range.clone(), - lsn_range: self.lsn_range.clone(), - } - .into() - } else { - ImageFileName { - key_range: self.key_range.clone(), - lsn: self.lsn_range.start, - } - .into() - } + fn layer_desc(&self) -> &PersistentLayerDesc { + &self.desc } fn local_path(&self) -> Option { @@ -168,15 +142,11 @@ impl PersistentLayer for RemoteLayer { true } - fn file_size(&self) -> u64 { - self.layer_metadata.file_size() - } - fn info(&self, reset: LayerAccessStatsReset) -> HistoricLayerInfo { let layer_file_name = self.filename().file_name(); let lsn_range = self.get_lsn_range(); - if self.is_delta { + if self.desc.is_delta { HistoricLayerInfo::Delta { layer_file_name, layer_file_size: self.layer_metadata.file_size(), @@ -210,13 +180,14 @@ impl RemoteLayer { access_stats: LayerAccessStats, ) -> RemoteLayer { RemoteLayer { - tenantid, - timelineid, - key_range: fname.key_range.clone(), - lsn_range: fname.lsn_as_range(), - is_delta: false, - is_incremental: false, - file_name: fname.to_owned().into(), + desc: PersistentLayerDesc::new_img( + tenantid, + timelineid, + fname.key_range.clone(), + fname.lsn, + false, + layer_metadata.file_size(), + ), layer_metadata: layer_metadata.clone(), ongoing_download: Arc::new(tokio::sync::Semaphore::new(1)), download_replacement_failure: std::sync::atomic::AtomicBool::default(), @@ -232,13 +203,13 @@ impl RemoteLayer { access_stats: LayerAccessStats, ) -> RemoteLayer { RemoteLayer { - tenantid, - timelineid, - key_range: fname.key_range.clone(), - lsn_range: fname.lsn_range.clone(), - is_delta: true, - is_incremental: true, - file_name: fname.to_owned().into(), + desc: PersistentLayerDesc::new_delta( + tenantid, + timelineid, + fname.key_range.clone(), + fname.lsn_range.clone(), + layer_metadata.file_size(), + ), layer_metadata: layer_metadata.clone(), ongoing_download: Arc::new(tokio::sync::Semaphore::new(1)), download_replacement_failure: std::sync::atomic::AtomicBool::default(), @@ -247,24 +218,18 @@ impl RemoteLayer { } /// Create a Layer struct representing this layer, after it has been downloaded. - pub fn create_downloaded_layer( + pub fn create_downloaded_layer( &self, - layer_map_lock_held_witness: &BatchedUpdates<'_, L>, + layer_map_lock_held_witness: &BatchedUpdates<'_>, conf: &'static PageServerConf, file_size: u64, - ) -> Arc - where - L: ?Sized + Layer, - { - if self.is_delta { - let fname = DeltaFileName { - key_range: self.key_range.clone(), - lsn_range: self.lsn_range.clone(), - }; + ) -> Arc { + if self.desc.is_delta { + let fname = self.desc.delta_file_name(); Arc::new(DeltaLayer::new( conf, - self.timelineid, - self.tenantid, + self.desc.timeline_id, + self.desc.tenant_id, &fname, file_size, self.access_stats.clone_for_residence_change( @@ -273,14 +238,11 @@ impl RemoteLayer { ), )) } else { - let fname = ImageFileName { - key_range: self.key_range.clone(), - lsn: self.lsn_range.start, - }; + let fname = self.desc.image_file_name(); Arc::new(ImageLayer::new( conf, - self.timelineid, - self.tenantid, + self.desc.timeline_id, + self.desc.tenant_id, &fname, file_size, self.access_stats.clone_for_residence_change( diff --git a/pageserver/src/tenant/tasks.rs b/pageserver/src/tenant/tasks.rs index b3c8a4a3bb..360818b5a7 100644 --- a/pageserver/src/tenant/tasks.rs +++ b/pageserver/src/tenant/tasks.rs @@ -12,8 +12,13 @@ use crate::task_mgr::{TaskKind, BACKGROUND_RUNTIME}; use crate::tenant::{Tenant, TenantState}; use tokio_util::sync::CancellationToken; use tracing::*; +use utils::completion; -pub fn start_background_loops(tenant: &Arc) { +/// Start per tenant background loops: compaction and gc. +pub fn start_background_loops( + tenant: &Arc, + background_jobs_can_start: Option<&completion::Barrier>, +) { let tenant_id = tenant.tenant_id; task_mgr::spawn( BACKGROUND_RUNTIME.handle(), @@ -24,8 +29,14 @@ pub fn start_background_loops(tenant: &Arc) { false, { let tenant = Arc::clone(tenant); + let background_jobs_can_start = background_jobs_can_start.cloned(); async move { - compaction_loop(tenant) + let cancel = task_mgr::shutdown_token(); + tokio::select! { + _ = cancel.cancelled() => { return Ok(()) }, + _ = completion::Barrier::maybe_wait(background_jobs_can_start) => {} + }; + compaction_loop(tenant, cancel) .instrument(info_span!("compaction_loop", tenant_id = %tenant_id)) .await; Ok(()) @@ -41,8 +52,14 @@ pub fn start_background_loops(tenant: &Arc) { false, { let tenant = Arc::clone(tenant); + let background_jobs_can_start = background_jobs_can_start.cloned(); async move { - gc_loop(tenant) + let cancel = task_mgr::shutdown_token(); + tokio::select! { + _ = cancel.cancelled() => { return Ok(()) }, + _ = completion::Barrier::maybe_wait(background_jobs_can_start) => {} + }; + gc_loop(tenant, cancel) .instrument(info_span!("gc_loop", tenant_id = %tenant_id)) .await; Ok(()) @@ -54,12 +71,11 @@ pub fn start_background_loops(tenant: &Arc) { /// /// Compaction task's main loop /// -async fn compaction_loop(tenant: Arc) { +async fn compaction_loop(tenant: Arc, cancel: CancellationToken) { let wait_duration = Duration::from_secs(2); info!("starting"); TENANT_TASK_EVENTS.with_label_values(&["start"]).inc(); async { - let cancel = task_mgr::shutdown_token(); let ctx = RequestContext::todo_child(TaskKind::Compaction, DownloadBehavior::Download); let mut first = true; loop { @@ -124,12 +140,11 @@ async fn compaction_loop(tenant: Arc) { /// /// GC task's main loop /// -async fn gc_loop(tenant: Arc) { +async fn gc_loop(tenant: Arc, cancel: CancellationToken) { let wait_duration = Duration::from_secs(2); info!("starting"); TENANT_TASK_EVENTS.with_label_values(&["start"]).inc(); async { - let cancel = task_mgr::shutdown_token(); // GC might require downloading, to find the cutoff LSN that corresponds to the // cutoff specified as time. let ctx = diff --git a/pageserver/src/tenant/timeline.rs b/pageserver/src/tenant/timeline.rs index b0aca45882..39c72a7e47 100644 --- a/pageserver/src/tenant/timeline.rs +++ b/pageserver/src/tenant/timeline.rs @@ -3,7 +3,7 @@ mod eviction_task; mod walreceiver; -use anyhow::{anyhow, bail, ensure, Context}; +use anyhow::{anyhow, bail, ensure, Context, Result}; use bytes::Bytes; use fail::fail_point; use futures::StreamExt; @@ -15,6 +15,7 @@ use pageserver_api::models::{ TimelineState, }; use remote_storage::GenericRemoteStorage; +use serde_with::serde_as; use storage_broker::BrokerClientChannel; use tokio::sync::{oneshot, watch, Semaphore, TryAcquireError}; use tokio_util::sync::CancellationToken; @@ -28,7 +29,7 @@ use std::ops::{Deref, Range}; use std::path::{Path, PathBuf}; use std::pin::pin; use std::sync::atomic::{AtomicI64, Ordering as AtomicOrdering}; -use std::sync::{Arc, Mutex, MutexGuard, RwLock, Weak}; +use std::sync::{Arc, Mutex, RwLock, Weak}; use std::time::{Duration, Instant, SystemTime}; use crate::context::{DownloadBehavior, RequestContext}; @@ -47,7 +48,10 @@ use crate::tenant::{ use crate::config::PageServerConf; use crate::keyspace::{KeyPartitioning, KeySpace, KeySpaceRandomAccum}; -use crate::metrics::{TimelineMetrics, UNEXPECTED_ONDEMAND_DOWNLOADS}; +use crate::metrics::{ + TimelineMetrics, MATERIALIZED_PAGE_CACHE_HIT, MATERIALIZED_PAGE_CACHE_HIT_DIRECT, + RECONSTRUCT_TIME, UNEXPECTED_ONDEMAND_DOWNLOADS, +}; use crate::pgdatadir_mapping::LsnForTimestamp; use crate::pgdatadir_mapping::{is_rel_fsm_block_key, is_rel_vm_block_key}; use crate::pgdatadir_mapping::{BlockNumber, CalculateLogicalSizeError}; @@ -57,6 +61,7 @@ use pageserver_api::reltag::RelTag; use postgres_connection::PgConnectionConfig; use postgres_ffi::to_pg_timestamp; use utils::{ + completion, id::{TenantId, TimelineId}, lsn::{AtomicLsn, Lsn, RecordLsn}, seqwait::SeqWait, @@ -80,12 +85,19 @@ use super::config::TenantConf; use super::layer_map::BatchedUpdates; use super::remote_timeline_client::index::IndexPart; use super::remote_timeline_client::RemoteTimelineClient; -use super::storage_layer::{DeltaLayer, ImageLayer, Layer, LayerAccessStatsReset}; +use super::storage_layer::{ + DeltaLayer, ImageLayer, Layer, LayerAccessStatsReset, PersistentLayerDesc, PersistentLayerKey, +}; #[derive(Debug, PartialEq, Eq, Clone, Copy)] -enum FlushLoopState { +pub(super) enum FlushLoopState { NotStarted, - Running, + Running { + #[cfg(test)] + expect_initdb_optimization: bool, + #[cfg(test)] + initdb_optimization_count: usize, + }, Exited, } @@ -108,6 +120,92 @@ impl PartialOrd for Hole { } } +pub struct LayerFileManager(HashMap>); + +impl LayerFileManager { + fn get_from_desc(&self, desc: &PersistentLayerDesc) -> Arc { + // The assumption for the `expect()` is that all code maintains the following invariant: + // A layer's descriptor is present in the LayerMap => the LayerFileManager contains a layer for the descriptor. + self.0 + .get(&desc.key()) + .with_context(|| format!("get layer from desc: {}", desc.filename().file_name())) + .expect("not found") + .clone() + } + + pub(crate) fn insert(&mut self, layer: Arc) { + let present = self.0.insert(layer.layer_desc().key(), layer.clone()); + if present.is_some() && cfg!(debug_assertions) { + panic!("overwriting a layer: {:?}", layer.layer_desc()) + } + } + + pub(crate) fn new() -> Self { + Self(HashMap::new()) + } + + pub(crate) fn remove(&mut self, layer: Arc) { + let present = self.0.remove(&layer.layer_desc().key()); + if present.is_none() && cfg!(debug_assertions) { + panic!( + "removing layer that is not present in layer mapping: {:?}", + layer.layer_desc() + ) + } + } + + pub(crate) fn replace_and_verify( + &mut self, + expected: Arc, + new: Arc, + ) -> Result<()> { + let key = expected.layer_desc().key(); + let other = new.layer_desc().key(); + + let expected_l0 = LayerMap::is_l0(expected.layer_desc()); + let new_l0 = LayerMap::is_l0(new.layer_desc()); + + fail::fail_point!("layermap-replace-notfound", |_| anyhow::bail!( + "layermap-replace-notfound" + )); + + anyhow::ensure!( + key == other, + "expected and new layer have different keys: {key:?} != {other:?}" + ); + + anyhow::ensure!( + expected_l0 == new_l0, + "one layer is l0 while the other is not: {expected_l0} != {new_l0}" + ); + + if let Some(layer) = self.0.get_mut(&expected.layer_desc().key()) { + anyhow::ensure!( + compare_arced_layers(&expected, layer), + "another layer was found instead of expected, expected={expected:?}, new={new:?}", + expected = Arc::as_ptr(&expected), + new = Arc::as_ptr(layer), + ); + *layer = new; + Ok(()) + } else { + anyhow::bail!("layer was not found"); + } + } +} + +/// Temporary function for immutable storage state refactor, ensures we are dropping mutex guard instead of other things. +/// Can be removed after all refactors are done. +fn drop_rlock(rlock: tokio::sync::OwnedRwLockReadGuard) { + drop(rlock) +} + +/// Temporary function for immutable storage state refactor, ensures we are dropping mutex guard instead of other things. +/// Can be removed after all refactors are done. +fn drop_wlock(rlock: tokio::sync::RwLockWriteGuard<'_, T>) { + drop(rlock) +} + pub struct Timeline { conf: &'static PageServerConf, tenant_conf: Arc>, @@ -119,7 +217,24 @@ pub struct Timeline { pub pg_version: u32, - pub(super) layers: RwLock>, + /// The tuple has two elements. + /// 1. `LayerFileManager` keeps track of the various physical representations of the layer files (inmem, local, remote). + /// 2. `LayerMap`, the acceleration data structure for `get_reconstruct_data`. + /// + /// `LayerMap` maps out the `(PAGE,LSN) / (KEY,LSN)` space, which is composed of `(KeyRange, LsnRange)` rectangles. + /// We describe these rectangles through the `PersistentLayerDesc` struct. + /// + /// When we want to reconstruct a page, we first find the `PersistentLayerDesc`'s that we need for page reconstruction, + /// using `LayerMap`. Then, we use `LayerFileManager` to get the `PersistentLayer`'s that correspond to the + /// `PersistentLayerDesc`'s. + /// + /// Hence, it's important to keep things coherent. The `LayerFileManager` must always have an entry for all + /// `PersistentLayerDesc`'s in the `LayerMap`. If it doesn't, `LayerFileManager::get_from_desc` will panic at + /// runtime, e.g., during page reconstruction. + /// + /// In the future, we'll be able to split up the tuple of LayerMap and `LayerFileManager`, + /// so that e.g. on-demand-download/eviction, and layer spreading, can operate just on `LayerFileManager`. + pub(crate) layers: Arc>, /// Set of key ranges which should be covered by image layers to /// allow GC to remove old layers. This set is created by GC and its cutoff LSN is also stored. @@ -179,10 +294,10 @@ pub struct Timeline { /// Locked automatically by [`TimelineWriter`] and checkpointer. /// Must always be acquired before the layer map/individual layer lock /// to avoid deadlock. - write_lock: Mutex<()>, + write_lock: tokio::sync::Mutex<()>, /// Used to avoid multiple `flush_loop` tasks running - flush_loop_state: Mutex, + pub(super) flush_loop_state: Mutex, /// layer_flush_start_tx can be used to wake up the layer-flushing task. /// The value is a counter, incremented every time a new flush cycle is requested. @@ -195,8 +310,9 @@ pub struct Timeline { /// Layer removal lock. /// A lock to ensure that no layer of the timeline is removed concurrently by other tasks. /// This lock is acquired in [`Timeline::gc`], [`Timeline::compact`], - /// and [`Tenant::delete_timeline`]. - pub(super) layer_removal_cs: tokio::sync::Mutex<()>, + /// and [`Tenant::delete_timeline`]. This is an `Arc` lock because we need an owned + /// lock guard in functions that will be spawned to tokio I/O pool (which requires `'static`). + pub(super) layer_removal_cs: Arc>, // Needed to ensure that we can't create a branch at a point that was already garbage collected pub latest_gc_cutoff_lsn: Rcu, @@ -235,7 +351,18 @@ pub struct Timeline { state: watch::Sender, + /// Prevent two tasks from deleting the timeline at the same time. If held, the + /// timeline is being deleted. If 'true', the timeline has already been deleted. + pub delete_lock: Arc>, + eviction_task_timeline_state: tokio::sync::Mutex, + + /// Barrier to wait before doing initial logical size calculation. Used only during startup. + initial_logical_size_can_start: Option, + + /// Completion shared between all timelines loaded during startup; used to delay heavier + /// background tasks until some logical sizes have been calculated. + initial_logical_size_attempt: Mutex>, } /// Internal structure to hold all data needed for logical size calculation. @@ -520,7 +647,10 @@ impl Timeline { Some((cached_lsn, cached_img)) => { match cached_lsn.cmp(&lsn) { Ordering::Less => {} // there might be WAL between cached_lsn and lsn, we need to check - Ordering::Equal => return Ok(cached_img), // exact LSN match, return the image + Ordering::Equal => { + MATERIALIZED_PAGE_CACHE_HIT_DIRECT.inc(); + return Ok(cached_img); // exact LSN match, return the image + } Ordering::Greater => { unreachable!("the returned lsn should never be after the requested lsn") } @@ -535,11 +665,12 @@ impl Timeline { img: cached_page_img, }; + let timer = self.metrics.get_reconstruct_data_time_histo.start_timer(); self.get_reconstruct_data(key, lsn, &mut reconstruct_state, ctx) .await?; + timer.stop_and_record(); - self.metrics - .reconstruct_time_histo + RECONSTRUCT_TIME .observe_closure_duration(|| self.reconstruct_value(key, lsn, reconstruct_state)) } @@ -572,8 +703,9 @@ impl Timeline { /// The sum of the file size of all historic layers in the layer map. /// This method makes no distinction between local and remote layers. /// Hence, the result **does not represent local filesystem usage**. - pub fn layer_size_sum(&self) -> u64 { - let layer_map = self.layers.read().unwrap(); + pub async fn layer_size_sum(&self) -> u64 { + let guard = self.layers.read().await; + let (layer_map, _) = &*guard; let mut size = 0; for l in layer_map.iter_historic_layers() { size += l.file_size(); @@ -664,12 +796,12 @@ impl Timeline { /// Flush to disk all data that was written with the put_* functions #[instrument(skip(self), fields(tenant_id=%self.tenant_id, timeline_id=%self.timeline_id))] pub async fn freeze_and_flush(&self) -> anyhow::Result<()> { - self.freeze_inmem_layer(false); + self.freeze_inmem_layer(false).await; self.flush_frozen_layers_and_wait().await } /// Outermost timeline compaction operation; downloads needed layers. - pub async fn compact(&self, ctx: &RequestContext) -> anyhow::Result<()> { + pub async fn compact(self: &Arc, ctx: &RequestContext) -> anyhow::Result<()> { const ROUNDS: usize = 2; let last_record_lsn = self.get_last_record_lsn(); @@ -758,7 +890,7 @@ impl Timeline { } /// Compaction which might need to be retried after downloading remote layers. - async fn compact_inner(&self, ctx: &RequestContext) -> Result<(), CompactionError> { + async fn compact_inner(self: &Arc, ctx: &RequestContext) -> Result<(), CompactionError> { // // High level strategy for compaction / image creation: // @@ -793,10 +925,9 @@ impl Timeline { // Below are functions compact_level0() and create_image_layers() // but they are a bit ad hoc and don't quite work like it's explained // above. Rewrite it. - let layer_removal_cs = self.layer_removal_cs.lock().await; + let layer_removal_cs = Arc::new(self.layer_removal_cs.clone().lock_owned().await); // Is the timeline being deleted? - let state = *self.state.borrow(); - if state == TimelineState::Stopping { + if self.is_stopping() { return Err(anyhow::anyhow!("timeline is Stopping").into()); } @@ -827,7 +958,7 @@ impl Timeline { // 3. Compact let timer = self.metrics.compact_time_histo.start_timer(); - self.compact_level0(&layer_removal_cs, target_file_size, ctx) + self.compact_level0(layer_removal_cs.clone(), target_file_size, ctx) .await?; timer.stop_and_record(); } @@ -844,10 +975,10 @@ impl Timeline { } /// Mutate the timeline with a [`TimelineWriter`]. - pub fn writer(&self) -> TimelineWriter<'_> { + pub async fn writer(&self) -> TimelineWriter<'_> { TimelineWriter { tl: self, - _write_guard: self.write_lock.lock().unwrap(), + _write_guard: self.write_lock.lock().await, } } @@ -881,45 +1012,53 @@ impl Timeline { /// /// Also flush after a period of time without new data -- it helps /// safekeepers to regard pageserver as caught up and suspend activity. - pub fn check_checkpoint_distance(self: &Arc) -> anyhow::Result<()> { + pub async fn check_checkpoint_distance(self: &Arc) -> anyhow::Result<()> { let last_lsn = self.get_last_record_lsn(); - let layers = self.layers.read().unwrap(); - if let Some(open_layer) = &layers.open_layer { - let open_layer_size = open_layer.size()?; - drop(layers); - let last_freeze_at = self.last_freeze_at.load(); - let last_freeze_ts = *(self.last_freeze_ts.read().unwrap()); - let distance = last_lsn.widening_sub(last_freeze_at); - // Checkpointing the open layer can be triggered by layer size or LSN range. - // S3 has a 5 GB limit on the size of one upload (without multi-part upload), and - // we want to stay below that with a big margin. The LSN distance determines how - // much WAL the safekeepers need to store. - if distance >= self.get_checkpoint_distance().into() - || open_layer_size > self.get_checkpoint_distance() - || (distance > 0 && last_freeze_ts.elapsed() >= self.get_checkpoint_timeout()) - { - info!( - "check_checkpoint_distance {}, layer size {}, elapsed since last flush {:?}", - distance, - open_layer_size, - last_freeze_ts.elapsed() - ); + let open_layer_size = { + let guard = self.layers.read().await; + let (layers, _) = &*guard; + let Some(open_layer) = layers.open_layer.as_ref() else { + return Ok(()); + }; + open_layer.size()? + }; + let last_freeze_at = self.last_freeze_at.load(); + let last_freeze_ts = *(self.last_freeze_ts.read().unwrap()); + let distance = last_lsn.widening_sub(last_freeze_at); + // Checkpointing the open layer can be triggered by layer size or LSN range. + // S3 has a 5 GB limit on the size of one upload (without multi-part upload), and + // we want to stay below that with a big margin. The LSN distance determines how + // much WAL the safekeepers need to store. + if distance >= self.get_checkpoint_distance().into() + || open_layer_size > self.get_checkpoint_distance() + || (distance > 0 && last_freeze_ts.elapsed() >= self.get_checkpoint_timeout()) + { + info!( + "check_checkpoint_distance {}, layer size {}, elapsed since last flush {:?}", + distance, + open_layer_size, + last_freeze_ts.elapsed() + ); - self.freeze_inmem_layer(true); - self.last_freeze_at.store(last_lsn); - *(self.last_freeze_ts.write().unwrap()) = Instant::now(); + self.freeze_inmem_layer(true).await; + self.last_freeze_at.store(last_lsn); + *(self.last_freeze_ts.write().unwrap()) = Instant::now(); - // Wake up the layer flusher - self.flush_frozen_layers(); - } + // Wake up the layer flusher + self.flush_frozen_layers(); } Ok(()) } - pub fn activate(self: &Arc, broker_client: BrokerClientChannel, ctx: &RequestContext) { + pub fn activate( + self: &Arc, + broker_client: BrokerClientChannel, + background_jobs_can_start: Option<&completion::Barrier>, + ctx: &RequestContext, + ) { self.launch_wal_receiver(ctx, broker_client); self.set_state(TimelineState::Active); - self.launch_eviction_task(); + self.launch_eviction_task(background_jobs_can_start); } pub fn set_state(&self, new_state: TimelineState) { @@ -930,26 +1069,54 @@ impl Timeline { (st, TimelineState::Loading) => { error!("ignoring transition from {st:?} into Loading state"); } - (TimelineState::Broken, _) => { - error!("Ignoring state update {new_state:?} for broken tenant"); + (TimelineState::Broken { .. }, new_state) => { + error!("Ignoring state update {new_state:?} for broken timeline"); } (TimelineState::Stopping, TimelineState::Active) => { error!("Not activating a Stopping timeline"); } (_, new_state) => { + if matches!( + new_state, + TimelineState::Stopping | TimelineState::Broken { .. } + ) { + // drop the copmletion guard, if any; it might be holding off the completion + // forever needlessly + self.initial_logical_size_attempt + .lock() + .unwrap_or_else(|e| e.into_inner()) + .take(); + } self.state.send_replace(new_state); } } } + pub fn set_broken(&self, reason: String) { + let backtrace_str: String = format!("{}", std::backtrace::Backtrace::force_capture()); + let broken_state = TimelineState::Broken { + reason, + backtrace: backtrace_str, + }; + self.set_state(broken_state) + } + pub fn current_state(&self) -> TimelineState { - *self.state.borrow() + self.state.borrow().clone() + } + + pub fn is_broken(&self) -> bool { + matches!(&*self.state.borrow(), TimelineState::Broken { .. }) } pub fn is_active(&self) -> bool { self.current_state() == TimelineState::Active } + pub fn is_stopping(&self) -> bool { + self.current_state() == TimelineState::Stopping + } + pub fn subscribe_for_state_updates(&self) -> watch::Receiver { self.state.subscribe() } @@ -960,7 +1127,7 @@ impl Timeline { ) -> Result<(), TimelineState> { let mut receiver = self.state.subscribe(); loop { - let current_state = *receiver.borrow_and_update(); + let current_state = receiver.borrow().clone(); match current_state { TimelineState::Loading => { receiver @@ -979,8 +1146,9 @@ impl Timeline { } } - pub fn layer_map_info(&self, reset: LayerAccessStatsReset) -> LayerMapInfo { - let layer_map = self.layers.read().unwrap(); + pub async fn layer_map_info(&self, reset: LayerAccessStatsReset) -> LayerMapInfo { + let guard = self.layers.read().await; + let (layer_map, mapping) = &*guard; let mut in_memory_layers = Vec::with_capacity(layer_map.frozen_layers.len() + 1); if let Some(open_layer) = &layer_map.open_layer { in_memory_layers.push(open_layer.info()); @@ -991,6 +1159,7 @@ impl Timeline { let mut historic_layers = Vec::new(); for historic_layer in layer_map.iter_historic_layers() { + let historic_layer = mapping.get_from_desc(&historic_layer); historic_layers.push(historic_layer.info(reset)); } @@ -1002,7 +1171,7 @@ impl Timeline { #[instrument(skip_all, fields(tenant = %self.tenant_id, timeline = %self.timeline_id))] pub async fn download_layer(&self, layer_file_name: &str) -> anyhow::Result> { - let Some(layer) = self.find_layer(layer_file_name) else { return Ok(None) }; + let Some(layer) = self.find_layer(layer_file_name).await else { return Ok(None) }; let Some(remote_layer) = layer.downcast_remote_layer() else { return Ok(Some(false)) }; if self.remote_client.is_none() { return Ok(Some(false)); @@ -1015,7 +1184,7 @@ impl Timeline { /// Like [`evict_layer_batch`], but for just one layer. /// Additional case `Ok(None)` covers the case where the layer could not be found by its `layer_file_name`. pub async fn evict_layer(&self, layer_file_name: &str) -> anyhow::Result> { - let Some(local_layer) = self.find_layer(layer_file_name) else { return Ok(None) }; + let Some(local_layer) = self.find_layer(layer_file_name).await else { return Ok(None) }; let remote_client = self .remote_client .as_ref() @@ -1100,7 +1269,8 @@ impl Timeline { } // start the batch update - let mut layer_map = self.layers.write().unwrap(); + let mut guard = self.layers.write().await; + let (layer_map, mapping) = &mut *guard; let mut batch_updates = layer_map.batch_update(); let mut results = Vec::with_capacity(layers_to_evict.len()); @@ -1109,14 +1279,19 @@ impl Timeline { let res = if cancel.is_cancelled() { None } else { - Some(self.evict_layer_batch_impl(&layer_removal_guard, l, &mut batch_updates)) + Some(self.evict_layer_batch_impl( + &layer_removal_guard, + l, + &mut batch_updates, + mapping, + )) }; results.push(res); } // commit the updates & release locks batch_updates.flush(); - drop(layer_map); + drop_wlock(guard); drop(layer_removal_guard); assert_eq!(results.len(), layers_to_evict.len()); @@ -1127,10 +1302,9 @@ impl Timeline { &self, _layer_removal_cs: &tokio::sync::MutexGuard<'_, ()>, local_layer: &Arc, - batch_updates: &mut BatchedUpdates<'_, dyn PersistentLayer>, + batch_updates: &mut BatchedUpdates<'_>, + mapping: &mut LayerFileManager, ) -> anyhow::Result { - use super::layer_map::Replacement; - if local_layer.is_remote_layer() { // TODO(issue #3851): consider returning an err here instead of false, // which is the same out the match later @@ -1178,8 +1352,10 @@ impl Timeline { ), }); - let replaced = match batch_updates.replace_historic(local_layer, new_remote_layer)? { - Replacement::Replaced { .. } => { + assert_eq!(local_layer.layer_desc(), new_remote_layer.layer_desc()); + + let succeed = match mapping.replace_and_verify(local_layer.clone(), new_remote_layer) { + Ok(()) => { if let Err(e) = local_layer.delete_resident_layer_file() { error!("failed to remove layer file on evict after replacement: {e:#?}"); } @@ -1212,27 +1388,23 @@ impl Timeline { true } - Replacement::NotFound => { - debug!(evicted=?local_layer, "layer was no longer in layer map"); - false - } - Replacement::RemovalBuffered => { - unreachable!("not doing anything else in this batch") - } - Replacement::Unexpected(other) => { - error!( - local_layer.ptr=?Arc::as_ptr(local_layer), - other.ptr=?Arc::as_ptr(&other), - ?other, - "failed to replace"); + Err(err) => { + if cfg!(debug_assertions) { + panic!("failed to replace: {err}, evicted: {local_layer:?}"); + } else { + error!(evicted=?local_layer, "failed to replace: {err}"); + } false } }; - Ok(replaced) + Ok(succeed) } } +/// Number of times we will compute partition within a checkpoint distance. +const REPARTITION_FREQ_IN_CHECKPOINT_DISTANCE: u64 = 10; + // Private functions impl Timeline { fn get_checkpoint_distance(&self) -> u64 { @@ -1286,6 +1458,13 @@ impl Timeline { .unwrap_or(default_tenant_conf.evictions_low_residence_duration_metric_threshold) } + fn get_gc_feedback(&self) -> bool { + let tenant_conf = self.tenant_conf.read().unwrap(); + tenant_conf + .gc_feedback + .unwrap_or(self.conf.default_tenant_conf.gc_feedback) + } + pub(super) fn tenant_conf_updated(&self) { // NB: Most tenant conf options are read by background loops, so, // changes will automatically be picked up. @@ -1320,6 +1499,8 @@ impl Timeline { walredo_mgr: Arc, remote_client: Option, pg_version: u32, + initial_logical_size_can_start: Option, + initial_logical_size_attempt: Option, ) -> Arc { let disk_consistent_lsn = metadata.disk_consistent_lsn(); let (state, _) = watch::channel(TimelineState::Loading); @@ -1344,7 +1525,10 @@ impl Timeline { timeline_id, tenant_id, pg_version, - layers: RwLock::new(LayerMap::default()), + layers: Arc::new(tokio::sync::RwLock::new(( + LayerMap::default(), + LayerFileManager::new(), + ))), wanted_image_layers: Mutex::new(None), walredo_mgr, @@ -1379,7 +1563,7 @@ impl Timeline { layer_flush_start_tx, layer_flush_done_tx, - write_lock: Mutex::new(()), + write_lock: tokio::sync::Mutex::new(()), layer_removal_cs: Default::default(), gc_info: std::sync::RwLock::new(GcInfo { @@ -1413,8 +1597,13 @@ impl Timeline { eviction_task_timeline_state: tokio::sync::Mutex::new( EvictionTaskTimelineState::default(), ), + delete_lock: Arc::new(tokio::sync::Mutex::new(false)), + + initial_logical_size_can_start, + initial_logical_size_attempt: Mutex::new(initial_logical_size_attempt), }; - result.repartition_threshold = result.get_checkpoint_distance() / 10; + result.repartition_threshold = + result.get_checkpoint_distance() / REPARTITION_FREQ_IN_CHECKPOINT_DISTANCE; result .metrics .last_record_gauge @@ -1427,7 +1616,7 @@ impl Timeline { let mut flush_loop_state = self.flush_loop_state.lock().unwrap(); match *flush_loop_state { FlushLoopState::NotStarted => (), - FlushLoopState::Running => { + FlushLoopState::Running { .. } => { info!( "skipping attempt to start flush_loop twice {}/{}", self.tenant_id, self.timeline_id @@ -1447,6 +1636,12 @@ impl Timeline { let self_clone = Arc::clone(self); info!("spawning flush loop"); + *flush_loop_state = FlushLoopState::Running { + #[cfg(test)] + expect_initdb_optimization: false, + #[cfg(test)] + initdb_optimization_count: 0, + }; task_mgr::spawn( task_mgr::BACKGROUND_RUNTIME.handle(), task_mgr::TaskKind::LayerFlushTask, @@ -1458,14 +1653,12 @@ impl Timeline { let background_ctx = RequestContext::todo_child(TaskKind::LayerFlushTask, DownloadBehavior::Error); self_clone.flush_loop(layer_flush_start_rx, &background_ctx).await; let mut flush_loop_state = self_clone.flush_loop_state.lock().unwrap(); - assert_eq!(*flush_loop_state, FlushLoopState::Running); + assert!(matches!(*flush_loop_state, FlushLoopState::Running{ ..})); *flush_loop_state = FlushLoopState::Exited; Ok(()) } .instrument(info_span!(parent: None, "layer flush task", tenant = %self.tenant_id, timeline = %self.timeline_id)) ); - - *flush_loop_state = FlushLoopState::Running; } /// Creates and starts the wal receiver. @@ -1514,11 +1707,21 @@ impl Timeline { } /// - /// Scan the timeline directory to populate the layer map. - /// Returns all timeline-related files that were found and loaded. + /// Initialize with an empty layer map. Used when creating a new timeline. /// - pub(super) fn load_layer_map(&self, disk_consistent_lsn: Lsn) -> anyhow::Result<()> { - let mut layers = self.layers.write().unwrap(); + pub(super) fn init_empty_layer_map(&self, start_lsn: Lsn) { + let mut layers = self.layers.try_write().expect( + "in the context where we call this function, no other task has access to the object", + ); + layers.0.next_open_layer_at = Some(Lsn(start_lsn.0)); + } + + /// + /// Scan the timeline directory to populate the layer map. + /// + pub(super) async fn load_layer_map(&self, disk_consistent_lsn: Lsn) -> anyhow::Result<()> { + let mut guard = self.layers.write().await; + let (layers, mapping) = &mut *guard; let mut updates = layers.batch_update(); let mut num_layers = 0; @@ -1561,7 +1764,7 @@ impl Timeline { trace!("found layer {}", layer.path().display()); total_physical_size += file_size; - updates.insert_historic(Arc::new(layer)); + self.insert_historic_layer(Arc::new(layer), &mut updates, mapping); num_layers += 1; } else if let Some(deltafilename) = DeltaFileName::parse_str(&fname) { // Create a DeltaLayer struct for each delta file. @@ -1593,7 +1796,7 @@ impl Timeline { trace!("found layer {}", layer.path().display()); total_physical_size += file_size; - updates.insert_historic(Arc::new(layer)); + self.insert_historic_layer(Arc::new(layer), &mut updates, mapping); num_layers += 1; } else if fname == METADATA_FILE_NAME || fname.ends_with(".old") { // ignore these @@ -1647,7 +1850,8 @@ impl Timeline { // We're holding a layer map lock for a while but this // method is only called during init so it's fine. - let mut layer_map = self.layers.write().unwrap(); + let mut guard = self.layers.write().await; + let (layer_map, mapping) = &mut *guard; let mut updates = layer_map.batch_update(); for remote_layer_name in &index_part.timeline_layers { let local_layer = local_only_layers.remove(remote_layer_name); @@ -1692,7 +1896,7 @@ impl Timeline { anyhow::bail!("could not rename file {local_layer_path:?}: {err:?}"); } else { self.metrics.resident_physical_size_gauge.sub(local_size); - updates.remove_historic(local_layer); + self.remove_historic_layer(local_layer, &mut updates, mapping); // fall-through to adding the remote layer } } else { @@ -1731,7 +1935,7 @@ impl Timeline { ); let remote_layer = Arc::new(remote_layer); - updates.insert_historic(remote_layer); + self.insert_historic_layer(remote_layer, &mut updates, mapping); } LayerFileName::Delta(deltafilename) => { // Create a RemoteLayer for the delta file. @@ -1758,7 +1962,7 @@ impl Timeline { ), ); let remote_layer = Arc::new(remote_layer); - updates.insert_historic(remote_layer); + self.insert_historic_layer(remote_layer, &mut updates, mapping); } } } @@ -1797,13 +2001,14 @@ impl Timeline { let disk_consistent_lsn = up_to_date_metadata.disk_consistent_lsn(); - let local_layers = self - .layers - .read() - .unwrap() - .iter_historic_layers() - .map(|l| (l.filename(), l)) - .collect::>(); + let local_layers = { + let guard = self.layers.read().await; + let (layers, mapping) = &*guard; + layers + .iter_historic_layers() + .map(|l| (l.filename(), mapping.get_from_desc(&l))) + .collect::>() + }; // If no writes happen, new branches do not have any layers, only the metadata file. let has_local_layers = !local_layers.is_empty(); @@ -1901,9 +2106,30 @@ impl Timeline { false, // NB: don't log errors here, task_mgr will do that. async move { - // no cancellation here, because nothing really waits for this to complete compared + + let cancel = task_mgr::shutdown_token(); + + // in case we were created during pageserver initialization, wait for + // initialization to complete before proceeding. startup time init runs on the same + // runtime. + tokio::select! { + _ = cancel.cancelled() => { return Ok(()); }, + _ = completion::Barrier::maybe_wait(self_clone.initial_logical_size_can_start.clone()) => {} + }; + + // hold off background tasks from starting until all timelines get to try at least + // once initial logical size calculation; though retry will rarely be useful. + // holding off is done because heavier tasks execute blockingly on the same + // runtime. + // + // dropping this at every outcome is probably better than trying to cling on to it, + // delay will be terminated by a timeout regardless. + let _completion = { self_clone.initial_logical_size_attempt.lock().expect("unexpected initial_logical_size_attempt poisoned").take() }; + + // no extra cancellation here, because nothing really waits for this to complete compared // to spawn_ondemand_logical_size_calculation. let cancel = CancellationToken::new(); + let calculated_size = match self_clone .logical_size_calculation_task(lsn, LogicalSizeCalculationCause::Initial, &background_ctx, cancel) .await @@ -2029,11 +2255,11 @@ impl Timeline { loop { match timeline_state_updates.changed().await { Ok(()) => { - let new_state = *timeline_state_updates.borrow(); + let new_state = timeline_state_updates.borrow().clone(); match new_state { // we're running this job for active timelines only TimelineState::Active => continue, - TimelineState::Broken + TimelineState::Broken { .. } | TimelineState::Stopping | TimelineState::Loading => { break format!("aborted because timeline became inactive (new state: {new_state:?})") @@ -2152,26 +2378,54 @@ impl Timeline { } } - fn find_layer(&self, layer_file_name: &str) -> Option> { - for historic_layer in self.layers.read().unwrap().iter_historic_layers() { + async fn find_layer(&self, layer_file_name: &str) -> Option> { + let guard = self.layers.read().await; + let (layers, mapping) = &*guard; + for historic_layer in layers.iter_historic_layers() { let historic_layer_name = historic_layer.filename().file_name(); if layer_file_name == historic_layer_name { - return Some(historic_layer); + return Some(mapping.get_from_desc(&historic_layer)); } } None } + /// Helper function to insert a layer from both layer map and layer file manager. Will be removed in the future + /// after we introduce `LayerMapManager`. + fn insert_historic_layer( + &self, + layer: Arc, + updates: &mut BatchedUpdates<'_>, + mapping: &mut LayerFileManager, + ) { + updates.insert_historic(layer.layer_desc().clone()); + mapping.insert(layer); + } + + /// Helper function to remove a layer from both layer map and layer file manager. Will be removed in the future + /// after we introduce `LayerMapManager`. + fn remove_historic_layer( + &self, + layer: Arc, + updates: &mut BatchedUpdates<'_>, + mapping: &mut LayerFileManager, + ) { + updates.remove_historic(layer.layer_desc().clone()); + mapping.remove(layer); + } + /// Removes the layer from local FS (if present) and from memory. /// Remote storage is not affected by this operation. fn delete_historic_layer( &self, // we cannot remove layers otherwise, since gc and compaction will race - _layer_removal_cs: &tokio::sync::MutexGuard<'_, ()>, - layer: Arc, - updates: &mut BatchedUpdates<'_, dyn PersistentLayer>, + _layer_removal_cs: Arc>, + layer: Arc, + updates: &mut BatchedUpdates<'_>, + mapping: &mut LayerFileManager, ) -> anyhow::Result<()> { + let layer = mapping.get_from_desc(&layer); if !layer.is_remote_layer() { layer.delete_resident_layer_file()?; let layer_file_size = layer.file_size(); @@ -2185,7 +2439,8 @@ impl Timeline { // won't be needed for page reconstruction for this timeline, // and mark what we can't delete yet as deleted from the layer // map index without actually rebuilding the index. - updates.remove_historic(layer); + updates.remove_historic(layer.layer_desc().clone()); + mapping.remove(layer); Ok(()) } @@ -2247,6 +2502,9 @@ impl Timeline { let mut timeline_owned; let mut timeline = self; + let mut read_count = + scopeguard::guard(0, |cnt| self.metrics.read_num_fs_layers.observe(cnt as f64)); + // For debugging purposes, collect the path of layers that we traversed // through. It's included in the error message if we fail to find the key. let mut traversal_path = Vec::::new(); @@ -2273,7 +2531,7 @@ impl Timeline { ValueReconstructResult::Continue => { // If we reached an earlier cached page image, we're done. if cont_lsn == cached_lsn + 1 { - self.metrics.materialized_page_cache_hit_counter.inc_by(1); + MATERIALIZED_PAGE_CACHE_HIT.inc_by(1); return Ok(()); } if prev_lsn <= cont_lsn { @@ -2291,10 +2549,17 @@ impl Timeline { } ValueReconstructResult::Missing => { return Err(layer_traversal_error( - format!( - "could not find data for key {} at LSN {}, for request at LSN {}", - key, cont_lsn, request_lsn - ), + if cfg!(test) { + format!( + "could not find data for key {} at LSN {}, for request at LSN {}\n{}", + key, cont_lsn, request_lsn, std::backtrace::Backtrace::force_capture(), + ) + } else { + format!( + "could not find data for key {} at LSN {}, for request at LSN {}", + key, cont_lsn, request_lsn + ) + }, traversal_path, )); } @@ -2360,7 +2625,8 @@ impl Timeline { #[allow(clippy::never_loop)] // see comment at bottom of this loop 'layer_map_search: loop { let remote_layer = { - let layers = timeline.layers.read().unwrap(); + let guard = timeline.layers.read().await; + let (layers, mapping) = &*guard; // Check the open and frozen in-memory layers first, in order from newest // to oldest. @@ -2381,6 +2647,7 @@ impl Timeline { Err(e) => return Err(PageReconstructError::from(e)), }; cont_lsn = lsn_floor; + // metrics: open_layer does not count as fs access, so we are not updating `read_count` traversal_path.push(( result, cont_lsn, @@ -2407,6 +2674,7 @@ impl Timeline { Err(e) => return Err(PageReconstructError::from(e)), }; cont_lsn = lsn_floor; + // metrics: open_layer does not count as fs access, so we are not updating `read_count` traversal_path.push(( result, cont_lsn, @@ -2420,6 +2688,7 @@ impl Timeline { } if let Some(SearchResult { lsn_floor, layer }) = layers.search(key, cont_lsn) { + let layer = mapping.get_from_desc(&layer); // If it's a remote layer, download it and retry. if let Some(remote_layer) = super::storage_layer::downcast_remote_layer(&layer) @@ -2441,6 +2710,7 @@ impl Timeline { Err(e) => return Err(PageReconstructError::from(e)), }; cont_lsn = lsn_floor; + *read_count += 1; traversal_path.push(( result, cont_lsn, @@ -2506,7 +2776,7 @@ impl Timeline { (DownloadBehavior::Error, false) => { return Err(PageReconstructError::NeedsDownload( TenantTimelineId::new(self.tenant_id, self.timeline_id), - remote_layer.file_name.clone(), + remote_layer.filename(), )) } } @@ -2539,24 +2809,30 @@ impl Timeline { /// /// Get a handle to the latest layer for appending. /// - fn get_layer_for_write(&self, lsn: Lsn) -> anyhow::Result> { - let mut layers = self.layers.write().unwrap(); + async fn get_layer_for_write(&self, lsn: Lsn) -> anyhow::Result> { + let mut guard = self.layers.write().await; + let (layers, _) = &mut *guard; ensure!(lsn.is_aligned()); let last_record_lsn = self.get_last_record_lsn(); ensure!( lsn > last_record_lsn, - "cannot modify relation after advancing last_record_lsn (incoming_lsn={}, last_record_lsn={})", + "cannot modify relation after advancing last_record_lsn (incoming_lsn={}, last_record_lsn={})\n{}", lsn, last_record_lsn, + std::backtrace::Backtrace::force_capture(), ); // Do we have a layer open for writing already? let layer; if let Some(open_layer) = &layers.open_layer { if open_layer.get_lsn_range().start > lsn { - bail!("unexpected open layer in the future"); + bail!( + "unexpected open layer in the future: open layers starts at {}, write lsn {}", + open_layer.get_lsn_range().start, + lsn + ); } layer = Arc::clone(open_layer); @@ -2584,17 +2860,16 @@ impl Timeline { Ok(layer) } - fn put_value(&self, key: Key, lsn: Lsn, val: &Value) -> anyhow::Result<()> { + async fn put_value(&self, key: Key, lsn: Lsn, val: &Value) -> anyhow::Result<()> { //info!("PUT: key {} at {}", key, lsn); - let layer = self.get_layer_for_write(lsn)?; + let layer = self.get_layer_for_write(lsn).await?; layer.put_value(key, lsn, val)?; Ok(()) } - fn put_tombstone(&self, key_range: Range, lsn: Lsn) -> anyhow::Result<()> { - let layer = self.get_layer_for_write(lsn)?; - layer.put_tombstone(key_range, lsn)?; - + async fn put_tombstone(&self, key_range: Range, lsn: Lsn) -> anyhow::Result<()> { + let layer = self.get_layer_for_write(lsn).await?; + layer.put_tombstone(key_range, lsn).await?; Ok(()) } @@ -2605,15 +2880,16 @@ impl Timeline { self.last_record_lsn.advance(new_lsn); } - fn freeze_inmem_layer(&self, write_lock_held: bool) { + async fn freeze_inmem_layer(&self, write_lock_held: bool) { // Freeze the current open in-memory layer. It will be written to disk on next // iteration. let _write_guard = if write_lock_held { None } else { - Some(self.write_lock.lock().unwrap()) + Some(self.write_lock.lock().await) }; - let mut layers = self.layers.write().unwrap(); + let mut guard = self.layers.write().await; + let (layers, _) = &mut *guard; if let Some(open_layer) = &layers.open_layer { let open_layer_rc = Arc::clone(open_layer); // Does this layer need freezing? @@ -2627,12 +2903,12 @@ impl Timeline { layers.next_open_layer_at = Some(end_lsn); self.last_freeze_at.store(end_lsn); } - drop(layers); + drop_wlock(guard); } /// Layer flusher task's main loop. async fn flush_loop( - &self, + self: &Arc, mut layer_flush_start_rx: tokio::sync::watch::Receiver, ctx: &RequestContext, ) { @@ -2651,7 +2927,8 @@ impl Timeline { let flush_counter = *layer_flush_start_rx.borrow(); let result = loop { let layer_to_flush = { - let layers = self.layers.read().unwrap(); + let guard = self.layers.read().await; + let (layers, _) = &*guard; layers.frozen_layers.front().cloned() // drop 'layers' lock to allow concurrent reads and writes }; @@ -2684,7 +2961,7 @@ impl Timeline { let mut my_flush_request = 0; let flush_loop_state = { *self.flush_loop_state.lock().unwrap() }; - if flush_loop_state != FlushLoopState::Running { + if !matches!(flush_loop_state, FlushLoopState::Running { .. }) { anyhow::bail!("cannot flush frozen layers when flush_loop is not running, state is {flush_loop_state:?}") } @@ -2721,9 +2998,9 @@ impl Timeline { } /// Flush one frozen in-memory layer to disk, as a new delta layer. - #[instrument(skip(self, frozen_layer, ctx), fields(tenant_id=%self.tenant_id, timeline_id=%self.timeline_id, layer=%frozen_layer.short_id()))] + #[instrument(skip_all, fields(tenant_id=%self.tenant_id, timeline_id=%self.timeline_id, layer=%frozen_layer.short_id()))] async fn flush_frozen_layer( - &self, + self: &Arc, frozen_layer: Arc, ctx: &RequestContext, ) -> anyhow::Result<()> { @@ -2734,6 +3011,18 @@ impl Timeline { let lsn_range = frozen_layer.get_lsn_range(); let layer_paths_to_upload = if lsn_range.start == self.initdb_lsn && lsn_range.end == Lsn(self.initdb_lsn.0 + 1) { + #[cfg(test)] + match &mut *self.flush_loop_state.lock().unwrap() { + FlushLoopState::NotStarted | FlushLoopState::Exited => { + panic!("flush loop not running") + } + FlushLoopState::Running { + initdb_optimization_count, + .. + } => { + *initdb_optimization_count += 1; + } + } // Note: The 'ctx' in use here has DownloadBehavior::Error. We should not // require downloading anything during initial import. let (partitioning, _lsn) = self @@ -2742,23 +3031,37 @@ impl Timeline { self.create_image_layers(&partitioning, self.initdb_lsn, true, ctx) .await? } else { + #[cfg(test)] + match &mut *self.flush_loop_state.lock().unwrap() { + FlushLoopState::NotStarted | FlushLoopState::Exited => { + panic!("flush loop not running") + } + FlushLoopState::Running { + expect_initdb_optimization, + .. + } => { + assert!(!*expect_initdb_optimization, "expected initdb optimization"); + } + } // normal case, write out a L0 delta layer file. - let (delta_path, metadata) = self.create_delta_layer(&frozen_layer)?; + let (delta_path, metadata) = self.create_delta_layer(&frozen_layer).await?; HashMap::from([(delta_path, metadata)]) }; fail_point!("flush-frozen-before-sync"); // The new on-disk layers are now in the layer map. We can remove the - // in-memory layer from the map now. + // in-memory layer from the map now. We do not modify `LayerFileManager` because + // it only contains persistent layers. The flushed layer is stored in + // the mapping in `create_delta_layer`. { - let mut layers = self.layers.write().unwrap(); - let l = layers.frozen_layers.pop_front(); + let mut layers = self.layers.write().await; + let l = layers.0.frozen_layers.pop_front(); // Only one thread may call this function at a time (for this // timeline). If two threads tried to flush the same frozen // layer to disk at the same time, that would not work. - assert!(LayerMap::compare_arced_layers(&l.unwrap(), &frozen_layer)); + assert!(compare_arced_layers(&l.unwrap(), &frozen_layer)); // release lock on 'layers' } @@ -2846,49 +3149,68 @@ impl Timeline { } // Write out the given frozen in-memory layer as a new L0 delta file - fn create_delta_layer( - &self, - frozen_layer: &InMemoryLayer, + async fn create_delta_layer( + self: &Arc, + frozen_layer: &Arc, ) -> anyhow::Result<(LayerFileName, LayerFileMetadata)> { - // Write it out - let new_delta = frozen_layer.write_to_disk()?; - let new_delta_path = new_delta.path(); - let new_delta_filename = new_delta.filename(); + let span = tracing::info_span!("blocking"); + let new_delta: DeltaLayer = tokio::task::spawn_blocking({ + let _g = span.entered(); + let self_clone = Arc::clone(self); + let frozen_layer = Arc::clone(frozen_layer); + move || { + // Write it out + let new_delta = frozen_layer.write_to_disk()?; + let new_delta_path = new_delta.path(); - // Sync it to disk. - // - // We must also fsync the timeline dir to ensure the directory entries for - // new layer files are durable - // - // TODO: If we're running inside 'flush_frozen_layers' and there are multiple - // files to flush, it might be better to first write them all, and then fsync - // them all in parallel. - par_fsync::par_fsync(&[ - new_delta_path.clone(), - self.conf.timeline_path(&self.timeline_id, &self.tenant_id), - ])?; + // Sync it to disk. + // + // We must also fsync the timeline dir to ensure the directory entries for + // new layer files are durable. + // + // NB: timeline dir must be synced _after_ the file contents are durable. + // So, two separate fsyncs are required, they mustn't be batched. + // + // TODO: If we're running inside 'flush_frozen_layers' and there are multiple + // files to flush, the fsync overhead can be reduces as follows: + // 1. write them all to temporary file names + // 2. fsync them + // 3. rename to the final name + // 4. fsync the parent directory. + // Note that (1),(2),(3) today happen inside write_to_disk(). + par_fsync::par_fsync(&[new_delta_path]).context("fsync of delta layer")?; + par_fsync::par_fsync(&[self_clone + .conf + .timeline_path(&self_clone.timeline_id, &self_clone.tenant_id)]) + .context("fsync of timeline dir")?; + + anyhow::Ok(new_delta) + } + }) + .await + .context("spawn_blocking")??; + let new_delta_name = new_delta.filename(); + let sz = new_delta.desc.file_size; // Add it to the layer map let l = Arc::new(new_delta); - let mut layers = self.layers.write().unwrap(); + let mut guard = self.layers.write().await; + let (layers, mapping) = &mut *guard; let mut batch_updates = layers.batch_update(); l.access_stats().record_residence_event( &batch_updates, LayerResidenceStatus::Resident, LayerResidenceEventReason::LayerCreate, ); - batch_updates.insert_historic(l); + self.insert_historic_layer(l, &mut batch_updates, mapping); batch_updates.flush(); - // update the timeline's physical size - let sz = new_delta_path.metadata()?.len(); - - self.metrics.resident_physical_size_gauge.add(sz); // update metrics + self.metrics.resident_physical_size_gauge.add(sz); self.metrics.num_persistent_files_created.inc_by(1); self.metrics.persistent_bytes_written.inc_by(sz); - Ok((new_delta_filename, LayerFileMetadata::new(sz))) + Ok((new_delta_name, LayerFileMetadata::new(sz))) } async fn repartition( @@ -2922,10 +3244,15 @@ impl Timeline { } // Is it time to create a new image layer for the given partition? - fn time_for_new_image_layer(&self, partition: &KeySpace, lsn: Lsn) -> anyhow::Result { + async fn time_for_new_image_layer( + &self, + partition: &KeySpace, + lsn: Lsn, + ) -> anyhow::Result { let threshold = self.get_image_creation_threshold(); - let layers = self.layers.read().unwrap(); + let guard = self.layers.read().await; + let (layers, _) = &*guard; let mut max_deltas = 0; { @@ -3020,13 +3347,14 @@ impl Timeline { for partition in partitioning.parts.iter() { let img_range = start..partition.ranges.last().unwrap().end; start = img_range.end; - if force || self.time_for_new_image_layer(partition, lsn)? { + if force || self.time_for_new_image_layer(partition, lsn).await? { let mut image_layer_writer = ImageLayerWriter::new( self.conf, self.timeline_id, self.tenant_id, &img_range, lsn, + false, // image layer always covers the full range )?; fail_point!("image-layer-writer-fail-before-finish", |_| { @@ -3090,17 +3418,23 @@ impl Timeline { let all_paths = image_layers .iter() .map(|layer| layer.path()) - .chain(std::iter::once( - self.conf.timeline_path(&self.timeline_id, &self.tenant_id), - )) .collect::>(); - par_fsync::par_fsync(&all_paths).context("fsync of newly created layer files")?; + + par_fsync::par_fsync_async(&all_paths) + .await + .context("fsync of newly created layer files")?; + + par_fsync::par_fsync_async(&[self.conf.timeline_path(&self.timeline_id, &self.tenant_id)]) + .await + .context("fsync of timeline dir")?; let mut layer_paths_to_upload = HashMap::with_capacity(image_layers.len()); - let mut layers = self.layers.write().unwrap(); + let mut guard = self.layers.write().await; + let (layers, mapping) = &mut *guard; let mut updates = layers.batch_update(); let timeline_path = self.conf.timeline_path(&self.timeline_id, &self.tenant_id); + for l in image_layers { let path = l.filename(); let metadata = timeline_path @@ -3119,10 +3453,10 @@ impl Timeline { LayerResidenceStatus::Resident, LayerResidenceEventReason::LayerCreate, ); - updates.insert_historic(l); + self.insert_historic_layer(l, &mut updates, mapping); } updates.flush(); - drop(layers); + drop_wlock(guard); timer.stop_and_record(); Ok(layer_paths_to_upload) @@ -3132,7 +3466,7 @@ impl Timeline { #[derive(Default)] struct CompactLevel0Phase1Result { new_layers: Vec, - deltas_to_compact: Vec>, + deltas_to_compact: Vec>, } /// Top-level failure to compact. @@ -3153,22 +3487,150 @@ impl From for CompactionError { } } +#[serde_as] +#[derive(serde::Serialize)] +struct RecordedDuration(#[serde_as(as = "serde_with::DurationMicroSeconds")] Duration); + +#[derive(Default)] +enum DurationRecorder { + #[default] + NotStarted, + Recorded(RecordedDuration, tokio::time::Instant), +} + +impl DurationRecorder { + pub fn till_now(&self) -> DurationRecorder { + match self { + DurationRecorder::NotStarted => { + panic!("must only call on recorded measurements") + } + DurationRecorder::Recorded(_, ended) => { + let now = tokio::time::Instant::now(); + DurationRecorder::Recorded(RecordedDuration(now - *ended), now) + } + } + } + pub fn into_recorded(self) -> Option { + match self { + DurationRecorder::NotStarted => None, + DurationRecorder::Recorded(recorded, _) => Some(recorded), + } + } +} + +#[derive(Default)] +struct CompactLevel0Phase1StatsBuilder { + version: Option, + tenant_id: Option, + timeline_id: Option, + read_lock_acquisition_micros: DurationRecorder, + read_lock_held_spawn_blocking_startup_micros: DurationRecorder, + read_lock_held_prerequisites_micros: DurationRecorder, + read_lock_held_compute_holes_micros: DurationRecorder, + read_lock_drop_micros: DurationRecorder, + prepare_iterators_micros: DurationRecorder, + write_layer_files_micros: DurationRecorder, + level0_deltas_count: Option, + new_deltas_count: Option, + new_deltas_size: Option, +} + +#[serde_as] +#[derive(serde::Serialize)] +struct CompactLevel0Phase1Stats { + version: u64, + #[serde_as(as = "serde_with::DisplayFromStr")] + tenant_id: TenantId, + #[serde_as(as = "serde_with::DisplayFromStr")] + timeline_id: TimelineId, + read_lock_acquisition_micros: RecordedDuration, + read_lock_held_spawn_blocking_startup_micros: RecordedDuration, + read_lock_held_prerequisites_micros: RecordedDuration, + read_lock_held_compute_holes_micros: RecordedDuration, + read_lock_drop_micros: RecordedDuration, + prepare_iterators_micros: RecordedDuration, + write_layer_files_micros: RecordedDuration, + level0_deltas_count: usize, + new_deltas_count: usize, + new_deltas_size: u64, +} + +impl TryFrom for CompactLevel0Phase1Stats { + type Error = anyhow::Error; + + fn try_from(value: CompactLevel0Phase1StatsBuilder) -> Result { + Ok(Self { + version: value.version.ok_or_else(|| anyhow!("version not set"))?, + tenant_id: value + .tenant_id + .ok_or_else(|| anyhow!("tenant_id not set"))?, + timeline_id: value + .timeline_id + .ok_or_else(|| anyhow!("timeline_id not set"))?, + read_lock_acquisition_micros: value + .read_lock_acquisition_micros + .into_recorded() + .ok_or_else(|| anyhow!("read_lock_acquisition_micros not set"))?, + read_lock_held_spawn_blocking_startup_micros: value + .read_lock_held_spawn_blocking_startup_micros + .into_recorded() + .ok_or_else(|| anyhow!("read_lock_held_spawn_blocking_startup_micros not set"))?, + read_lock_held_prerequisites_micros: value + .read_lock_held_prerequisites_micros + .into_recorded() + .ok_or_else(|| anyhow!("read_lock_held_prerequisites_micros not set"))?, + read_lock_held_compute_holes_micros: value + .read_lock_held_compute_holes_micros + .into_recorded() + .ok_or_else(|| anyhow!("read_lock_held_compute_holes_micros not set"))?, + read_lock_drop_micros: value + .read_lock_drop_micros + .into_recorded() + .ok_or_else(|| anyhow!("read_lock_drop_micros not set"))?, + prepare_iterators_micros: value + .prepare_iterators_micros + .into_recorded() + .ok_or_else(|| anyhow!("prepare_iterators_micros not set"))?, + write_layer_files_micros: value + .write_layer_files_micros + .into_recorded() + .ok_or_else(|| anyhow!("write_layer_files_micros not set"))?, + level0_deltas_count: value + .level0_deltas_count + .ok_or_else(|| anyhow!("level0_deltas_count not set"))?, + new_deltas_count: value + .new_deltas_count + .ok_or_else(|| anyhow!("new_deltas_count not set"))?, + new_deltas_size: value + .new_deltas_size + .ok_or_else(|| anyhow!("new_deltas_size not set"))?, + }) + } +} + impl Timeline { /// Level0 files first phase of compaction, explained in the [`compact_inner`] comment. /// /// This method takes the `_layer_removal_cs` guard to highlight it required downloads are /// returned as an error. If the `layer_removal_cs` boundary is changed not to be taken in the /// start of level0 files compaction, the on-demand download should be revisited as well. - async fn compact_level0_phase1( - &self, - _layer_removal_cs: &tokio::sync::MutexGuard<'_, ()>, + fn compact_level0_phase1( + self: Arc, + _layer_removal_cs: Arc>, + guard: tokio::sync::OwnedRwLockReadGuard<(LayerMap, LayerFileManager)>, + mut stats: CompactLevel0Phase1StatsBuilder, target_file_size: u64, ctx: &RequestContext, ) -> Result { - let layers = self.layers.read().unwrap(); - let mut level0_deltas = layers.get_level0_deltas()?; - drop(layers); - + stats.read_lock_held_spawn_blocking_startup_micros = + stats.read_lock_acquisition_micros.till_now(); // set by caller + let (layers, mapping) = &*guard; + let level0_deltas = layers.get_level0_deltas()?; + let mut level0_deltas = level0_deltas + .into_iter() + .map(|x| mapping.get_from_desc(&x)) + .collect_vec(); + stats.level0_deltas_count = Some(level0_deltas.len()); // Only compact if enough layers have accumulated. let threshold = self.get_compaction_threshold(); if level0_deltas.is_empty() || level0_deltas.len() < threshold { @@ -3246,6 +3708,53 @@ impl Timeline { // we don't accidentally use it later in the function. drop(level0_deltas); + stats.read_lock_held_prerequisites_micros = stats + .read_lock_held_spawn_blocking_startup_micros + .till_now(); + + // Determine N largest holes where N is number of compacted layers. + let max_holes = deltas_to_compact.len(); + let last_record_lsn = self.get_last_record_lsn(); + let min_hole_range = (target_file_size / page_cache::PAGE_SZ as u64) as i128; + let min_hole_coverage_size = 3; // TODO: something more flexible? + + // min-heap (reserve space for one more element added before eviction) + let mut heap: BinaryHeap = BinaryHeap::with_capacity(max_holes + 1); + let mut prev: Option = None; + for (next_key, _next_lsn, _size) in itertools::process_results( + deltas_to_compact.iter().map(|l| l.key_iter(ctx)), + |iter_iter| iter_iter.kmerge_by(|a, b| a.0 <= b.0), + )? { + if let Some(prev_key) = prev { + // just first fast filter + if next_key.to_i128() - prev_key.to_i128() >= min_hole_range { + let key_range = prev_key..next_key; + // Measuring hole by just subtraction of i128 representation of key range boundaries + // has not so much sense, because largest holes will corresponds field1/field2 changes. + // But we are mostly interested to eliminate holes which cause generation of excessive image layers. + // That is why it is better to measure size of hole as number of covering image layers. + let coverage_size = layers.image_coverage(&key_range, last_record_lsn)?.len(); + if coverage_size >= min_hole_coverage_size { + heap.push(Hole { + key_range, + coverage_size, + }); + if heap.len() > max_holes { + heap.pop(); // remove smallest hole + } + } + } + } + prev = Some(next_key.next()); + } + stats.read_lock_held_compute_holes_micros = + stats.read_lock_held_prerequisites_micros.till_now(); + drop_rlock(guard); + stats.read_lock_drop_micros = stats.read_lock_held_compute_holes_micros.till_now(); + let mut holes = heap.into_vec(); + holes.sort_unstable_by_key(|hole| hole.key_range.start); + let mut next_hole = 0; // index of next hole in holes vector + // This iterator walks through all key-value pairs from all the layers // we're compacting, in key, LSN order. let all_values_iter = itertools::process_results( @@ -3285,46 +3794,7 @@ impl Timeline { }, )?; - // Determine N largest holes where N is number of compacted layers. - let max_holes = deltas_to_compact.len(); - let last_record_lsn = self.get_last_record_lsn(); - let layers = self.layers.read().unwrap(); // Is'n it better to hold original layers lock till here? - let min_hole_range = (target_file_size / page_cache::PAGE_SZ as u64) as i128; - let min_hole_coverage_size = 3; // TODO: something more flexible? - - // min-heap (reserve space for one more element added before eviction) - let mut heap: BinaryHeap = BinaryHeap::with_capacity(max_holes + 1); - let mut prev: Option = None; - for (next_key, _next_lsn, _size) in itertools::process_results( - deltas_to_compact.iter().map(|l| l.key_iter(ctx)), - |iter_iter| iter_iter.kmerge_by(|a, b| a.0 <= b.0), - )? { - if let Some(prev_key) = prev { - // just first fast filter - if next_key.to_i128() - prev_key.to_i128() >= min_hole_range { - let key_range = prev_key..next_key; - // Measuring hole by just subtraction of i128 representation of key range boundaries - // has not so much sense, because largest holes will corresponds field1/field2 changes. - // But we are mostly interested to eliminate holes which cause generation of excessive image layers. - // That is why it is better to measure size of hole as number of covering image layers. - let coverage_size = layers.image_coverage(&key_range, last_record_lsn)?.len(); - if coverage_size >= min_hole_coverage_size { - heap.push(Hole { - key_range, - coverage_size, - }); - if heap.len() > max_holes { - heap.pop(); // remove smallest hole - } - } - } - } - prev = Some(next_key.next()); - } - drop(layers); - let mut holes = heap.into_vec(); - holes.sort_unstable_by_key(|hole| hole.key_range.start); - let mut next_hole = 0; // index of next hole in holes vector + stats.prepare_iterators_micros = stats.read_lock_drop_micros.till_now(); // Merge the contents of all the input delta layers into a new set // of delta layers, based on the current partitioning. @@ -3474,21 +3944,42 @@ impl Timeline { if !new_layers.is_empty() { let mut layer_paths: Vec = new_layers.iter().map(|l| l.path()).collect(); - // also sync the directory - layer_paths.push(self.conf.timeline_path(&self.timeline_id, &self.tenant_id)); - // Fsync all the layer files and directory using multiple threads to // minimize latency. par_fsync::par_fsync(&layer_paths).context("fsync all new layers")?; + par_fsync::par_fsync(&[self.conf.timeline_path(&self.timeline_id, &self.tenant_id)]) + .context("fsync of timeline dir")?; + layer_paths.pop().unwrap(); } + stats.write_layer_files_micros = stats.prepare_iterators_micros.till_now(); + stats.new_deltas_count = Some(new_layers.len()); + stats.new_deltas_size = Some(new_layers.iter().map(|l| l.desc.file_size).sum()); + drop(all_keys_iter); // So that deltas_to_compact is no longer borrowed + match TryInto::::try_into(stats) + .and_then(|stats| serde_json::to_string(&stats).context("serde_json::to_string")) + { + Ok(stats_json) => { + info!( + stats_json = stats_json.as_str(), + "compact_level0_phase1 stats available" + ) + } + Err(e) => { + warn!("compact_level0_phase1 stats failed to serialize: {:#}", e); + } + } + Ok(CompactLevel0Phase1Result { new_layers, - deltas_to_compact, + deltas_to_compact: deltas_to_compact + .into_iter() + .map(|x| Arc::new(x.layer_desc().clone())) + .collect(), }) } @@ -3497,17 +3988,44 @@ impl Timeline { /// as Level 1 files. /// async fn compact_level0( - &self, - layer_removal_cs: &tokio::sync::MutexGuard<'_, ()>, + self: &Arc, + layer_removal_cs: Arc>, target_file_size: u64, ctx: &RequestContext, ) -> Result<(), CompactionError> { let CompactLevel0Phase1Result { new_layers, deltas_to_compact, - } = self - .compact_level0_phase1(layer_removal_cs, target_file_size, ctx) - .await?; + } = { + let phase1_span = info_span!("compact_level0_phase1"); + let myself = Arc::clone(self); + let ctx = ctx.attached_child(); // technically, the spawn_blocking can outlive this future + let mut stats = CompactLevel0Phase1StatsBuilder { + version: Some(2), + tenant_id: Some(self.tenant_id), + timeline_id: Some(self.timeline_id), + ..Default::default() + }; + + let begin = tokio::time::Instant::now(); + let phase1_layers_locked = Arc::clone(&self.layers).read_owned().await; + let now = tokio::time::Instant::now(); + stats.read_lock_acquisition_micros = + DurationRecorder::Recorded(RecordedDuration(now - begin), now); + let layer_removal_cs = layer_removal_cs.clone(); + tokio::task::spawn_blocking(move || { + let _entered = phase1_span.enter(); + myself.compact_level0_phase1( + layer_removal_cs, + phase1_layers_locked, + stats, + target_file_size, + &ctx, + ) + }) + .await + .context("spawn_blocking")?? + }; if new_layers.is_empty() && deltas_to_compact.is_empty() { // nothing to do @@ -3525,7 +4043,8 @@ impl Timeline { .context("wait for layer upload ops to complete")?; } - let mut layers = self.layers.write().unwrap(); + let mut guard = self.layers.write().await; + let (layers, mapping) = &mut *guard; let mut updates = layers.batch_update(); let mut new_layer_paths = HashMap::with_capacity(new_layers.len()); for l in new_layers { @@ -3557,7 +4076,7 @@ impl Timeline { LayerResidenceStatus::Resident, LayerResidenceEventReason::LayerCreate, ); - updates.insert_historic(x); + self.insert_historic_layer(x, &mut updates, mapping); } // Now that we have reshuffled the data to set of new delta layers, we can @@ -3565,10 +4084,13 @@ impl Timeline { let mut layer_names_to_delete = Vec::with_capacity(deltas_to_compact.len()); for l in deltas_to_compact { layer_names_to_delete.push(l.filename()); - self.delete_historic_layer(layer_removal_cs, l, &mut updates)?; + // NB: the layer file identified by descriptor `l` is guaranteed to be present + // in the LayerFileManager because we kept holding `layer_removal_cs` the entire + // time, even though we dropped `Timeline::layers` inbetween. + self.delete_historic_layer(layer_removal_cs.clone(), l, &mut updates, mapping)?; } updates.flush(); - drop(layers); + drop_wlock(guard); // Also schedule the deletions in remote storage if let Some(remote_client) = &self.remote_client { @@ -3612,6 +4134,7 @@ impl Timeline { /// for example. The caller should hold `Tenant::gc_cs` lock to ensure /// that. /// + #[instrument(skip_all, fields(timeline_id=%self.timeline_id))] pub(super) async fn update_gc_info( &self, retain_lsns: Vec, @@ -3685,10 +4208,9 @@ impl Timeline { fail_point!("before-timeline-gc"); - let layer_removal_cs = self.layer_removal_cs.lock().await; + let layer_removal_cs = Arc::new(self.layer_removal_cs.clone().lock_owned().await); // Is the timeline being deleted? - let state = *self.state.borrow(); - if state == TimelineState::Stopping { + if self.is_stopping() { anyhow::bail!("timeline is Stopping"); } @@ -3705,7 +4227,7 @@ impl Timeline { let res = self .gc_timeline( - &layer_removal_cs, + layer_removal_cs.clone(), horizon_cutoff, pitr_cutoff, retain_lsns, @@ -3724,7 +4246,7 @@ impl Timeline { async fn gc_timeline( &self, - layer_removal_cs: &tokio::sync::MutexGuard<'_, ()>, + layer_removal_cs: Arc>, horizon_cutoff: Lsn, pitr_cutoff: Lsn, retain_lsns: Vec, @@ -3785,7 +4307,8 @@ impl Timeline { // 4. newer on-disk image layers cover the layer's whole key range // // TODO holding a write lock is too agressive and avoidable - let mut layers = self.layers.write().unwrap(); + let mut guard = self.layers.write().await; + let (layers, mapping) = &mut *guard; 'outer: for l in layers.iter_historic_layers() { result.layers_total += 1; @@ -3864,7 +4387,7 @@ impl Timeline { // delta layers. Image layers can form "stairs" preventing old image from been deleted. // But image layers are in any case less sparse than delta layers. Also we need some // protection from replacing recent image layers with new one after each GC iteration. - if l.is_incremental() && !LayerMap::is_l0(&*l) { + if self.get_gc_feedback() && l.is_incremental() && !LayerMap::is_l0(&l) { wanted_image_layers.add_range(l.get_key_range()); } result.layers_not_updated += 1; @@ -3897,7 +4420,12 @@ impl Timeline { { for doomed_layer in layers_to_remove { layer_names_to_delete.push(doomed_layer.filename()); - self.delete_historic_layer(layer_removal_cs, doomed_layer, &mut updates)?; // FIXME: schedule succeeded deletions before returning? + self.delete_historic_layer( + layer_removal_cs.clone(), + doomed_layer, + &mut updates, + mapping, + )?; // FIXME: schedule succeeded deletions before returning? result.layers_removed += 1; } } @@ -4069,7 +4597,7 @@ impl Timeline { // Does retries + exponential back-off internally. // When this fails, don't layer further retry attempts here. let result = remote_client - .download_layer_file(&remote_layer.file_name, &remote_layer.layer_metadata) + .download_layer_file(&remote_layer.filename(), &remote_layer.layer_metadata) .await; if let Ok(size) = &result { @@ -4081,42 +4609,15 @@ impl Timeline { // Download complete. Replace the RemoteLayer with the corresponding // Delta- or ImageLayer in the layer map. - let mut layers = self_clone.layers.write().unwrap(); - let mut updates = layers.batch_update(); - let new_layer = remote_layer.create_downloaded_layer(&updates, self_clone.conf, *size); + let mut guard = self_clone.layers.write().await; + let (layers, mapping) = &mut *guard; + let updates = layers.batch_update(); + let new_layer = + remote_layer.create_downloaded_layer(&updates, self_clone.conf, *size); { - use crate::tenant::layer_map::Replacement; let l: Arc = remote_layer.clone(); - let failure = match updates.replace_historic(&l, new_layer) { - Ok(Replacement::Replaced { .. }) => false, - Ok(Replacement::NotFound) => { - // TODO: the downloaded file should probably be removed, otherwise - // it will be added to the layermap on next load? we should - // probably restart any get_reconstruct_data search as well. - // - // See: https://github.com/neondatabase/neon/issues/3533 - error!("replacing downloaded layer into layermap failed because layer was not found"); - true - } - Ok(Replacement::RemovalBuffered) => { - unreachable!("current implementation does not remove anything") - } - Ok(Replacement::Unexpected(other)) => { - // if the other layer would have the same pointer value as - // expected, it means they differ only on vtables. - // - // otherwise there's no known reason for this to happen as - // compacted layers should have different covering rectangle - // leading to produce Replacement::NotFound. - - error!( - expected.ptr = ?Arc::as_ptr(&l), - other.ptr = ?Arc::as_ptr(&other), - ?other, - "replacing downloaded layer into layermap failed because another layer was found instead of expected" - ); - true - } + let failure = match mapping.replace_and_verify(l, new_layer) { + Ok(()) => false, Err(e) => { // this is a precondition failure, the layer filename derived // attributes didn't match up, which doesn't seem likely. @@ -4144,7 +4645,7 @@ impl Timeline { } } updates.flush(); - drop(layers); + drop_wlock(guard); info!("on-demand download successful"); @@ -4155,7 +4656,10 @@ impl Timeline { remote_layer.ongoing_download.close(); } else { // Keep semaphore open. We'll drop the permit at the end of the function. - error!("layer file download failed: {:?}", result.as_ref().unwrap_err()); + error!( + "layer file download failed: {:?}", + result.as_ref().unwrap_err() + ); } // Don't treat it as an error if the task that triggered the download @@ -4169,7 +4673,8 @@ impl Timeline { drop(permit); Ok(()) - }.in_current_span(), + } + .in_current_span(), ); receiver.await.context("download task cancelled")? @@ -4239,9 +4744,11 @@ impl Timeline { ) { let mut downloads = Vec::new(); { - let layers = self.layers.read().unwrap(); + let guard = self.layers.read().await; + let (layers, mapping) = &*guard; layers .iter_historic_layers() + .map(|l| mapping.get_from_desc(&l)) .filter_map(|l| l.downcast_remote_layer()) .map(|l| self.download_remote_layer(l)) .for_each(|dl| downloads.push(dl)) @@ -4341,8 +4848,9 @@ impl LocalLayerInfoForDiskUsageEviction { } impl Timeline { - pub(crate) fn get_local_layers_for_disk_usage_eviction(&self) -> DiskUsageEvictionInfo { - let layers = self.layers.read().unwrap(); + pub(crate) async fn get_local_layers_for_disk_usage_eviction(&self) -> DiskUsageEvictionInfo { + let guard = self.layers.read().await; + let (layers, mapping) = &*guard; let mut max_layer_size: Option = None; let mut resident_layers = Vec::new(); @@ -4351,6 +4859,8 @@ impl Timeline { let file_size = l.file_size(); max_layer_size = max_layer_size.map_or(Some(file_size), |m| Some(m.max(file_size))); + let l = mapping.get_from_desc(&l); + if l.is_remote_layer() { continue; } @@ -4414,7 +4924,7 @@ fn layer_traversal_error(msg: String, path: Vec) -> PageRecon // but will cause large code changes. pub struct TimelineWriter<'a> { tl: &'a Timeline, - _write_guard: MutexGuard<'a, ()>, + _write_guard: tokio::sync::MutexGuard<'a, ()>, } impl Deref for TimelineWriter<'_> { @@ -4430,12 +4940,12 @@ impl<'a> TimelineWriter<'a> { /// /// This will implicitly extend the relation, if the page is beyond the /// current end-of-file. - pub fn put(&self, key: Key, lsn: Lsn, value: &Value) -> anyhow::Result<()> { - self.tl.put_value(key, lsn, value) + pub async fn put(&self, key: Key, lsn: Lsn, value: &Value) -> anyhow::Result<()> { + self.tl.put_value(key, lsn, value).await } - pub fn delete(&self, key_range: Range, lsn: Lsn) -> anyhow::Result<()> { - self.tl.put_tombstone(key_range, lsn) + pub async fn delete(&self, key_range: Range, lsn: Lsn) -> anyhow::Result<()> { + self.tl.put_tombstone(key_range, lsn).await } /// Track the end of the latest digested WAL record. @@ -4455,6 +4965,14 @@ impl<'a> TimelineWriter<'a> { } } +// We need TimelineWriter to be send in upcoming conversion of +// Timeline::layers to tokio::sync::RwLock. +#[test] +fn is_send() { + fn _assert_send() {} + _assert_send::>(); +} + /// Add a suffix to a layer file's name: .{num}.old /// Uses the first available num (starts at 0) fn rename_to_backup(path: &Path) -> anyhow::Result<()> { @@ -4501,3 +5019,31 @@ pub(crate) fn debug_assert_current_span_has_tenant_and_timeline_id() { ), } } + +/// Similar to `Arc::ptr_eq`, but only compares the object pointers, not vtables. +/// +/// Returns `true` if the two `Arc` point to the same layer, false otherwise. +/// +/// If comparing persistent layers, ALWAYS compare the layer descriptor key. +#[inline(always)] +pub fn compare_arced_layers(left: &Arc, right: &Arc) -> bool { + // "dyn Trait" objects are "fat pointers" in that they have two components: + // - pointer to the object + // - pointer to the vtable + // + // rust does not provide a guarantee that these vtables are unique, but however + // `Arc::ptr_eq` as of writing (at least up to 1.67) uses a comparison where both the + // pointer and the vtable need to be equal. + // + // See: https://github.com/rust-lang/rust/issues/103763 + // + // A future version of rust will most likely use this form below, where we cast each + // pointer into a pointer to unit, which drops the inaccessible vtable pointer, making it + // not affect the comparison. + // + // See: https://github.com/rust-lang/rust/pull/106450 + let left = Arc::as_ptr(left) as *const (); + let right = Arc::as_ptr(right) as *const (); + + left == right +} diff --git a/pageserver/src/tenant/timeline/eviction_task.rs b/pageserver/src/tenant/timeline/eviction_task.rs index 558600692e..03cf2d89ad 100644 --- a/pageserver/src/tenant/timeline/eviction_task.rs +++ b/pageserver/src/tenant/timeline/eviction_task.rs @@ -34,6 +34,8 @@ use crate::{ }, }; +use utils::completion; + use super::Timeline; #[derive(Default)] @@ -47,8 +49,12 @@ pub struct EvictionTaskTenantState { } impl Timeline { - pub(super) fn launch_eviction_task(self: &Arc) { + pub(super) fn launch_eviction_task( + self: &Arc, + background_tasks_can_start: Option<&completion::Barrier>, + ) { let self_clone = Arc::clone(self); + let background_tasks_can_start = background_tasks_can_start.cloned(); task_mgr::spawn( BACKGROUND_RUNTIME.handle(), TaskKind::Eviction, @@ -57,7 +63,13 @@ impl Timeline { &format!("layer eviction for {}/{}", self.tenant_id, self.timeline_id), false, async move { - self_clone.eviction_task(task_mgr::shutdown_token()).await; + let cancel = task_mgr::shutdown_token(); + tokio::select! { + _ = cancel.cancelled() => { return Ok(()); } + _ = completion::Barrier::maybe_wait(background_tasks_can_start) => {} + }; + + self_clone.eviction_task(cancel).await; info!("eviction task finishing"); Ok(()) }, @@ -185,9 +197,11 @@ impl Timeline { // We don't want to hold the layer map lock during eviction. // So, we just need to deal with this. let candidates: Vec> = { - let layers = self.layers.read().unwrap(); + let guard = self.layers.read().await; + let (layers, mapping) = &*guard; let mut candidates = Vec::new(); for hist_layer in layers.iter_historic_layers() { + let hist_layer = mapping.get_from_desc(&hist_layer); if hist_layer.is_remote_layer() { continue; } diff --git a/pageserver/src/tenant/timeline/walreceiver.rs b/pageserver/src/tenant/timeline/walreceiver.rs index 7ebf3cf172..ccff735c3c 100644 --- a/pageserver/src/tenant/timeline/walreceiver.rs +++ b/pageserver/src/tenant/timeline/walreceiver.rs @@ -25,6 +25,7 @@ mod walreceiver_connection; use crate::context::{DownloadBehavior, RequestContext}; use crate::task_mgr::{self, TaskKind, WALRECEIVER_RUNTIME}; +use crate::tenant::debug_assert_current_span_has_tenant_and_timeline_id; use crate::tenant::timeline::walreceiver::connection_manager::{ connection_manager_loop_step, ConnectionManagerState, }; @@ -85,7 +86,8 @@ impl WalReceiver { &format!("walreceiver for timeline {tenant_id}/{timeline_id}"), false, async move { - info!("WAL receiver manager started, connecting to broker"); + debug_assert_current_span_has_tenant_and_timeline_id(); + debug!("WAL receiver manager started, connecting to broker"); let mut connection_manager_state = ConnectionManagerState::new( timeline, conf, @@ -93,7 +95,7 @@ impl WalReceiver { loop { select! { _ = task_mgr::shutdown_watcher() => { - info!("WAL receiver shutdown requested, shutting down"); + trace!("WAL receiver shutdown requested, shutting down"); break; }, loop_step_result = connection_manager_loop_step( @@ -104,7 +106,7 @@ impl WalReceiver { ) => match loop_step_result { ControlFlow::Continue(()) => continue, ControlFlow::Break(()) => { - info!("Connection manager loop ended, shutting down"); + trace!("Connection manager loop ended, shutting down"); break; } }, @@ -115,7 +117,7 @@ impl WalReceiver { *loop_status.write().unwrap() = None; Ok(()) } - .instrument(info_span!(parent: None, "wal_connection_manager", tenant = %tenant_id, timeline = %timeline_id)) + .instrument(info_span!(parent: None, "wal_connection_manager", tenant_id = %tenant_id, timeline_id = %timeline_id)) ); Self { @@ -198,29 +200,19 @@ impl TaskHandle { TaskEvent::End(match self.join_handle.as_mut() { Some(jh) => { if !jh.is_finished() { - // Barring any implementation errors in this module, we can - // only arrive here while the task that executes the future - // passed to `Self::spawn()` is still execution. Cf the comment - // in Self::spawn(). - // - // This was logging at warning level in earlier versions, presumably - // to leave some breadcrumbs in case we had an implementation - // error that would would make us get stuck in `jh.await`. - // - // There hasn't been such a bug so far. - // But in a busy system, e.g., during pageserver restart, - // we arrive here often enough that the warning-level logs - // became a distraction. - // So, tone them down to info-level. - // - // XXX: rewrite this module to eliminate the race condition. - info!("sender is dropped while join handle is still alive"); + // See: https://github.com/neondatabase/neon/issues/2885 + trace!("sender is dropped while join handle is still alive"); } - let res = jh - .await - .map_err(|e| anyhow::anyhow!("Failed to join task: {e}")) - .and_then(|x| x); + let res = match jh.await { + Ok(res) => res, + Err(je) if je.is_cancelled() => unreachable!("not used"), + Err(je) if je.is_panic() => { + // already logged + Ok(()) + } + Err(je) => Err(anyhow::Error::new(je).context("join walreceiver task")), + }; // For cancellation-safety, drop join_handle only after successful .await. self.join_handle = None; @@ -243,12 +235,12 @@ impl TaskHandle { match jh.await { Ok(Ok(())) => debug!("Shutdown success"), Ok(Err(e)) => error!("Shutdown task error: {e:?}"), - Err(join_error) => { - if join_error.is_cancelled() { - error!("Shutdown task was cancelled"); - } else { - error!("Shutdown task join error: {join_error}") - } + Err(je) if je.is_cancelled() => unreachable!("not used"), + Err(je) if je.is_panic() => { + // already logged + } + Err(je) => { + error!("Shutdown task join error: {je}") } } } diff --git a/pageserver/src/tenant/timeline/walreceiver/connection_manager.rs b/pageserver/src/tenant/timeline/walreceiver/connection_manager.rs index 6b65e1fd42..fa23ae765d 100644 --- a/pageserver/src/tenant/timeline/walreceiver/connection_manager.rs +++ b/pageserver/src/tenant/timeline/walreceiver/connection_manager.rs @@ -18,7 +18,7 @@ use crate::metrics::{ WALRECEIVER_CANDIDATES_REMOVED, WALRECEIVER_SWITCHES, }; use crate::task_mgr::TaskKind; -use crate::tenant::Timeline; +use crate::tenant::{debug_assert_current_span_has_tenant_and_timeline_id, Timeline}; use anyhow::Context; use chrono::{NaiveDateTime, Utc}; use pageserver_api::models::TimelineState; @@ -55,8 +55,11 @@ pub(super) async fn connection_manager_loop_step( .await { Ok(()) => {} - Err(_) => { - info!("Timeline dropped state updates sender before becoming active, stopping wal connection manager loop"); + Err(new_state) => { + debug!( + ?new_state, + "state changed, stopping wal connection manager loop" + ); return ControlFlow::Break(()); } } @@ -79,7 +82,7 @@ pub(super) async fn connection_manager_loop_step( // with other streams on this client (other connection managers). When // object goes out of scope, stream finishes in drop() automatically. let mut broker_subscription = subscribe_for_timeline_updates(broker_client, id).await; - info!("Subscribed for broker timeline updates"); + debug!("Subscribed for broker timeline updates"); loop { let time_until_next_retry = connection_manager_state.time_until_next_retry(); @@ -150,13 +153,13 @@ pub(super) async fn connection_manager_loop_step( match new_state { // we're already active as walreceiver, no need to reactivate TimelineState::Active => continue, - TimelineState::Broken | TimelineState::Stopping => { - info!("timeline entered terminal state {new_state:?}, stopping wal connection manager loop"); + TimelineState::Broken { .. } | TimelineState::Stopping => { + debug!("timeline entered terminal state {new_state:?}, stopping wal connection manager loop"); return ControlFlow::Break(()); } TimelineState::Loading => { warn!("timeline transitioned back to Loading state, that should not happen"); - return ControlFlow::Continue(new_state); + return ControlFlow::Continue(()); } } } @@ -164,12 +167,11 @@ pub(super) async fn connection_manager_loop_step( } } } => match new_event { - ControlFlow::Continue(new_state) => { - info!("observed timeline state change, new state is {new_state:?}"); + ControlFlow::Continue(()) => { return ControlFlow::Continue(()); } ControlFlow::Break(()) => { - info!("Timeline dropped state updates sender, stopping wal connection manager loop"); + debug!("Timeline is no longer active, stopping wal connection manager loop"); return ControlFlow::Break(()); } }, @@ -390,7 +392,6 @@ impl ConnectionManagerState { self.drop_old_connection(true).await; - let id = self.id; let node_id = new_sk.safekeeper_id; let connect_timeout = self.conf.wal_connect_timeout; let timeline = Arc::clone(&self.timeline); @@ -398,9 +399,13 @@ impl ConnectionManagerState { TaskKind::WalReceiverConnectionHandler, DownloadBehavior::Download, ); + + let span = info_span!("connection", %node_id); let connection_handle = TaskHandle::spawn(move |events_sender, cancellation| { async move { - super::walreceiver_connection::handle_walreceiver_connection( + debug_assert_current_span_has_tenant_and_timeline_id(); + + let res = super::walreceiver_connection::handle_walreceiver_connection( timeline, new_sk.wal_source_connconf, events_sender, @@ -409,12 +414,23 @@ impl ConnectionManagerState { ctx, node_id, ) - .await - .context("walreceiver connection handling failure") + .await; + + match res { + Ok(()) => Ok(()), + Err(e) => { + use super::walreceiver_connection::ExpectedError; + if e.is_expected() { + info!("walreceiver connection handling ended: {e:#}"); + Ok(()) + } else { + // give out an error to have task_mgr give it a really verbose logging + Err(e).context("walreceiver connection handling failure") + } + } + } } - .instrument( - info_span!("walreceiver_connection", tenant_id = %id.tenant_id, timeline_id = %id.timeline_id, %node_id), - ) + .instrument(span) }); let now = Utc::now().naive_utc(); @@ -1305,10 +1321,11 @@ mod tests { const DUMMY_SAFEKEEPER_HOST: &str = "safekeeper_connstr"; - async fn dummy_state(harness: &TenantHarness<'_>) -> ConnectionManagerState { + async fn dummy_state(harness: &TenantHarness) -> ConnectionManagerState { let (tenant, ctx) = harness.load().await; let timeline = tenant - .create_test_timeline(TIMELINE_ID, Lsn(0), crate::DEFAULT_PG_VERSION, &ctx) + .create_test_timeline(TIMELINE_ID, Lsn(0x8), crate::DEFAULT_PG_VERSION, &ctx) + .await .expect("Failed to create an empty timeline for dummy wal connection manager"); ConnectionManagerState { diff --git a/pageserver/src/tenant/timeline/walreceiver/walreceiver_connection.rs b/pageserver/src/tenant/timeline/walreceiver/walreceiver_connection.rs index 1cbed3416c..1c1fe87305 100644 --- a/pageserver/src/tenant/timeline/walreceiver/walreceiver_connection.rs +++ b/pageserver/src/tenant/timeline/walreceiver/walreceiver_connection.rs @@ -21,16 +21,16 @@ use postgres_types::PgLsn; use tokio::{select, sync::watch, time}; use tokio_postgres::{replication::ReplicationStream, Client}; use tokio_util::sync::CancellationToken; -use tracing::{debug, error, info, trace, warn}; +use tracing::{debug, error, info, trace, warn, Instrument}; use super::TaskStateUpdate; -use crate::metrics::LIVE_CONNECTIONS_COUNT; -use crate::{context::RequestContext, metrics::WALRECEIVER_STARTED_CONNECTIONS}; use crate::{ + context::RequestContext, + metrics::{LIVE_CONNECTIONS_COUNT, WALRECEIVER_STARTED_CONNECTIONS}, task_mgr, task_mgr::TaskKind, task_mgr::WALRECEIVER_RUNTIME, - tenant::{Timeline, WalReceiverInfo}, + tenant::{debug_assert_current_span_has_tenant_and_timeline_id, Timeline, WalReceiverInfo}, walingest::WalIngest, walrecord::DecodedWALRecord, }; @@ -81,13 +81,8 @@ pub(super) async fn handle_walreceiver_connection( config.application_name("pageserver"); config.replication_mode(tokio_postgres::config::ReplicationMode::Physical); match time::timeout(connect_timeout, config.connect(postgres::NoTls)).await { - Ok(Ok(client_and_conn)) => client_and_conn, - Ok(Err(conn_err)) => { - let expected_error = ignore_expected_errors(conn_err)?; - info!("DB connection stream finished: {expected_error}"); - return Ok(()); - } - Err(_) => { + Ok(client_and_conn) => client_and_conn?, + Err(_elapsed) => { // Timing out to connect to a safekeeper node could happen long time, due to // many reasons that pageserver cannot control. // Do not produce an error, but make it visible, that timeouts happen by logging the `event. @@ -97,7 +92,7 @@ pub(super) async fn handle_walreceiver_connection( } }; - info!("connected!"); + debug!("connected!"); let mut connection_status = WalConnectionStatus { is_connected: true, has_processed_wal: false, @@ -127,20 +122,25 @@ pub(super) async fn handle_walreceiver_connection( "walreceiver connection", false, async move { + debug_assert_current_span_has_tenant_and_timeline_id(); + select! { connection_result = connection => match connection_result { - Ok(()) => info!("Walreceiver db connection closed"), + Ok(()) => debug!("Walreceiver db connection closed"), Err(connection_error) => { - if let Err(e) = ignore_expected_errors(connection_error) { - warn!("Connection aborted: {e:#}") + if connection_error.is_expected() { + // silence, because most likely we've already exited the outer call + // with a similar error. + } else { + warn!("Connection aborted: {connection_error:#}") } } }, - // Future: replace connection_cancellation with connection_ctx cancellation - _ = connection_cancellation.cancelled() => info!("Connection cancelled"), + _ = connection_cancellation.cancelled() => debug!("Connection cancelled"), } Ok(()) - }, + } + .instrument(tracing::info_span!("poller")), ); // Immediately increment the gauge, then create a job to decrement it on task exit. @@ -203,20 +203,13 @@ pub(super) async fn handle_walreceiver_connection( while let Some(replication_message) = { select! { _ = cancellation.cancelled() => { - info!("walreceiver interrupted"); + debug!("walreceiver interrupted"); None } replication_message = physical_stream.next() => replication_message, } } { - let replication_message = match replication_message { - Ok(message) => message, - Err(replication_error) => { - let expected_error = ignore_expected_errors(replication_error)?; - info!("Replication stream finished: {expected_error}"); - return Ok(()); - } - }; + let replication_message = replication_message?; let now = Utc::now().naive_utc(); let last_rec_lsn_before_msg = last_rec_lsn; @@ -261,8 +254,6 @@ pub(super) async fn handle_walreceiver_connection( let mut decoded = DecodedWALRecord::default(); let mut modification = timeline.begin_modification(endlsn); while let Some((lsn, recdata)) = waldecoder.poll_decode()? { - // let _enter = info_span!("processing record", lsn = %lsn).entered(); - // It is important to deal with the aligned records as lsn in getPage@LSN is // aligned and can be several bytes bigger. Without this alignment we are // at risk of hitting a deadlock. @@ -313,12 +304,15 @@ pub(super) async fn handle_walreceiver_connection( } } - timeline.check_checkpoint_distance().with_context(|| { - format!( - "Failed to check checkpoint distance for timeline {}", - timeline.timeline_id - ) - })?; + timeline + .check_checkpoint_distance() + .await + .with_context(|| { + format!( + "Failed to check checkpoint distance for timeline {}", + timeline.timeline_id + ) + })?; if let Some(last_lsn) = status_update { let timeline_remote_consistent_lsn = @@ -421,31 +415,50 @@ async fn identify_system(client: &mut Client) -> anyhow::Result } } -/// We don't want to report connectivity problems as real errors towards connection manager because -/// 1. they happen frequently enough to make server logs hard to read and -/// 2. the connection manager can retry other safekeeper. -/// -/// If this function returns `Ok(pg_error)`, it's such an error. -/// The caller should log it at info level and then report to connection manager that we're done handling this connection. -/// Connection manager will then handle reconnections. -/// -/// If this function returns an `Err()`, the caller can bubble it up using `?`. -/// The connection manager will log the error at ERROR level. -fn ignore_expected_errors(pg_error: postgres::Error) -> anyhow::Result { - if pg_error.is_closed() - || pg_error - .source() - .and_then(|source| source.downcast_ref::()) - .map(is_expected_io_error) - .unwrap_or(false) - { - return Ok(pg_error); - } else if let Some(db_error) = pg_error.as_db_error() { - if db_error.code() == &SqlState::SUCCESSFUL_COMPLETION - && db_error.message().contains("ending streaming") - { - return Ok(pg_error); - } - } - Err(pg_error).context("connection error") +/// Trait for avoid reporting walreceiver specific expected or "normal" or "ok" errors. +pub(super) trait ExpectedError { + /// Test if this error is an ok error. + /// + /// We don't want to report connectivity problems as real errors towards connection manager because + /// 1. they happen frequently enough to make server logs hard to read and + /// 2. the connection manager can retry other safekeeper. + /// + /// If this function returns `true`, it's such an error. + /// The caller should log it at info level and then report to connection manager that we're done handling this connection. + /// Connection manager will then handle reconnections. + /// + /// If this function returns an `false` the error should be propagated and the connection manager + /// will log the error at ERROR level. + fn is_expected(&self) -> bool; +} + +impl ExpectedError for postgres::Error { + fn is_expected(&self) -> bool { + self.is_closed() + || self + .source() + .and_then(|source| source.downcast_ref::()) + .map(is_expected_io_error) + .unwrap_or(false) + || self + .as_db_error() + .filter(|db_error| { + db_error.code() == &SqlState::SUCCESSFUL_COMPLETION + && db_error.message().contains("ending streaming") + }) + .is_some() + } +} + +impl ExpectedError for anyhow::Error { + fn is_expected(&self) -> bool { + let head = self.downcast_ref::(); + + let tail = self + .chain() + .filter_map(|e| e.downcast_ref::()); + + // check if self or any of the chained/sourced errors are expected + head.into_iter().chain(tail).any(|e| e.is_expected()) + } } diff --git a/pageserver/src/tenant/upload_queue.rs b/pageserver/src/tenant/upload_queue.rs index 8f5faff627..a62cc99adf 100644 --- a/pageserver/src/tenant/upload_queue.rs +++ b/pageserver/src/tenant/upload_queue.rs @@ -76,6 +76,12 @@ pub(crate) struct UploadQueueInitialized { pub(crate) queued_operations: VecDeque, } +impl UploadQueueInitialized { + pub(super) fn no_pending_work(&self) -> bool { + self.inprogress_tasks.is_empty() && self.queued_operations.is_empty() + } +} + #[derive(Clone, Copy)] pub(super) enum SetDeletedFlagProgress { NotRunning, @@ -84,9 +90,7 @@ pub(super) enum SetDeletedFlagProgress { } pub(super) struct UploadQueueStopped { - pub(super) latest_files: HashMap, - pub(super) last_uploaded_consistent_lsn: Lsn, - pub(super) latest_metadata: TimelineMetadata, + pub(super) upload_queue_for_deletion: UploadQueueInitialized, pub(super) deleted_at: SetDeletedFlagProgress, } @@ -187,6 +191,15 @@ impl UploadQueue { UploadQueue::Initialized(x) => Ok(x), } } + + pub(crate) fn stopped_mut(&mut self) -> anyhow::Result<&mut UploadQueueStopped> { + match self { + UploadQueue::Initialized(_) | UploadQueue::Uninitialized => { + anyhow::bail!("queue is in state {}", self.as_str()) + } + UploadQueue::Stopped(stopped) => Ok(stopped), + } + } } /// An in-progress upload or delete task. @@ -199,6 +212,13 @@ pub(crate) struct UploadTask { pub(crate) op: UploadOp, } +#[derive(Debug)] +pub(crate) struct Delete { + pub(crate) file_kind: RemoteOpFileKind, + pub(crate) layer_file_name: LayerFileName, + pub(crate) scheduled_from_timeline_delete: bool, +} + #[derive(Debug)] pub(crate) enum UploadOp { /// Upload a layer file @@ -207,8 +227,8 @@ pub(crate) enum UploadOp { /// Upload the metadata file UploadMetadata(IndexPart, Lsn), - /// Delete a file. - Delete(RemoteOpFileKind, LayerFileName), + /// Delete a layer file + Delete(Delete), /// Barrier. When the barrier operation is reached, Barrier(tokio::sync::watch::Sender<()>), @@ -226,7 +246,12 @@ impl std::fmt::Display for UploadOp { ) } UploadOp::UploadMetadata(_, lsn) => write!(f, "UploadMetadata(lsn: {})", lsn), - UploadOp::Delete(_, path) => write!(f, "Delete({})", path.file_name()), + UploadOp::Delete(delete) => write!( + f, + "Delete(path: {}, scheduled_from_timeline_delete: {})", + delete.layer_file_name.file_name(), + delete.scheduled_from_timeline_delete + ), UploadOp::Barrier(_) => write!(f, "Barrier"), } } diff --git a/pageserver/src/walingest.rs b/pageserver/src/walingest.rs index 4b8e6aa515..8d4c1842bd 100644 --- a/pageserver/src/walingest.rs +++ b/pageserver/src/walingest.rs @@ -25,7 +25,7 @@ use postgres_ffi::v14::nonrelfile_utils::clogpage_precedes; use postgres_ffi::v14::nonrelfile_utils::slru_may_delete_clogsegment; use postgres_ffi::{fsm_logical_to_physical, page_is_new, page_set_lsn}; -use anyhow::Result; +use anyhow::{Context, Result}; use bytes::{Buf, Bytes, BytesMut}; use tracing::*; @@ -333,7 +333,7 @@ impl<'a> WalIngest<'a> { // Now that this record has been fully handled, including updating the // checkpoint data, let the repository know that it is up-to-date to this LSN - modification.commit()?; + modification.commit().await?; Ok(()) } @@ -1082,7 +1082,10 @@ impl<'a> WalIngest<'a> { .await? { // create it with 0 size initially, the logic below will extend it - modification.put_rel_creation(rel, 0, ctx).await?; + modification + .put_rel_creation(rel, 0, ctx) + .await + .context("Relation Error")?; 0 } else { self.timeline.get_rel_size(rel, last_lsn, true, ctx).await? @@ -1171,7 +1174,6 @@ impl<'a> WalIngest<'a> { #[cfg(test)] mod tests { use super::*; - use crate::pgdatadir_mapping::create_test_timeline; use crate::tenant::harness::*; use crate::tenant::Timeline; use postgres_ffi::v14::xlog_utils::SIZEOF_CHECKPOINT; @@ -1200,7 +1202,7 @@ mod tests { let mut m = tline.begin_modification(Lsn(0x10)); m.put_checkpoint(ZERO_CHECKPOINT.clone())?; m.put_relmap_file(0, 111, Bytes::from(""), ctx).await?; // dummy relmapper file - m.commit()?; + m.commit().await?; let walingest = WalIngest::new(tline, Lsn(0x10), ctx).await?; Ok(walingest) @@ -1209,7 +1211,9 @@ mod tests { #[tokio::test] async fn test_relsize() -> Result<()> { let (tenant, ctx) = TenantHarness::create("test_relsize")?.load().await; - let tline = create_test_timeline(&tenant, TIMELINE_ID, DEFAULT_PG_VERSION, &ctx)?; + let tline = tenant + .create_test_timeline(TIMELINE_ID, Lsn(8), DEFAULT_PG_VERSION, &ctx) + .await?; let mut walingest = init_walingest_test(&tline, &ctx).await?; let mut m = tline.begin_modification(Lsn(0x20)); @@ -1217,22 +1221,22 @@ mod tests { walingest .put_rel_page_image(&mut m, TESTREL_A, 0, TEST_IMG("foo blk 0 at 2"), &ctx) .await?; - m.commit()?; + m.commit().await?; let mut m = tline.begin_modification(Lsn(0x30)); walingest .put_rel_page_image(&mut m, TESTREL_A, 0, TEST_IMG("foo blk 0 at 3"), &ctx) .await?; - m.commit()?; + m.commit().await?; let mut m = tline.begin_modification(Lsn(0x40)); walingest .put_rel_page_image(&mut m, TESTREL_A, 1, TEST_IMG("foo blk 1 at 4"), &ctx) .await?; - m.commit()?; + m.commit().await?; let mut m = tline.begin_modification(Lsn(0x50)); walingest .put_rel_page_image(&mut m, TESTREL_A, 2, TEST_IMG("foo blk 2 at 5"), &ctx) .await?; - m.commit()?; + m.commit().await?; assert_current_logical_size(&tline, Lsn(0x50)); @@ -1318,7 +1322,7 @@ mod tests { walingest .put_rel_truncation(&mut m, TESTREL_A, 2, &ctx) .await?; - m.commit()?; + m.commit().await?; assert_current_logical_size(&tline, Lsn(0x60)); // Check reported size and contents after truncation @@ -1360,7 +1364,7 @@ mod tests { walingest .put_rel_truncation(&mut m, TESTREL_A, 0, &ctx) .await?; - m.commit()?; + m.commit().await?; assert_eq!( tline .get_rel_size(TESTREL_A, Lsn(0x68), false, &ctx) @@ -1373,7 +1377,7 @@ mod tests { walingest .put_rel_page_image(&mut m, TESTREL_A, 1, TEST_IMG("foo blk 1"), &ctx) .await?; - m.commit()?; + m.commit().await?; assert_eq!( tline .get_rel_size(TESTREL_A, Lsn(0x70), false, &ctx) @@ -1398,7 +1402,7 @@ mod tests { walingest .put_rel_page_image(&mut m, TESTREL_A, 1500, TEST_IMG("foo blk 1500"), &ctx) .await?; - m.commit()?; + m.commit().await?; assert_eq!( tline .get_rel_size(TESTREL_A, Lsn(0x80), false, &ctx) @@ -1428,14 +1432,16 @@ mod tests { #[tokio::test] async fn test_drop_extend() -> Result<()> { let (tenant, ctx) = TenantHarness::create("test_drop_extend")?.load().await; - let tline = create_test_timeline(&tenant, TIMELINE_ID, DEFAULT_PG_VERSION, &ctx)?; + let tline = tenant + .create_test_timeline(TIMELINE_ID, Lsn(8), DEFAULT_PG_VERSION, &ctx) + .await?; let mut walingest = init_walingest_test(&tline, &ctx).await?; let mut m = tline.begin_modification(Lsn(0x20)); walingest .put_rel_page_image(&mut m, TESTREL_A, 0, TEST_IMG("foo blk 0 at 2"), &ctx) .await?; - m.commit()?; + m.commit().await?; // Check that rel exists and size is correct assert_eq!( @@ -1454,7 +1460,7 @@ mod tests { // Drop rel let mut m = tline.begin_modification(Lsn(0x30)); walingest.put_rel_drop(&mut m, TESTREL_A, &ctx).await?; - m.commit()?; + m.commit().await?; // Check that rel is not visible anymore assert_eq!( @@ -1472,7 +1478,7 @@ mod tests { walingest .put_rel_page_image(&mut m, TESTREL_A, 0, TEST_IMG("foo blk 0 at 4"), &ctx) .await?; - m.commit()?; + m.commit().await?; // Check that rel exists and size is correct assert_eq!( @@ -1497,7 +1503,9 @@ mod tests { #[tokio::test] async fn test_truncate_extend() -> Result<()> { let (tenant, ctx) = TenantHarness::create("test_truncate_extend")?.load().await; - let tline = create_test_timeline(&tenant, TIMELINE_ID, DEFAULT_PG_VERSION, &ctx)?; + let tline = tenant + .create_test_timeline(TIMELINE_ID, Lsn(8), DEFAULT_PG_VERSION, &ctx) + .await?; let mut walingest = init_walingest_test(&tline, &ctx).await?; // Create a 20 MB relation (the size is arbitrary) @@ -1509,7 +1517,7 @@ mod tests { .put_rel_page_image(&mut m, TESTREL_A, blkno, TEST_IMG(&data), &ctx) .await?; } - m.commit()?; + m.commit().await?; // The relation was created at LSN 20, not visible at LSN 1 yet. assert_eq!( @@ -1554,7 +1562,7 @@ mod tests { walingest .put_rel_truncation(&mut m, TESTREL_A, 1, &ctx) .await?; - m.commit()?; + m.commit().await?; // Check reported size and contents after truncation assert_eq!( @@ -1603,7 +1611,7 @@ mod tests { .put_rel_page_image(&mut m, TESTREL_A, blkno, TEST_IMG(&data), &ctx) .await?; } - m.commit()?; + m.commit().await?; assert_eq!( tline @@ -1637,7 +1645,9 @@ mod tests { #[tokio::test] async fn test_large_rel() -> Result<()> { let (tenant, ctx) = TenantHarness::create("test_large_rel")?.load().await; - let tline = create_test_timeline(&tenant, TIMELINE_ID, DEFAULT_PG_VERSION, &ctx)?; + let tline = tenant + .create_test_timeline(TIMELINE_ID, Lsn(8), DEFAULT_PG_VERSION, &ctx) + .await?; let mut walingest = init_walingest_test(&tline, &ctx).await?; let mut lsn = 0x10; @@ -1648,7 +1658,7 @@ mod tests { walingest .put_rel_page_image(&mut m, TESTREL_A, blknum as BlockNumber, img, &ctx) .await?; - m.commit()?; + m.commit().await?; } assert_current_logical_size(&tline, Lsn(lsn)); @@ -1664,7 +1674,7 @@ mod tests { walingest .put_rel_truncation(&mut m, TESTREL_A, RELSEG_SIZE, &ctx) .await?; - m.commit()?; + m.commit().await?; assert_eq!( tline.get_rel_size(TESTREL_A, Lsn(lsn), false, &ctx).await?, RELSEG_SIZE @@ -1677,7 +1687,7 @@ mod tests { walingest .put_rel_truncation(&mut m, TESTREL_A, RELSEG_SIZE - 1, &ctx) .await?; - m.commit()?; + m.commit().await?; assert_eq!( tline.get_rel_size(TESTREL_A, Lsn(lsn), false, &ctx).await?, RELSEG_SIZE - 1 @@ -1693,7 +1703,7 @@ mod tests { walingest .put_rel_truncation(&mut m, TESTREL_A, size as BlockNumber, &ctx) .await?; - m.commit()?; + m.commit().await?; assert_eq!( tline.get_rel_size(TESTREL_A, Lsn(lsn), false, &ctx).await?, size as BlockNumber diff --git a/pgxn/hnsw/Makefile b/pgxn/hnsw/Makefile new file mode 100644 index 0000000000..66436b5920 --- /dev/null +++ b/pgxn/hnsw/Makefile @@ -0,0 +1,26 @@ +EXTENSION = hnsw +EXTVERSION = 0.1.0 + +MODULE_big = hnsw +DATA = $(wildcard *--*.sql) +OBJS = hnsw.o hnswalg.o + +TESTS = $(wildcard test/sql/*.sql) +REGRESS = $(patsubst test/sql/%.sql,%,$(TESTS)) +REGRESS_OPTS = --inputdir=test --load-extension=hnsw + +# For auto-vectorization: +# - GCC (needs -ftree-vectorize OR -O3) - https://gcc.gnu.org/projects/tree-ssa/vectorization.html +PG_CFLAGS += -O3 +PG_CXXFLAGS += -O3 -std=c++11 +PG_LDFLAGS += -lstdc++ + +all: $(EXTENSION)--$(EXTVERSION).sql + +PG_CONFIG ?= pg_config +PGXS := $(shell $(PG_CONFIG) --pgxs) +include $(PGXS) + +dist: + mkdir -p dist + git archive --format zip --prefix=$(EXTENSION)-$(EXTVERSION)/ --output dist/$(EXTENSION)-$(EXTVERSION).zip master diff --git a/pgxn/hnsw/README.md b/pgxn/hnsw/README.md new file mode 100644 index 0000000000..bc9c8d571c --- /dev/null +++ b/pgxn/hnsw/README.md @@ -0,0 +1,25 @@ +# Revisiting the Inverted Indices for Billion-Scale Approximate Nearest Neighbors + +This ANN extension of Postgres is based +on [ivf-hnsw](https://github.com/dbaranchuk/ivf-hnsw.git) implementation of [HNSW](https://www.pinecone.io/learn/hnsw), +the code for the current state-of-the-art billion-scale nearest neighbor search system presented in the paper: + +[Revisiting the Inverted Indices for Billion-Scale Approximate Nearest Neighbors](http://openaccess.thecvf.com/content_ECCV_2018/html/Dmitry_Baranchuk_Revisiting_the_Inverted_ECCV_2018_paper.html), +
+Dmitry Baranchuk, Artem Babenko, Yury Malkov + +# Postgres extension + +HNSW index is hold in memory (built on demand) and it's maxial size is limited +by `maxelements` index parameter. Another required parameter is nubmer of dimensions (if it is not specified in column type). +Optional parameter `ef` specifies number of neighbors which are considered during index construction and search (corresponds `efConstruction` and `efSearch` parameters +described in the article). + +# Example of usage: + +``` +create extension hnsw; +create table embeddings(id integer primary key, payload real[]); +create index on embeddings using hnsw(payload) with (maxelements=1000000, dims=100, m=32); +select id from embeddings order by payload <-> array[1.0, 2.0,...] limit 100; +``` \ No newline at end of file diff --git a/pgxn/hnsw/hnsw--0.1.0.sql b/pgxn/hnsw/hnsw--0.1.0.sql new file mode 100644 index 0000000000..ebf424326d --- /dev/null +++ b/pgxn/hnsw/hnsw--0.1.0.sql @@ -0,0 +1,29 @@ +-- complain if script is sourced in psql, rather than via CREATE EXTENSION +\echo Use "CREATE EXTENSION hnsw" to load this file. \quit + +-- functions + +CREATE FUNCTION l2_distance(real[], real[]) RETURNS real + AS 'MODULE_PATHNAME' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE; + +-- operators + +CREATE OPERATOR <-> ( + LEFTARG = real[], RIGHTARG = real[], PROCEDURE = l2_distance, + COMMUTATOR = '<->' +); + +-- access method + +CREATE FUNCTION hnsw_handler(internal) RETURNS index_am_handler + AS 'MODULE_PATHNAME' LANGUAGE C; + +CREATE ACCESS METHOD hnsw TYPE INDEX HANDLER hnsw_handler; + +COMMENT ON ACCESS METHOD hnsw IS 'hnsw index access method'; + +-- opclasses + +CREATE OPERATOR CLASS knn_ops + DEFAULT FOR TYPE real[] USING hnsw AS + OPERATOR 1 <-> (real[], real[]) FOR ORDER BY float_ops; diff --git a/pgxn/hnsw/hnsw.c b/pgxn/hnsw/hnsw.c new file mode 100644 index 0000000000..434f4986f8 --- /dev/null +++ b/pgxn/hnsw/hnsw.c @@ -0,0 +1,551 @@ +#include "postgres.h" + +#include "access/amapi.h" +#include "access/generic_xlog.h" +#include "access/relation.h" +#include "access/reloptions.h" +#include "access/tableam.h" +#include "catalog/index.h" +#include "commands/vacuum.h" +#include "nodes/execnodes.h" +#include "storage/bufmgr.h" +#include "utils/guc.h" +#include "utils/selfuncs.h" + +#include +#include + +#include "hnsw.h" + +PG_MODULE_MAGIC; + +typedef struct { + int32 vl_len_; /* varlena header (do not touch directly!) */ + int dims; + int maxelements; + int efConstruction; + int efSearch; + int M; +} HnswOptions; + +static relopt_kind hnsw_relopt_kind; + +typedef struct { + HierarchicalNSW* hnsw; + size_t curr; + size_t n_results; + ItemPointer results; +} HnswScanOpaqueData; + +typedef HnswScanOpaqueData* HnswScanOpaque; + +typedef struct { + Oid relid; + uint32 status; + HierarchicalNSW* hnsw; +} HnswHashEntry; + + +#define SH_PREFIX hnsw_index +#define SH_ELEMENT_TYPE HnswHashEntry +#define SH_KEY_TYPE Oid +#define SH_KEY relid +#define SH_STORE_HASH +#define SH_GET_HASH(tb, a) ((a)->relid) +#define SH_HASH_KEY(tb, key) (key) +#define SH_EQUAL(tb, a, b) ((a) == (b)) +#define SH_SCOPE static inline +#define SH_DEFINE +#define SH_DECLARE +#include "lib/simplehash.h" + +#define INDEX_HASH_SIZE 11 + +#define DEFAULT_EF_SEARCH 64 + +PGDLLEXPORT void _PG_init(void); + +static hnsw_index_hash *hnsw_indexes; + +/* + * Initialize index options and variables + */ +void +_PG_init(void) +{ + hnsw_relopt_kind = add_reloption_kind(); + add_int_reloption(hnsw_relopt_kind, "dims", "Number of dimensions", + 0, 0, INT_MAX, AccessExclusiveLock); + add_int_reloption(hnsw_relopt_kind, "maxelements", "Maximal number of elements", + 0, 0, INT_MAX, AccessExclusiveLock); + add_int_reloption(hnsw_relopt_kind, "m", "Number of neighbors of each vertex", + 100, 0, INT_MAX, AccessExclusiveLock); + add_int_reloption(hnsw_relopt_kind, "efconstruction", "Number of inspected neighbors during index construction", + 16, 1, INT_MAX, AccessExclusiveLock); + add_int_reloption(hnsw_relopt_kind, "efsearch", "Number of inspected neighbors during index search", + 64, 1, INT_MAX, AccessExclusiveLock); + hnsw_indexes = hnsw_index_create(TopMemoryContext, INDEX_HASH_SIZE, NULL); +} + + +static void +hnsw_build_callback(Relation index, ItemPointer tid, Datum *values, + bool *isnull, bool tupleIsAlive, void *state) +{ + HierarchicalNSW* hnsw = (HierarchicalNSW*) state; + ArrayType* array; + int n_items; + label_t label = 0; + + /* Skip nulls */ + if (isnull[0]) + return; + + array = DatumGetArrayTypeP(values[0]); + n_items = ArrayGetNItems(ARR_NDIM(array), ARR_DIMS(array)); + if (n_items != hnsw_dimensions(hnsw)) + { + elog(ERROR, "Wrong number of dimensions: %d instead of %d expected", + n_items, hnsw_dimensions(hnsw)); + } + + memcpy(&label, tid, sizeof(*tid)); + hnsw_add_point(hnsw, (coord_t*)ARR_DATA_PTR(array), label); +} + +static void +hnsw_populate(HierarchicalNSW* hnsw, Relation indexRel, Relation heapRel) +{ + IndexInfo* indexInfo = BuildIndexInfo(indexRel); + Assert(indexInfo->ii_NumIndexAttrs == 1); + table_index_build_scan(heapRel, indexRel, indexInfo, + true, true, hnsw_build_callback, (void *) hnsw, NULL); +} + +static HierarchicalNSW* +hnsw_get_index(Relation indexRel, Relation heapRel) +{ + HierarchicalNSW* hnsw; + Oid indexoid = RelationGetRelid(indexRel); + HnswHashEntry* entry = hnsw_index_lookup(hnsw_indexes, indexoid); + if (entry == NULL) + { + size_t dims, maxelements; + size_t M; + size_t maxM; + size_t size_links_level0; + size_t size_data_per_element; + size_t data_size; + dsm_handle handle = indexoid << 1; /* make it even */ + void* impl_private = NULL; + void* mapped_address = NULL; + Size mapped_size = 0; + Size shmem_size; + bool exists = true; + bool found; + HnswOptions *opts = (HnswOptions *) indexRel->rd_options; + if (opts == NULL || opts->maxelements == 0 || opts->dims == 0) { + elog(ERROR, "HNSW index requires 'maxelements' and 'dims' to be specified"); + } + dims = opts->dims; + maxelements = opts->maxelements; + M = opts->M; + maxM = M * 2; + data_size = dims * sizeof(coord_t); + size_links_level0 = (maxM + 1) * sizeof(idx_t); + size_data_per_element = size_links_level0 + data_size + sizeof(label_t); + shmem_size = hnsw_sizeof() + maxelements * size_data_per_element; + + /* first try to attach to existed index */ + if (!dsm_impl_op(DSM_OP_ATTACH, handle, 0, &impl_private, + &mapped_address, &mapped_size, DEBUG1)) + { + /* index doesn't exists: try to create it */ + if (!dsm_impl_op(DSM_OP_CREATE, handle, shmem_size, &impl_private, + &mapped_address, &mapped_size, DEBUG1)) + { + /* We can do it under shared lock, so some other backend may + * try to initialize index. If create is failed because index already + * created by somebody else, then try to attach to it once again + */ + if (!dsm_impl_op(DSM_OP_ATTACH, handle, 0, &impl_private, + &mapped_address, &mapped_size, ERROR)) + { + return NULL; + } + } + else + { + exists = false; + } + } + Assert(mapped_size == shmem_size); + hnsw = (HierarchicalNSW*)mapped_address; + + if (!exists) + { + hnsw_init(hnsw, dims, maxelements, M, maxM, opts->efConstruction); + hnsw_populate(hnsw, indexRel, heapRel); + } + entry = hnsw_index_insert(hnsw_indexes, indexoid, &found); + Assert(!found); + entry->hnsw = hnsw; + } + else + { + hnsw = entry->hnsw; + } + return hnsw; +} + +/* + * Start or restart an index scan + */ +static IndexScanDesc +hnsw_beginscan(Relation index, int nkeys, int norderbys) +{ + IndexScanDesc scan = RelationGetIndexScan(index, nkeys, norderbys); + HnswScanOpaque so = (HnswScanOpaque) palloc(sizeof(HnswScanOpaqueData)); + Relation heap = relation_open(index->rd_index->indrelid, NoLock); + so->hnsw = hnsw_get_index(index, heap); + relation_close(heap, NoLock); + so->curr = 0; + so->n_results = 0; + so->results = NULL; + scan->opaque = so; + return scan; +} + +/* + * Start or restart an index scan + */ +static void +hnsw_rescan(IndexScanDesc scan, ScanKey keys, int nkeys, ScanKey orderbys, int norderbys) +{ + HnswScanOpaque so = (HnswScanOpaque) scan->opaque; + if (so->results) + { + pfree(so->results); + so->results = NULL; + } + so->curr = 0; + if (orderbys && scan->numberOfOrderBys > 0) + memmove(scan->orderByData, orderbys, scan->numberOfOrderBys * sizeof(ScanKeyData)); +} + +/* + * Fetch the next tuple in the given scan + */ +static bool +hnsw_gettuple(IndexScanDesc scan, ScanDirection dir) +{ + HnswScanOpaque so = (HnswScanOpaque) scan->opaque; + + /* + * Index can be used to scan backward, but Postgres doesn't support + * backward scan on operators + */ + Assert(ScanDirectionIsForward(dir)); + + if (so->curr == 0) + { + Datum value; + ArrayType* array; + int n_items; + size_t n_results; + label_t* results; + HnswOptions *opts = (HnswOptions *) scan->indexRelation->rd_options; + size_t efSearch = opts ? opts->efSearch : DEFAULT_EF_SEARCH; + + /* Safety check */ + if (scan->orderByData == NULL) + elog(ERROR, "cannot scan HNSW index without order"); + + /* No items will match if null */ + if (scan->orderByData->sk_flags & SK_ISNULL) + return false; + + value = scan->orderByData->sk_argument; + array = DatumGetArrayTypeP(value); + n_items = ArrayGetNItems(ARR_NDIM(array), ARR_DIMS(array)); + if (n_items != hnsw_dimensions(so->hnsw)) + { + elog(ERROR, "Wrong number of dimensions: %d instead of %d expected", + n_items, hnsw_dimensions(so->hnsw)); + } + + if (!hnsw_search(so->hnsw, (coord_t*)ARR_DATA_PTR(array), efSearch, &n_results, &results)) + elog(ERROR, "HNSW index search failed"); + so->results = (ItemPointer)palloc(n_results*sizeof(ItemPointerData)); + so->n_results = n_results; + for (size_t i = 0; i < n_results; i++) + { + memcpy(&so->results[i], &results[i], sizeof(so->results[i])); + } + free(results); + } + if (so->curr >= so->n_results) + { + return false; + } + else + { + scan->xs_heaptid = so->results[so->curr++]; + scan->xs_recheckorderby = false; + return true; + } +} + +/* + * End a scan and release resources + */ +static void +hnsw_endscan(IndexScanDesc scan) +{ + HnswScanOpaque so = (HnswScanOpaque) scan->opaque; + if (so->results) + pfree(so->results); + pfree(so); + scan->opaque = NULL; +} + + +/* + * Estimate the cost of an index scan + */ +static void +hnsw_costestimate(PlannerInfo *root, IndexPath *path, double loop_count, + Cost *indexStartupCost, Cost *indexTotalCost, + Selectivity *indexSelectivity, double *indexCorrelation + ,double *indexPages +) +{ + GenericCosts costs; + + /* Never use index without order */ + if (path->indexorderbys == NULL) + { + *indexStartupCost = DBL_MAX; + *indexTotalCost = DBL_MAX; + *indexSelectivity = 0; + *indexCorrelation = 0; + *indexPages = 0; + return; + } + + MemSet(&costs, 0, sizeof(costs)); + + genericcostestimate(root, path, loop_count, &costs); + + /* Startup cost and total cost are same */ + *indexStartupCost = costs.indexTotalCost; + *indexTotalCost = costs.indexTotalCost; + *indexSelectivity = costs.indexSelectivity; + *indexCorrelation = costs.indexCorrelation; + *indexPages = costs.numIndexPages; +} + +/* + * Parse and validate the reloptions + */ +static bytea * +hnsw_options(Datum reloptions, bool validate) +{ + static const relopt_parse_elt tab[] = { + {"dims", RELOPT_TYPE_INT, offsetof(HnswOptions, dims)}, + {"maxelements", RELOPT_TYPE_INT, offsetof(HnswOptions, maxelements)}, + {"efconstruction", RELOPT_TYPE_INT, offsetof(HnswOptions, efConstruction)}, + {"efsearch", RELOPT_TYPE_INT, offsetof(HnswOptions, efSearch)}, + {"m", RELOPT_TYPE_INT, offsetof(HnswOptions, M)} + }; + + return (bytea *) build_reloptions(reloptions, validate, + hnsw_relopt_kind, + sizeof(HnswOptions), + tab, lengthof(tab)); +} + +/* + * Validate catalog entries for the specified operator class + */ +static bool +hnsw_validate(Oid opclassoid) +{ + return true; +} + +/* + * Build the index for a logged table + */ +static IndexBuildResult * +hnsw_build(Relation heap, Relation index, IndexInfo *indexInfo) +{ + HierarchicalNSW* hnsw = hnsw_get_index(index, heap); + IndexBuildResult* result = (IndexBuildResult *) palloc(sizeof(IndexBuildResult)); + result->heap_tuples = result->index_tuples = hnsw_count(hnsw); + + return result; +} + +/* + * Insert a tuple into the index + */ +static bool +hnsw_insert(Relation index, Datum *values, bool *isnull, ItemPointer heap_tid, + Relation heap, IndexUniqueCheck checkUnique, + bool indexUnchanged, + IndexInfo *indexInfo) +{ + HierarchicalNSW* hnsw = hnsw_get_index(index, heap); + Datum value; + ArrayType* array; + int n_items; + label_t label = 0; + + /* Skip nulls */ + if (isnull[0]) + return false; + + /* Detoast value */ + value = PointerGetDatum(PG_DETOAST_DATUM(values[0])); + array = DatumGetArrayTypeP(value); + n_items = ArrayGetNItems(ARR_NDIM(array), ARR_DIMS(array)); + if (n_items != hnsw_dimensions(hnsw)) + { + elog(ERROR, "Wrong number of dimensions: %d instead of %d expected", + n_items, hnsw_dimensions(hnsw)); + } + memcpy(&label, heap_tid, sizeof(*heap_tid)); + if (!hnsw_add_point(hnsw, (coord_t*)ARR_DATA_PTR(array), label)) + elog(ERROR, "HNSW index insert failed"); + return true; +} + +/* + * Build the index for an unlogged table + */ +static void +hnsw_buildempty(Relation index) +{ + /* index will be constructed on dema nd when accessed */ +} + +/* + * Clean up after a VACUUM operation + */ +static IndexBulkDeleteResult * +hnsw_vacuumcleanup(IndexVacuumInfo *info, IndexBulkDeleteResult *stats) +{ + Relation rel = info->index; + + if (stats == NULL) + return NULL; + + stats->num_pages = RelationGetNumberOfBlocks(rel); + + return stats; +} + +/* + * Bulk delete tuples from the index + */ +static IndexBulkDeleteResult * +hnsw_bulkdelete(IndexVacuumInfo *info, IndexBulkDeleteResult *stats, + IndexBulkDeleteCallback callback, void *callback_state) +{ + if (stats == NULL) + stats = (IndexBulkDeleteResult *) palloc0(sizeof(IndexBulkDeleteResult)); + return stats; +} + +/* + * Define index handler + * + * See https://www.postgresql.org/docs/current/index-api.html + */ +PGDLLEXPORT PG_FUNCTION_INFO_V1(hnsw_handler); +Datum +hnsw_handler(PG_FUNCTION_ARGS) +{ + IndexAmRoutine *amroutine = makeNode(IndexAmRoutine); + + amroutine->amstrategies = 0; + amroutine->amsupport = 0; + amroutine->amoptsprocnum = 0; + amroutine->amcanorder = false; + amroutine->amcanorderbyop = true; + amroutine->amcanbackward = false; /* can change direction mid-scan */ + amroutine->amcanunique = false; + amroutine->amcanmulticol = false; + amroutine->amoptionalkey = true; + amroutine->amsearcharray = false; + amroutine->amsearchnulls = false; + amroutine->amstorage = false; + amroutine->amclusterable = false; + amroutine->ampredlocks = false; + amroutine->amcanparallel = false; + amroutine->amcaninclude = false; + amroutine->amusemaintenanceworkmem = false; /* not used during VACUUM */ + amroutine->amparallelvacuumoptions = VACUUM_OPTION_PARALLEL_BULKDEL; + amroutine->amkeytype = InvalidOid; + + /* Interface functions */ + amroutine->ambuild = hnsw_build; + amroutine->ambuildempty = hnsw_buildempty; + amroutine->aminsert = hnsw_insert; + amroutine->ambulkdelete = hnsw_bulkdelete; + amroutine->amvacuumcleanup = hnsw_vacuumcleanup; + amroutine->amcanreturn = NULL; /* tuple not included in heapsort */ + amroutine->amcostestimate = hnsw_costestimate; + amroutine->amoptions = hnsw_options; + amroutine->amproperty = NULL; /* TODO AMPROP_DISTANCE_ORDERABLE */ + amroutine->ambuildphasename = NULL; + amroutine->amvalidate = hnsw_validate; + amroutine->amadjustmembers = NULL; + amroutine->ambeginscan = hnsw_beginscan; + amroutine->amrescan = hnsw_rescan; + amroutine->amgettuple = hnsw_gettuple; + amroutine->amgetbitmap = NULL; + amroutine->amendscan = hnsw_endscan; + amroutine->ammarkpos = NULL; + amroutine->amrestrpos = NULL; + + /* Interface functions to support parallel index scans */ + amroutine->amestimateparallelscan = NULL; + amroutine->aminitparallelscan = NULL; + amroutine->amparallelrescan = NULL; + + PG_RETURN_POINTER(amroutine); +} + +/* + * Get the L2 distance between vectors + */ +PGDLLEXPORT PG_FUNCTION_INFO_V1(l2_distance); +Datum +l2_distance(PG_FUNCTION_ARGS) +{ + ArrayType *a = PG_GETARG_ARRAYTYPE_P(0); + ArrayType *b = PG_GETARG_ARRAYTYPE_P(1); + int a_dim = ArrayGetNItems(ARR_NDIM(a), ARR_DIMS(a)); + int b_dim = ArrayGetNItems(ARR_NDIM(b), ARR_DIMS(b)); + dist_t distance = 0.0; + dist_t diff; + coord_t *ax = (coord_t*)ARR_DATA_PTR(a); + coord_t *bx = (coord_t*)ARR_DATA_PTR(b); + + if (a_dim != b_dim) + { + ereport(ERROR, + (errcode(ERRCODE_DATA_EXCEPTION), + errmsg("different array dimensions %d and %d", a_dim, b_dim))); + } + + for (int i = 0; i < a_dim; i++) + { + diff = ax[i] - bx[i]; + distance += diff * diff; + } + + PG_RETURN_FLOAT4((dist_t)sqrt(distance)); +} diff --git a/pgxn/hnsw/hnsw.control b/pgxn/hnsw/hnsw.control new file mode 100644 index 0000000000..b292b96026 --- /dev/null +++ b/pgxn/hnsw/hnsw.control @@ -0,0 +1,5 @@ +comment = 'hNsw index' +default_version = '0.1.0' +module_pathname = '$libdir/hnsw' +relocatable = true +trusted = true diff --git a/pgxn/hnsw/hnsw.h b/pgxn/hnsw/hnsw.h new file mode 100644 index 0000000000..d4065ab8fe --- /dev/null +++ b/pgxn/hnsw/hnsw.h @@ -0,0 +1,15 @@ +#pragma once + +typedef float coord_t; +typedef float dist_t; +typedef uint32_t idx_t; +typedef uint64_t label_t; + +typedef struct HierarchicalNSW HierarchicalNSW; + +bool hnsw_search(HierarchicalNSW* hnsw, const coord_t *point, size_t efSearch, size_t* n_results, label_t** results); +bool hnsw_add_point(HierarchicalNSW* hnsw, const coord_t *point, label_t label); +void hnsw_init(HierarchicalNSW* hnsw, size_t dim, size_t maxelements, size_t M, size_t maxM, size_t efConstruction); +int hnsw_dimensions(HierarchicalNSW* hnsw); +size_t hnsw_count(HierarchicalNSW* hnsw); +size_t hnsw_sizeof(void); diff --git a/pgxn/hnsw/hnswalg.cpp b/pgxn/hnsw/hnswalg.cpp new file mode 100644 index 0000000000..f6de3b8314 --- /dev/null +++ b/pgxn/hnsw/hnswalg.cpp @@ -0,0 +1,379 @@ +#include "hnswalg.h" + +#if defined(__GNUC__) +#define PORTABLE_ALIGN32 __attribute__((aligned(32))) +#define PREFETCH(addr,hint) __builtin_prefetch(addr, 0, hint) +#else +#define PORTABLE_ALIGN32 __declspec(align(32)) +#define PREFETCH(addr,hint) +#endif + +HierarchicalNSW::HierarchicalNSW(size_t dim_, size_t maxelements_, size_t M_, size_t maxM_, size_t efConstruction_) +{ + dim = dim_; + data_size = dim * sizeof(coord_t); + + efConstruction = efConstruction_; + + maxelements = maxelements_; + M = M_; + maxM = maxM_; + size_links_level0 = (maxM + 1) * sizeof(idx_t); + size_data_per_element = size_links_level0 + data_size + sizeof(label_t); + offset_data = size_links_level0; + offset_label = offset_data + data_size; + + enterpoint_node = 0; + cur_element_count = 0; +#ifdef __x86_64__ + use_avx2 = __builtin_cpu_supports("avx2"); +#endif +} + +std::priority_queue> HierarchicalNSW::searchBaseLayer(const coord_t *point, size_t ef) +{ + std::vector visited; + visited.resize((cur_element_count + 31) >> 5); + + std::priority_queue> topResults; + std::priority_queue> candidateSet; + + dist_t dist = fstdistfunc(point, getDataByInternalId(enterpoint_node)); + + topResults.emplace(dist, enterpoint_node); + candidateSet.emplace(-dist, enterpoint_node); + visited[enterpoint_node >> 5] = 1 << (enterpoint_node & 31); + dist_t lowerBound = dist; + + while (!candidateSet.empty()) + { + std::pair curr_el_pair = candidateSet.top(); + if (-curr_el_pair.first > lowerBound) + break; + + candidateSet.pop(); + idx_t curNodeNum = curr_el_pair.second; + + idx_t* data = get_linklist0(curNodeNum); + size_t size = *data++; + + PREFETCH(getDataByInternalId(*data), 0); + + for (size_t j = 0; j < size; ++j) { + size_t tnum = *(data + j); + + PREFETCH(getDataByInternalId(*(data + j + 1)), 0); + + if (!(visited[tnum >> 5] & (1 << (tnum & 31)))) { + visited[tnum >> 5] |= 1 << (tnum & 31); + + dist = fstdistfunc(point, getDataByInternalId(tnum)); + + if (topResults.top().first > dist || topResults.size() < ef) { + candidateSet.emplace(-dist, tnum); + + PREFETCH(get_linklist0(candidateSet.top().second), 0); + topResults.emplace(dist, tnum); + + if (topResults.size() > ef) + topResults.pop(); + + lowerBound = topResults.top().first; + } + } + } + } + return topResults; +} + + +void HierarchicalNSW::getNeighborsByHeuristic(std::priority_queue> &topResults, size_t NN) +{ + if (topResults.size() < NN) + return; + + std::priority_queue> resultSet; + std::vector> returnlist; + + while (topResults.size() > 0) { + resultSet.emplace(-topResults.top().first, topResults.top().second); + topResults.pop(); + } + + while (resultSet.size()) { + if (returnlist.size() >= NN) + break; + std::pair curen = resultSet.top(); + dist_t dist_to_query = -curen.first; + resultSet.pop(); + bool good = true; + for (std::pair curen2 : returnlist) { + dist_t curdist = fstdistfunc(getDataByInternalId(curen2.second), + getDataByInternalId(curen.second)); + if (curdist < dist_to_query) { + good = false; + break; + } + } + if (good) returnlist.push_back(curen); + } + for (std::pair elem : returnlist) + topResults.emplace(-elem.first, elem.second); +} + +void HierarchicalNSW::mutuallyConnectNewElement(const coord_t *point, idx_t cur_c, + std::priority_queue> topResults) +{ + getNeighborsByHeuristic(topResults, M); + + std::vector res; + res.reserve(M); + while (topResults.size() > 0) { + res.push_back(topResults.top().second); + topResults.pop(); + } + { + idx_t* data = get_linklist0(cur_c); + if (*data) + throw std::runtime_error("Should be blank"); + + *data++ = res.size(); + + for (size_t idx = 0; idx < res.size(); idx++) { + if (data[idx]) + throw std::runtime_error("Should be blank"); + data[idx] = res[idx]; + } + } + for (size_t idx = 0; idx < res.size(); idx++) { + if (res[idx] == cur_c) + throw std::runtime_error("Connection to the same element"); + + size_t resMmax = maxM; + idx_t *ll_other = get_linklist0(res[idx]); + idx_t sz_link_list_other = *ll_other; + + if (sz_link_list_other > resMmax || sz_link_list_other < 0) + throw std::runtime_error("Bad sz_link_list_other"); + + if (sz_link_list_other < resMmax) { + idx_t *data = ll_other + 1; + data[sz_link_list_other] = cur_c; + *ll_other = sz_link_list_other + 1; + } else { + // finding the "weakest" element to replace it with the new one + idx_t *data = ll_other + 1; + dist_t d_max = fstdistfunc(getDataByInternalId(cur_c), getDataByInternalId(res[idx])); + // Heuristic: + std::priority_queue> candidates; + candidates.emplace(d_max, cur_c); + + for (size_t j = 0; j < sz_link_list_other; j++) + candidates.emplace(fstdistfunc(getDataByInternalId(data[j]), getDataByInternalId(res[idx])), data[j]); + + getNeighborsByHeuristic(candidates, resMmax); + + size_t indx = 0; + while (!candidates.empty()) { + data[indx] = candidates.top().second; + candidates.pop(); + indx++; + } + *ll_other = indx; + } + } +} + +void HierarchicalNSW::addPoint(const coord_t *point, label_t label) +{ + if (cur_element_count >= maxelements) { + throw std::runtime_error("The number of elements exceeds the specified limit"); + } + idx_t cur_c = cur_element_count++; + memset((char *) get_linklist0(cur_c), 0, size_data_per_element); + memcpy(getDataByInternalId(cur_c), point, data_size); + memcpy(getExternalLabel(cur_c), &label, sizeof label); + + // Do nothing for the first element + if (cur_c != 0) { + std::priority_queue > topResults = searchBaseLayer(point, efConstruction); + mutuallyConnectNewElement(point, cur_c, topResults); + } +}; + +std::priority_queue> HierarchicalNSW::searchKnn(const coord_t *query, size_t k) +{ + std::priority_queue> topResults; + auto topCandidates = searchBaseLayer(query, k); + while (topCandidates.size() > k) { + topCandidates.pop(); + } + while (!topCandidates.empty()) { + std::pair rez = topCandidates.top(); + label_t label; + memcpy(&label, getExternalLabel(rez.second), sizeof(label)); + topResults.push(std::pair(rez.first, label)); + topCandidates.pop(); + } + + return topResults; +}; + +dist_t fstdistfunc_scalar(const coord_t *x, const coord_t *y, size_t n) +{ + dist_t distance = 0.0; + + for (size_t i = 0; i < n; i++) + { + dist_t diff = x[i] - y[i]; + distance += diff * diff; + } + return distance; + +} + +#ifdef __x86_64__ +#include + +__attribute__((target("avx2"))) +dist_t fstdistfunc_avx2(const coord_t *x, const coord_t *y, size_t n) +{ + const size_t TmpResSz = sizeof(__m256) / sizeof(float); + float PORTABLE_ALIGN32 TmpRes[TmpResSz]; + size_t qty16 = n / 16; + const float *pEnd1 = x + (qty16 * 16); + __m256 diff, v1, v2; + __m256 sum = _mm256_set1_ps(0); + + while (x < pEnd1) { + v1 = _mm256_loadu_ps(x); + x += 8; + v2 = _mm256_loadu_ps(y); + y += 8; + diff = _mm256_sub_ps(v1, v2); + sum = _mm256_add_ps(sum, _mm256_mul_ps(diff, diff)); + + v1 = _mm256_loadu_ps(x); + x += 8; + v2 = _mm256_loadu_ps(y); + y += 8; + diff = _mm256_sub_ps(v1, v2); + sum = _mm256_add_ps(sum, _mm256_mul_ps(diff, diff)); + } + _mm256_store_ps(TmpRes, sum); + float res = TmpRes[0] + TmpRes[1] + TmpRes[2] + TmpRes[3] + TmpRes[4] + TmpRes[5] + TmpRes[6] + TmpRes[7]; + return (res); +} + +dist_t fstdistfunc_sse(const coord_t *x, const coord_t *y, size_t n) +{ + const size_t TmpResSz = sizeof(__m128) / sizeof(float); + float PORTABLE_ALIGN32 TmpRes[TmpResSz]; + size_t qty16 = n / 16; + const float *pEnd1 = x + (qty16 * 16); + + __m128 diff, v1, v2; + __m128 sum = _mm_set1_ps(0); + + while (x < pEnd1) { + v1 = _mm_loadu_ps(x); + x += 4; + v2 = _mm_loadu_ps(y); + y += 4; + diff = _mm_sub_ps(v1, v2); + sum = _mm_add_ps(sum, _mm_mul_ps(diff, diff)); + + v1 = _mm_loadu_ps(x); + x += 4; + v2 = _mm_loadu_ps(y); + y += 4; + diff = _mm_sub_ps(v1, v2); + sum = _mm_add_ps(sum, _mm_mul_ps(diff, diff)); + + v1 = _mm_loadu_ps(x); + x += 4; + v2 = _mm_loadu_ps(y); + y += 4; + diff = _mm_sub_ps(v1, v2); + sum = _mm_add_ps(sum, _mm_mul_ps(diff, diff)); + + v1 = _mm_loadu_ps(x); + x += 4; + v2 = _mm_loadu_ps(y); + y += 4; + diff = _mm_sub_ps(v1, v2); + sum = _mm_add_ps(sum, _mm_mul_ps(diff, diff)); + } + _mm_store_ps(TmpRes, sum); + float res = TmpRes[0] + TmpRes[1] + TmpRes[2] + TmpRes[3]; + return res; +} +#endif + +dist_t HierarchicalNSW::fstdistfunc(const coord_t *x, const coord_t *y) +{ +#ifndef __x86_64__ + return fstdistfunc_scalar(x, y, dim); +#else + if(use_avx2) + return fstdistfunc_avx2(x, y, dim); + + return fstdistfunc_sse(x, y, dim); +#endif +} + +bool hnsw_search(HierarchicalNSW* hnsw, const coord_t *point, size_t efSearch, size_t* n_results, label_t** results) +{ + try + { + auto result = hnsw->searchKnn(point, efSearch); + size_t nResults = result.size(); + *results = (label_t*)malloc(nResults*sizeof(label_t)); + for (size_t i = nResults; i-- != 0;) + { + (*results)[i] = result.top().second; + result.pop(); + } + *n_results = nResults; + return true; + } + catch (std::exception& x) + { + return false; + } +} + +bool hnsw_add_point(HierarchicalNSW* hnsw, const coord_t *point, label_t label) +{ + try + { + hnsw->addPoint(point, label); + return true; + } + catch (std::exception& x) + { + fprintf(stderr, "Catch %s\n", x.what()); + return false; + } +} + +void hnsw_init(HierarchicalNSW* hnsw, size_t dims, size_t maxelements, size_t M, size_t maxM, size_t efConstruction) +{ + new ((void*)hnsw) HierarchicalNSW(dims, maxelements, M, maxM, efConstruction); +} + + +int hnsw_dimensions(HierarchicalNSW* hnsw) +{ + return (int)hnsw->dim; +} + +size_t hnsw_count(HierarchicalNSW* hnsw) +{ + return hnsw->cur_element_count; +} + +size_t hnsw_sizeof(void) +{ + return sizeof(HierarchicalNSW); +} diff --git a/pgxn/hnsw/hnswalg.h b/pgxn/hnsw/hnswalg.h new file mode 100644 index 0000000000..f38aeac362 --- /dev/null +++ b/pgxn/hnsw/hnswalg.h @@ -0,0 +1,69 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +extern "C" { +#include "hnsw.h" +} + +struct HierarchicalNSW +{ + size_t maxelements; + size_t cur_element_count; + + idx_t enterpoint_node; + + size_t dim; + size_t data_size; + size_t offset_data; + size_t offset_label; + size_t size_data_per_element; + size_t M; + size_t maxM; + size_t size_links_level0; + size_t efConstruction; + +#ifdef __x86_64__ + bool use_avx2; +#endif + + char data_level0_memory[0]; // varying size + + public: + HierarchicalNSW(size_t dim, size_t maxelements, size_t M, size_t maxM, size_t efConstruction); + ~HierarchicalNSW(); + + + inline coord_t *getDataByInternalId(idx_t internal_id) const { + return (coord_t *)&data_level0_memory[internal_id * size_data_per_element + offset_data]; + } + + inline idx_t *get_linklist0(idx_t internal_id) const { + return (idx_t*)&data_level0_memory[internal_id * size_data_per_element]; + } + + inline label_t *getExternalLabel(idx_t internal_id) const { + return (label_t *)&data_level0_memory[internal_id * size_data_per_element + offset_label]; + } + + std::priority_queue> searchBaseLayer(const coord_t *x, size_t ef); + + void getNeighborsByHeuristic(std::priority_queue> &topResults, size_t NN); + + void mutuallyConnectNewElement(const coord_t *x, idx_t id, std::priority_queue> topResults); + + void addPoint(const coord_t *point, label_t label); + + std::priority_queue> searchKnn(const coord_t *query_data, size_t k); + + dist_t fstdistfunc(const coord_t *x, const coord_t *y); +}; diff --git a/pgxn/hnsw/test/expected/knn.out b/pgxn/hnsw/test/expected/knn.out new file mode 100644 index 0000000000..a1cee4525e --- /dev/null +++ b/pgxn/hnsw/test/expected/knn.out @@ -0,0 +1,28 @@ +SET enable_seqscan = off; +CREATE TABLE t (val real[]); +INSERT INTO t (val) VALUES ('{0,0,0}'), ('{1,2,3}'), ('{1,1,1}'), (NULL); +CREATE INDEX ON t USING hnsw (val) WITH (maxelements = 10, dims=3, m=3); +INSERT INTO t (val) VALUES (array[1,2,4]); +explain SELECT * FROM t ORDER BY val <-> array[3,3,3]; + QUERY PLAN +-------------------------------------------------------------------- + Index Scan using t_val_idx on t (cost=4.02..8.06 rows=3 width=36) + Order By: (val <-> '{3,3,3}'::real[]) +(2 rows) + +SELECT * FROM t ORDER BY val <-> array[3,3,3]; + val +--------- + {1,2,3} + {1,2,4} + {1,1,1} + {0,0,0} +(4 rows) + +SELECT COUNT(*) FROM t; + count +------- + 5 +(1 row) + +DROP TABLE t; diff --git a/pgxn/hnsw/test/sql/knn.sql b/pgxn/hnsw/test/sql/knn.sql new file mode 100644 index 0000000000..0635bda4a2 --- /dev/null +++ b/pgxn/hnsw/test/sql/knn.sql @@ -0,0 +1,13 @@ +SET enable_seqscan = off; + +CREATE TABLE t (val real[]); +INSERT INTO t (val) VALUES ('{0,0,0}'), ('{1,2,3}'), ('{1,1,1}'), (NULL); +CREATE INDEX ON t USING hnsw (val) WITH (maxelements = 10, dims=3, m=3); + +INSERT INTO t (val) VALUES (array[1,2,4]); + +explain SELECT * FROM t ORDER BY val <-> array[3,3,3]; +SELECT * FROM t ORDER BY val <-> array[3,3,3]; +SELECT COUNT(*) FROM t; + +DROP TABLE t; diff --git a/pgxn/neon/control_plane_connector.c b/pgxn/neon/control_plane_connector.c index 82e4af4b4a..debbbce117 100644 --- a/pgxn/neon/control_plane_connector.c +++ b/pgxn/neon/control_plane_connector.c @@ -32,6 +32,7 @@ #include "port.h" #include #include "utils/jsonb.h" +#include "libpq/crypt.h" static ProcessUtility_hook_type PreviousProcessUtilityHook = NULL; @@ -161,7 +162,22 @@ ConstructDeltaMessage() PushKeyValue(&state, "name", entry->name); if (entry->password) { +#if PG_MAJORVERSION_NUM == 14 + char *logdetail; +#else + const char *logdetail; +#endif PushKeyValue(&state, "password", (char *) entry->password); + char *encrypted_password = get_role_password(entry->name, &logdetail); + + if (encrypted_password) + { + PushKeyValue(&state, "encrypted_password", encrypted_password); + } + else + { + elog(ERROR, "Failed to get encrypted password: %s", logdetail); + } } if (entry->old_name[0] != '\0') { diff --git a/pgxn/neon/file_cache.c b/pgxn/neon/file_cache.c index 1ab2ae668a..02795bc8b8 100644 --- a/pgxn/neon/file_cache.c +++ b/pgxn/neon/file_cache.c @@ -190,7 +190,7 @@ lfc_change_limit_hook(int newval, void *extra) hash_search(lfc_hash, &victim->key, HASH_REMOVE, NULL); lfc_ctl->used -= 1; } - elog(LOG, "set local file cache limit to %d", new_size); + elog(DEBUG1, "set local file cache limit to %d", new_size); LWLockRelease(lfc_lock); } diff --git a/pgxn/neon/pagestore_smgr.c b/pgxn/neon/pagestore_smgr.c index 528d4eb051..79dec0881d 100644 --- a/pgxn/neon/pagestore_smgr.c +++ b/pgxn/neon/pagestore_smgr.c @@ -2675,7 +2675,6 @@ bool neon_redo_read_buffer_filter(XLogReaderState *record, uint8 block_id) { XLogRecPtr end_recptr = record->EndRecPtr; - XLogRecPtr prev_end_recptr = record->ReadRecPtr - 1; RelFileNode rnode; ForkNumber forknum; BlockNumber blkno; @@ -2719,16 +2718,15 @@ neon_redo_read_buffer_filter(XLogReaderState *record, uint8 block_id) no_redo_needed = buffer < 0; - /* we don't have the buffer in memory, update lwLsn past this record */ + /* In both cases st lwlsn past this WAL record */ + SetLastWrittenLSNForBlock(end_recptr, rnode, forknum, blkno); + + /* we don't have the buffer in memory, update lwLsn past this record, + * also evict page fro file cache + */ if (no_redo_needed) - { - SetLastWrittenLSNForBlock(end_recptr, rnode, forknum, blkno); lfc_evict(rnode, forknum, blkno); - } - else - { - SetLastWrittenLSNForBlock(prev_end_recptr, rnode, forknum, blkno); - } + LWLockRelease(partitionLock); @@ -2736,7 +2734,10 @@ neon_redo_read_buffer_filter(XLogReaderState *record, uint8 block_id) if (get_cached_relsize(rnode, forknum, &relsize)) { if (relsize < blkno + 1) + { update_cached_relsize(rnode, forknum, blkno + 1); + SetLastWrittenLSNForRelation(end_recptr, rnode, forknum); + } } else { @@ -2768,6 +2769,7 @@ neon_redo_read_buffer_filter(XLogReaderState *record, uint8 block_id) Assert(nbresponse->n_blocks > blkno); set_cached_relsize(rnode, forknum, nbresponse->n_blocks); + SetLastWrittenLSNForRelation(end_recptr, rnode, forknum); elog(SmgrTrace, "Set length to %d", nbresponse->n_blocks); } diff --git a/pgxn/neon/walproposer.c b/pgxn/neon/walproposer.c index a99be40955..8d82de6dc4 100644 --- a/pgxn/neon/walproposer.c +++ b/pgxn/neon/walproposer.c @@ -254,7 +254,7 @@ nwp_register_gucs(void) DefineCustomIntVariable( "neon.safekeeper_reconnect_timeout", - "Timeout for reconnecting to offline wal acceptor.", + "Walproposer reconnects to offline safekeepers once in this interval.", NULL, &wal_acceptor_reconnect_timeout, 1000, 0, INT_MAX, /* default, min, max */ @@ -264,10 +264,10 @@ nwp_register_gucs(void) DefineCustomIntVariable( "neon.safekeeper_connect_timeout", - "Timeout for connection establishement and it's maintenance against safekeeper", + "Connection or connection attempt to safekeeper is terminated if no message is received (or connection attempt doesn't finish) within this period.", NULL, &wal_acceptor_connection_timeout, - 5000, 0, INT_MAX, + 10000, 0, INT_MAX, PGC_SIGHUP, GUC_UNIT_MS, NULL, NULL, NULL); @@ -441,7 +441,7 @@ WalProposerPoll(void) if (TimestampDifferenceExceeds(sk->latestMsgReceivedAt, now, wal_acceptor_connection_timeout)) { - elog(WARNING, "failed to connect to node '%s:%s' in '%s' state: exceeded connection timeout %dms", + elog(WARNING, "terminating connection to safekeeper '%s:%s' in '%s' state: no messages received during the last %dms or connection attempt took longer than that", sk->host, sk->port, FormatSafekeeperState(sk->state), wal_acceptor_connection_timeout); ShutdownConnection(sk); } @@ -1035,9 +1035,16 @@ RecvAcceptorGreeting(Safekeeper *sk) if (!AsyncReadMessage(sk, (AcceptorProposerMessage *) & sk->greetResponse)) return; + elog(LOG, "received AcceptorGreeting from safekeeper %s:%s", sk->host, sk->port); + /* Protocol is all good, move to voting. */ sk->state = SS_VOTING; + /* + * Note: it would be better to track the counter on per safekeeper basis, + * but at worst walproposer would restart with 'term rejected', so leave as + * is for now. + */ ++n_connected; if (n_connected <= quorum) { diff --git a/poetry.lock b/poetry.lock index f544eb8d5c..8dc45f68b8 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.4.2 and should not be changed by hand. +# This file is automatically @generated by Poetry and should not be changed by hand. [[package]] name = "aiohttp" @@ -855,35 +855,31 @@ files = [ [[package]] name = "cryptography" -version = "39.0.1" +version = "41.0.0" description = "cryptography is a package which provides cryptographic recipes and primitives to Python developers." category = "main" optional = false -python-versions = ">=3.6" +python-versions = ">=3.7" files = [ - {file = "cryptography-39.0.1-cp36-abi3-macosx_10_12_universal2.whl", hash = "sha256:6687ef6d0a6497e2b58e7c5b852b53f62142cfa7cd1555795758934da363a965"}, - {file = "cryptography-39.0.1-cp36-abi3-macosx_10_12_x86_64.whl", hash = "sha256:706843b48f9a3f9b9911979761c91541e3d90db1ca905fd63fee540a217698bc"}, - {file = "cryptography-39.0.1-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_24_aarch64.whl", hash = "sha256:5d2d8b87a490bfcd407ed9d49093793d0f75198a35e6eb1a923ce1ee86c62b41"}, - {file = "cryptography-39.0.1-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:83e17b26de248c33f3acffb922748151d71827d6021d98c70e6c1a25ddd78505"}, - {file = "cryptography-39.0.1-cp36-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e124352fd3db36a9d4a21c1aa27fd5d051e621845cb87fb851c08f4f75ce8be6"}, - {file = "cryptography-39.0.1-cp36-abi3-manylinux_2_24_x86_64.whl", hash = "sha256:5aa67414fcdfa22cf052e640cb5ddc461924a045cacf325cd164e65312d99502"}, - {file = "cryptography-39.0.1-cp36-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:35f7c7d015d474f4011e859e93e789c87d21f6f4880ebdc29896a60403328f1f"}, - {file = "cryptography-39.0.1-cp36-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:f24077a3b5298a5a06a8e0536e3ea9ec60e4c7ac486755e5fb6e6ea9b3500106"}, - {file = "cryptography-39.0.1-cp36-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:f0c64d1bd842ca2633e74a1a28033d139368ad959872533b1bab8c80e8240a0c"}, - {file = "cryptography-39.0.1-cp36-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:0f8da300b5c8af9f98111ffd512910bc792b4c77392a9523624680f7956a99d4"}, - {file = "cryptography-39.0.1-cp36-abi3-win32.whl", hash = "sha256:fe913f20024eb2cb2f323e42a64bdf2911bb9738a15dba7d3cce48151034e3a8"}, - {file = "cryptography-39.0.1-cp36-abi3-win_amd64.whl", hash = "sha256:ced4e447ae29ca194449a3f1ce132ded8fcab06971ef5f618605aacaa612beac"}, - {file = "cryptography-39.0.1-pp38-pypy38_pp73-macosx_10_12_x86_64.whl", hash = "sha256:807ce09d4434881ca3a7594733669bd834f5b2c6d5c7e36f8c00f691887042ad"}, - {file = "cryptography-39.0.1-pp38-pypy38_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:c5caeb8188c24888c90b5108a441c106f7faa4c4c075a2bcae438c6e8ca73cef"}, - {file = "cryptography-39.0.1-pp38-pypy38_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:4789d1e3e257965e960232345002262ede4d094d1a19f4d3b52e48d4d8f3b885"}, - {file = "cryptography-39.0.1-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:96f1157a7c08b5b189b16b47bc9db2332269d6680a196341bf30046330d15388"}, - {file = "cryptography-39.0.1-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:e422abdec8b5fa8462aa016786680720d78bdce7a30c652b7fadf83a4ba35336"}, - {file = "cryptography-39.0.1-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_24_aarch64.whl", hash = "sha256:b0afd054cd42f3d213bf82c629efb1ee5f22eba35bf0eec88ea9ea7304f511a2"}, - {file = "cryptography-39.0.1-pp39-pypy39_pp73-manylinux_2_24_x86_64.whl", hash = "sha256:6f8ba7f0328b79f08bdacc3e4e66fb4d7aab0c3584e0bd41328dce5262e26b2e"}, - {file = "cryptography-39.0.1-pp39-pypy39_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:ef8b72fa70b348724ff1218267e7f7375b8de4e8194d1636ee60510aae104cd0"}, - {file = "cryptography-39.0.1-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:aec5a6c9864be7df2240c382740fcf3b96928c46604eaa7f3091f58b878c0bb6"}, - {file = "cryptography-39.0.1-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:fdd188c8a6ef8769f148f88f859884507b954cc64db6b52f66ef199bb9ad660a"}, - {file = "cryptography-39.0.1.tar.gz", hash = "sha256:d1f6198ee6d9148405e49887803907fe8962a23e6c6f83ea7d98f1c0de375695"}, + {file = "cryptography-41.0.0-cp37-abi3-macosx_10_12_universal2.whl", hash = "sha256:3c5ef25d060c80d6d9f7f9892e1d41bb1c79b78ce74805b8cb4aa373cb7d5ec8"}, + {file = "cryptography-41.0.0-cp37-abi3-macosx_10_12_x86_64.whl", hash = "sha256:8362565b3835ceacf4dc8f3b56471a2289cf51ac80946f9087e66dc283a810e0"}, + {file = "cryptography-41.0.0-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3680248309d340fda9611498a5319b0193a8dbdb73586a1acf8109d06f25b92d"}, + {file = "cryptography-41.0.0-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:84a165379cb9d411d58ed739e4af3396e544eac190805a54ba2e0322feb55c46"}, + {file = "cryptography-41.0.0-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:4ab14d567f7bbe7f1cdff1c53d5324ed4d3fc8bd17c481b395db224fb405c237"}, + {file = "cryptography-41.0.0-cp37-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:9f65e842cb02550fac96536edb1d17f24c0a338fd84eaf582be25926e993dde4"}, + {file = "cryptography-41.0.0-cp37-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:b7f2f5c525a642cecad24ee8670443ba27ac1fab81bba4cc24c7b6b41f2d0c75"}, + {file = "cryptography-41.0.0-cp37-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:7d92f0248d38faa411d17f4107fc0bce0c42cae0b0ba5415505df72d751bf62d"}, + {file = "cryptography-41.0.0-cp37-abi3-win32.whl", hash = "sha256:34d405ea69a8b34566ba3dfb0521379b210ea5d560fafedf9f800a9a94a41928"}, + {file = "cryptography-41.0.0-cp37-abi3-win_amd64.whl", hash = "sha256:344c6de9f8bda3c425b3a41b319522ba3208551b70c2ae00099c205f0d9fd3be"}, + {file = "cryptography-41.0.0-pp38-pypy38_pp73-macosx_10_12_x86_64.whl", hash = "sha256:88ff107f211ea696455ea8d911389f6d2b276aabf3231bf72c8853d22db755c5"}, + {file = "cryptography-41.0.0-pp38-pypy38_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:b846d59a8d5a9ba87e2c3d757ca019fa576793e8758174d3868aecb88d6fc8eb"}, + {file = "cryptography-41.0.0-pp38-pypy38_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:f5d0bf9b252f30a31664b6f64432b4730bb7038339bd18b1fafe129cfc2be9be"}, + {file = "cryptography-41.0.0-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:5c1f7293c31ebc72163a9a0df246f890d65f66b4a40d9ec80081969ba8c78cc9"}, + {file = "cryptography-41.0.0-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:bf8fc66012ca857d62f6a347007e166ed59c0bc150cefa49f28376ebe7d992a2"}, + {file = "cryptography-41.0.0-pp39-pypy39_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:a4fc68d1c5b951cfb72dfd54702afdbbf0fb7acdc9b7dc4301bbf2225a27714d"}, + {file = "cryptography-41.0.0-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:14754bcdae909d66ff24b7b5f166d69340ccc6cb15731670435efd5719294895"}, + {file = "cryptography-41.0.0-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:0ddaee209d1cf1f180f1efa338a68c4621154de0afaef92b89486f5f96047c55"}, + {file = "cryptography-41.0.0.tar.gz", hash = "sha256:6b71f64beeea341c9b4f963b48ee3b62d62d57ba93eb120e1196b31dc1025e78"}, ] [package.dependencies] @@ -892,12 +888,12 @@ cffi = ">=1.12" [package.extras] docs = ["sphinx (>=5.3.0)", "sphinx-rtd-theme (>=1.1.1)"] docstest = ["pyenchant (>=1.6.11)", "sphinxcontrib-spelling (>=4.0.1)", "twine (>=1.12.0)"] -pep8test = ["black", "check-manifest", "mypy", "ruff", "types-pytz", "types-requests"] -sdist = ["setuptools-rust (>=0.11.4)"] +nox = ["nox"] +pep8test = ["black", "check-sdist", "mypy", "ruff"] +sdist = ["build"] ssh = ["bcrypt (>=3.1.5)"] -test = ["hypothesis (>=1.11.4,!=3.79.2)", "iso8601", "pretend", "pytest (>=6.2.0)", "pytest-benchmark", "pytest-cov", "pytest-shard (>=0.1.2)", "pytest-subtests", "pytest-xdist", "pytz"] +test = ["pretend", "pytest (>=6.2.0)", "pytest-benchmark", "pytest-cov", "pytest-xdist"] test-randomorder = ["pytest-randomly"] -tox = ["tox"] [[package]] name = "docker" diff --git a/proxy/README.md b/proxy/README.md index cd76a2443f..d1f2e3f27b 100644 --- a/proxy/README.md +++ b/proxy/README.md @@ -93,6 +93,15 @@ With the current approach we made the following design decisions: and column oids. Command tag capturing was added to the rust-postgres functionality as part of this change. +### Output options + +User can pass several optional headers that will affect resulting json. + +1. `Neon-Raw-Text-Output: true`. Return postgres values as text, without parsing them. So numbers, objects, booleans, nulls and arrays will be returned as text. That can be useful in cases when client code wants to implement it's own parsing or reuse parsing libraries from e.g. node-postgres. +2. `Neon-Array-Mode: true`. Return postgres rows as arrays instead of objects. That is more compact representation and also helps in some edge +cases where it is hard to use rows represented as objects (e.g. when several fields have the same name). + + ## Using SNI-based routing on localhost Now proxy determines project name from the subdomain, request to the `round-rice-566201.somedomain.tld` will be routed to the project named `round-rice-566201`. Unfortunately, `/etc/hosts` does not support domain wildcards, so I usually use `*.localtest.me` which resolves to `127.0.0.1`. Now we can create self-signed certificate and play with proxy: diff --git a/proxy/src/bin/pg_sni_router.rs b/proxy/src/bin/pg_sni_router.rs index bba2d51caf..a5f50cc7c1 100644 --- a/proxy/src/bin/pg_sni_router.rs +++ b/proxy/src/bin/pg_sni_router.rs @@ -17,7 +17,7 @@ use tokio::io::{AsyncRead, AsyncWrite}; use tokio_util::sync::CancellationToken; use utils::{project_git_version, sentry_init::init_sentry}; -use tracing::{error, info, warn}; +use tracing::{error, info, warn, Instrument}; project_git_version!(GIT_VERSION); @@ -141,7 +141,6 @@ async fn task_main( tokio::select! { accept_result = listener.accept() => { let (socket, peer_addr) = accept_result?; - info!("accepted postgres client connection from {peer_addr}"); let session_id = uuid::Uuid::new_v4(); let tls_config = Arc::clone(&tls_config); @@ -149,18 +148,18 @@ async fn task_main( connections.spawn( async move { - info!("spawned a task for {peer_addr}"); - socket .set_nodelay(true) .context("failed to set socket option")?; - handle_client(dest_suffix, tls_config, session_id, socket).await + info!(%peer_addr, "serving"); + handle_client(dest_suffix, tls_config, socket).await } .unwrap_or_else(|e| { // Acknowledge that the task has finished with an error. error!("per-client task finished with an error: {e:#}"); - }), + }) + .instrument(tracing::info_span!("handle_client", ?session_id)) ); } _ = cancellation_token.cancelled() => { @@ -192,7 +191,6 @@ async fn ssl_handshake( let mut stream = PqStream::new(Stream::from_raw(raw_stream)); let msg = stream.read_startup_packet().await?; - info!("received {msg:?}"); use pq_proto::FeStartupPacket::*; match msg { @@ -215,15 +213,19 @@ async fn ssl_handshake( } Ok(raw.upgrade(tls_config).await?) } - _ => stream.throw_error_str(ERR_INSECURE_CONNECTION).await?, + unexpected => { + info!( + ?unexpected, + "unexpected startup packet, rejecting connection" + ); + stream.throw_error_str(ERR_INSECURE_CONNECTION).await? + } } } -#[tracing::instrument(fields(session_id = ?session_id), skip_all)] async fn handle_client( dest_suffix: Arc, tls_config: Arc, - session_id: uuid::Uuid, stream: impl AsyncRead + AsyncWrite + Unpin, ) -> anyhow::Result<()> { let tls_stream = ssl_handshake(stream, tls_config).await?; diff --git a/proxy/src/console.rs b/proxy/src/console.rs index 1f3ef99555..0e5eaaf845 100644 --- a/proxy/src/console.rs +++ b/proxy/src/console.rs @@ -1,5 +1,5 @@ -///! Various stuff for dealing with the Neon Console. -///! Later we might move some API wrappers here. +//! Various stuff for dealing with the Neon Console. +//! Later we might move some API wrappers here. /// Payloads used in the console's APIs. pub mod messages; diff --git a/proxy/src/http/sql_over_http.rs b/proxy/src/http/sql_over_http.rs index 0438a82c12..e8ad2d04f3 100644 --- a/proxy/src/http/sql_over_http.rs +++ b/proxy/src/http/sql_over_http.rs @@ -1,6 +1,9 @@ use futures::pin_mut; use futures::StreamExt; +use futures::TryFutureExt; use hyper::body::HttpBody; +use hyper::http::HeaderName; +use hyper::http::HeaderValue; use hyper::{Body, HeaderMap, Request}; use pq_proto::StartupMessageParams; use serde_json::json; @@ -9,8 +12,13 @@ use serde_json::Value; use tokio_postgres::types::Kind; use tokio_postgres::types::Type; use tokio_postgres::Row; +use tracing::error; +use tracing::info; +use tracing::instrument; use url::Url; +use crate::proxy::invalidate_cache; +use crate::proxy::NUM_RETRIES_WAKE_COMPUTE; use crate::{auth, config::ProxyConfig, console}; #[derive(serde::Deserialize)] @@ -23,21 +31,28 @@ const APP_NAME: &str = "sql_over_http"; const MAX_RESPONSE_SIZE: usize = 1024 * 1024; // 1 MB const MAX_REQUEST_SIZE: u64 = 1024 * 1024; // 1 MB +static RAW_TEXT_OUTPUT: HeaderName = HeaderName::from_static("neon-raw-text-output"); +static ARRAY_MODE: HeaderName = HeaderName::from_static("neon-array-mode"); +static HEADER_VALUE_TRUE: HeaderValue = HeaderValue::from_static("true"); + // // Convert json non-string types to strings, so that they can be passed to Postgres // as parameters. // -fn json_to_pg_text(json: Vec) -> Result, serde_json::Error> { +fn json_to_pg_text(json: Vec) -> Result>, serde_json::Error> { json.iter() .map(|value| { match value { - Value::Null => serde_json::to_string(value), - Value::Bool(_) => serde_json::to_string(value), - Value::Number(_) => serde_json::to_string(value), - Value::Object(_) => serde_json::to_string(value), + // special care for nulls + Value::Null => Ok(None), - // no need to escape - Value::String(s) => Ok(s.to_string()), + // convert to text with escaping + Value::Bool(_) => serde_json::to_string(value).map(Some), + Value::Number(_) => serde_json::to_string(value).map(Some), + Value::Object(_) => serde_json::to_string(value).map(Some), + + // avoid escaping here, as we pass this as a parameter + Value::String(s) => Ok(Some(s.to_string())), // special care for arrays Value::Array(_) => json_array_to_pg_array(value), @@ -54,33 +69,44 @@ fn json_to_pg_text(json: Vec) -> Result, serde_json::Error> { // // Example of the same escaping in node-postgres: packages/pg/lib/utils.js // -fn json_array_to_pg_array(value: &Value) -> Result { +fn json_array_to_pg_array(value: &Value) -> Result, serde_json::Error> { match value { - // same - Value::Null => serde_json::to_string(value), - Value::Bool(_) => serde_json::to_string(value), - Value::Number(_) => serde_json::to_string(value), - Value::Object(_) => serde_json::to_string(value), + // special care for nulls + Value::Null => Ok(None), - // now needs to be escaped, as it is part of the array - Value::String(_) => serde_json::to_string(value), + // convert to text with escaping + Value::Bool(_) => serde_json::to_string(value).map(Some), + Value::Number(_) => serde_json::to_string(value).map(Some), + Value::Object(_) => serde_json::to_string(value).map(Some), + + // here string needs to be escaped, as it is part of the array + Value::String(_) => serde_json::to_string(value).map(Some), // recurse into array Value::Array(arr) => { let vals = arr .iter() .map(json_array_to_pg_array) + .map(|r| r.map(|v| v.unwrap_or_else(|| "NULL".to_string()))) .collect::, _>>()? .join(","); - Ok(format!("{{{}}}", vals)) + + Ok(Some(format!("{{{}}}", vals))) } } } +struct ConnInfo { + username: String, + dbname: String, + hostname: String, + password: String, +} + fn get_conn_info( headers: &HeaderMap, sni_hostname: Option, -) -> Result<(String, String, String, String), anyhow::Error> { +) -> Result { let connection_string = headers .get("Neon-Connection-String") .ok_or(anyhow::anyhow!("missing connection string"))? @@ -133,12 +159,12 @@ fn get_conn_info( } } - Ok(( - username.to_owned(), - dbname.to_owned(), - hostname.to_owned(), - password.to_owned(), - )) + Ok(ConnInfo { + username: username.to_owned(), + dbname: dbname.to_owned(), + hostname: hostname.to_owned(), + password: password.to_owned(), + }) } // TODO: return different http error codes @@ -151,13 +177,18 @@ pub async fn handle( // Determine the destination and connection params // let headers = request.headers(); - let (username, dbname, hostname, password) = get_conn_info(headers, sni_hostname)?; + let conn_info = get_conn_info(headers, sni_hostname)?; let credential_params = StartupMessageParams::new([ - ("user", &username), - ("database", &dbname), + ("user", &conn_info.username), + ("database", &conn_info.dbname), ("application_name", APP_NAME), ]); + // Determine the output options. Default behaviour is 'false'. Anything that is not + // strictly 'true' assumed to be false. + let raw_output = headers.get(&RAW_TEXT_OUTPUT) == Some(&HEADER_VALUE_TRUE); + let array_mode = headers.get(&ARRAY_MODE) == Some(&HEADER_VALUE_TRUE); + // // Wake up the destination if needed. Code here is a bit involved because // we reuse the code from the usual proxy and we need to prepare few structures @@ -168,21 +199,20 @@ pub async fn handle( let creds = config .auth_backend .as_ref() - .map(|_| auth::ClientCredentials::parse(&credential_params, Some(&hostname), common_names)) + .map(|_| { + auth::ClientCredentials::parse( + &credential_params, + Some(&conn_info.hostname), + common_names, + ) + }) .transpose()?; let extra = console::ConsoleReqExtra { session_id: uuid::Uuid::new_v4(), application_name: Some(APP_NAME), }; - let node = creds.wake_compute(&extra).await?.expect("msg"); - let conf = node.value.config; - let port = *conf.get_ports().first().expect("no port"); - let host = match conf.get_hosts().first().expect("no host") { - tokio_postgres::config::Host::Tcp(host) => host, - tokio_postgres::config::Host::Unix(_) => { - return Err(anyhow::anyhow!("unix socket is not supported")); - } - }; + + let mut node_info = creds.wake_compute(&extra).await?.expect("msg"); let request_content_length = match request.body().size_hint().upper() { Some(v) => v, @@ -202,28 +232,10 @@ pub async fn handle( let QueryData { query, params } = serde_json::from_slice(&body)?; let query_params = json_to_pg_text(params)?; - // - // Connenct to the destination - // - let (client, connection) = tokio_postgres::Config::new() - .host(host) - .port(port) - .user(&username) - .password(&password) - .dbname(&dbname) - .max_backend_message_size(MAX_RESPONSE_SIZE) - .connect(tokio_postgres::NoTls) - .await?; - - tokio::spawn(async move { - if let Err(e) = connection.await { - eprintln!("connection error: {}", e); - } - }); - // // Now execute the query and return the result // + let client = connect_to_compute(&mut node_info, &extra, &creds, &conn_info).await?; let row_stream = client.query_raw_txt(query, query_params).await?; // Manually drain the stream into a vector to leave row_stream hanging @@ -262,6 +274,11 @@ pub async fn handle( json!({ "name": Value::String(c.name().to_owned()), "dataTypeID": Value::Number(c.type_().oid().into()), + "tableID": c.table_oid(), + "columnID": c.column_id(), + "dataTypeSize": c.type_size(), + "dataTypeModifier": c.type_modifier(), + "format": "text", }) }) .collect::>() @@ -272,7 +289,7 @@ pub async fn handle( // convert rows to JSON let rows = rows .iter() - .map(pg_text_row_to_json) + .map(|row| pg_text_row_to_json(row, raw_output, array_mode)) .collect::, _>>()?; // resulting JSON format is based on the format of node-postgres result @@ -281,26 +298,106 @@ pub async fn handle( "rowCount": command_tag_count, "rows": rows, "fields": fields, + "rowAsArray": array_mode, })) } +/// This function is a copy of `connect_to_compute` from `src/proxy.rs` with +/// the difference that it uses `tokio_postgres` for the connection. +#[instrument(skip_all)] +async fn connect_to_compute( + node_info: &mut console::CachedNodeInfo, + extra: &console::ConsoleReqExtra<'_>, + creds: &auth::BackendType<'_, auth::ClientCredentials<'_>>, + conn_info: &ConnInfo, +) -> anyhow::Result { + let mut num_retries: usize = NUM_RETRIES_WAKE_COMPUTE; + + loop { + match connect_to_compute_once(node_info, conn_info).await { + Err(e) if num_retries > 0 => { + info!("compute node's state has changed; requesting a wake-up"); + match creds.wake_compute(extra).await? { + // Update `node_info` and try one more time. + Some(new) => { + *node_info = new; + } + // Link auth doesn't work that way, so we just exit. + None => return Err(e), + } + } + other => return other, + } + + num_retries -= 1; + info!("retrying after wake-up ({num_retries} attempts left)"); + } +} + +async fn connect_to_compute_once( + node_info: &console::CachedNodeInfo, + conn_info: &ConnInfo, +) -> anyhow::Result { + let mut config = (*node_info.config).clone(); + + let (client, connection) = config + .user(&conn_info.username) + .password(&conn_info.password) + .dbname(&conn_info.dbname) + .max_backend_message_size(MAX_RESPONSE_SIZE) + .connect(tokio_postgres::NoTls) + .inspect_err(|e: &tokio_postgres::Error| { + error!( + "failed to connect to compute node hosts={:?} ports={:?}: {}", + node_info.config.get_hosts(), + node_info.config.get_ports(), + e + ); + invalidate_cache(node_info) + }) + .await?; + + tokio::spawn(async move { + if let Err(e) = connection.await { + error!("connection error: {}", e); + } + }); + + Ok(client) +} + // // Convert postgres row with text-encoded values to JSON object // -pub fn pg_text_row_to_json(row: &Row) -> Result { - let res = row - .columns() - .iter() - .enumerate() - .map(|(i, column)| { - let name = column.name(); - let pg_value = row.as_text(i)?; - let json_value = pg_text_to_json(pg_value, column.type_())?; - Ok((name.to_string(), json_value)) - }) - .collect::, anyhow::Error>>()?; +pub fn pg_text_row_to_json( + row: &Row, + raw_output: bool, + array_mode: bool, +) -> Result { + let iter = row.columns().iter().enumerate().map(|(i, column)| { + let name = column.name(); + let pg_value = row.as_text(i)?; + let json_value = if raw_output { + match pg_value { + Some(v) => Value::String(v.to_string()), + None => Value::Null, + } + } else { + pg_text_to_json(pg_value, column.type_())? + }; + Ok((name.to_string(), json_value)) + }); - Ok(Value::Object(res)) + if array_mode { + // drop keys and aggregate into array + let arr = iter + .map(|r| r.map(|(_key, val)| val)) + .collect::, anyhow::Error>>()?; + Ok(Value::Array(arr)) + } else { + let obj = iter.collect::, anyhow::Error>>()?; + Ok(Value::Object(obj)) + } } // @@ -308,10 +405,6 @@ pub fn pg_text_row_to_json(row: &Row) -> Result { // pub fn pg_text_to_json(pg_value: Option<&str>, pg_type: &Type) -> Result { if let Some(val) = pg_value { - if val == "NULL" { - return Ok(Value::Null); - } - if let Kind::Array(elem_type) = pg_type.kind() { return pg_array_parse(val, elem_type); } @@ -373,6 +466,27 @@ fn _pg_array_parse( } } + fn push_checked( + entry: &mut String, + entries: &mut Vec, + elem_type: &Type, + ) -> Result<(), anyhow::Error> { + if !entry.is_empty() { + // While in usual postgres response we get nulls as None and everything else + // as Some(&str), in arrays we get NULL as unquoted 'NULL' string (while + // string with value 'NULL' will be represented by '"NULL"'). So catch NULLs + // here while we have quotation info and convert them to None. + if entry == "NULL" { + entries.push(pg_text_to_json(None, elem_type)?); + } else { + entries.push(pg_text_to_json(Some(entry), elem_type)?); + } + entry.clear(); + } + + Ok(()) + } + while let Some((mut i, mut c)) = pg_array_chr.next() { let mut escaped = false; @@ -395,9 +509,7 @@ fn _pg_array_parse( '}' => { level -= 1; if level == 0 { - if !entry.is_empty() { - entries.push(pg_text_to_json(Some(&entry), elem_type)?); - } + push_checked(&mut entry, &mut entries, elem_type)?; if nested { return Ok((Value::Array(entries), i)); } @@ -405,17 +517,15 @@ fn _pg_array_parse( } '"' if !escaped => { if quote { - // push even if empty + // end of quoted string, so push it manually without any checks + // for emptiness or nulls entries.push(pg_text_to_json(Some(&entry), elem_type)?); - entry = String::new(); + entry.clear(); } quote = !quote; } ',' if !quote => { - if !entry.is_empty() { - entries.push(pg_text_to_json(Some(&entry), elem_type)?); - entry = String::new(); - } + push_checked(&mut entry, &mut entries, elem_type)?; } _ => { entry.push(c); @@ -439,30 +549,35 @@ mod tests { fn test_atomic_types_to_pg_params() { let json = vec![Value::Bool(true), Value::Bool(false)]; let pg_params = json_to_pg_text(json).unwrap(); - assert_eq!(pg_params, vec!["true", "false"]); + assert_eq!( + pg_params, + vec![Some("true".to_owned()), Some("false".to_owned())] + ); let json = vec![Value::Number(serde_json::Number::from(42))]; let pg_params = json_to_pg_text(json).unwrap(); - assert_eq!(pg_params, vec!["42"]); + assert_eq!(pg_params, vec![Some("42".to_owned())]); let json = vec![Value::String("foo\"".to_string())]; let pg_params = json_to_pg_text(json).unwrap(); - assert_eq!(pg_params, vec!["foo\""]); + assert_eq!(pg_params, vec![Some("foo\"".to_owned())]); let json = vec![Value::Null]; let pg_params = json_to_pg_text(json).unwrap(); - assert_eq!(pg_params, vec!["null"]); + assert_eq!(pg_params, vec![None]); } #[test] fn test_json_array_to_pg_array() { // atoms and escaping - let json = "[true, false, null, 42, \"foo\", \"bar\\\"-\\\\\"]"; + let json = "[true, false, null, \"NULL\", 42, \"foo\", \"bar\\\"-\\\\\"]"; let json: Value = serde_json::from_str(json).unwrap(); let pg_params = json_to_pg_text(vec![json]).unwrap(); assert_eq!( pg_params, - vec!["{true,false,null,42,\"foo\",\"bar\\\"-\\\\\"}"] + vec![Some( + "{true,false,NULL,\"NULL\",42,\"foo\",\"bar\\\"-\\\\\"}".to_owned() + )] ); // nested arrays @@ -471,7 +586,9 @@ mod tests { let pg_params = json_to_pg_text(vec![json]).unwrap(); assert_eq!( pg_params, - vec!["{{true,false},{null,42},{\"foo\",\"bar\\\"-\\\\\"}}"] + vec![Some( + "{{true,false},{NULL,42},{\"foo\",\"bar\\\"-\\\\\"}}".to_owned() + )] ); } diff --git a/proxy/src/http/websocket.rs b/proxy/src/http/websocket.rs index fbb602e3d2..9f467aceb7 100644 --- a/proxy/src/http/websocket.rs +++ b/proxy/src/http/websocket.rs @@ -26,7 +26,6 @@ use tls_listener::TlsListener; use tokio::{ io::{self, AsyncBufRead, AsyncRead, AsyncWrite, ReadBuf}, net::TcpListener, - select, }; use tokio_util::sync::CancellationToken; use tracing::{error, info, info_span, warn, Instrument}; @@ -193,14 +192,9 @@ async fn ws_handler( // TODO: that deserves a refactor as now this function also handles http json client besides websockets. // Right now I don't want to blow up sql-over-http patch with file renames and do that as a follow up instead. } else if request.uri().path() == "/sql" && request.method() == Method::POST { - let result = select! { - _ = tokio::time::sleep(std::time::Duration::from_secs(10)) => { - Err(anyhow::anyhow!("Query timed out")) - } - response = sql_over_http::handle(config, request, sni_hostname) => { - response - } - }; + let result = sql_over_http::handle(config, request, sni_hostname) + .instrument(info_span!("sql-over-http")) + .await; let status_code = match result { Ok(_) => StatusCode::OK, Err(_) => StatusCode::BAD_REQUEST, diff --git a/proxy/src/proxy.rs b/proxy/src/proxy.rs index cf2dd000db..8efb7005c8 100644 --- a/proxy/src/proxy.rs +++ b/proxy/src/proxy.rs @@ -22,7 +22,7 @@ use tracing::{error, info, warn}; use utils::measured_stream::MeasuredStream; /// Number of times we should retry the `/proxy_wake_compute` http request. -const NUM_RETRIES_WAKE_COMPUTE: usize = 1; +pub const NUM_RETRIES_WAKE_COMPUTE: usize = 1; const ERR_INSECURE_CONNECTION: &str = "connection is insecure (try using `sslmode=require`)"; const ERR_PROTO_VIOLATION: &str = "protocol violation"; @@ -283,34 +283,35 @@ async fn handshake( } } +/// If we couldn't connect, a cached connection info might be to blame +/// (e.g. the compute node's address might've changed at the wrong time). +/// Invalidate the cache entry (if any) to prevent subsequent errors. +#[tracing::instrument(name = "invalidate_cache", skip_all)] +pub fn invalidate_cache(node_info: &console::CachedNodeInfo) { + let is_cached = node_info.cached(); + if is_cached { + warn!("invalidating stalled compute node info cache entry"); + node_info.invalidate(); + } + + let label = match is_cached { + true => "compute_cached", + false => "compute_uncached", + }; + NUM_CONNECTION_FAILURES.with_label_values(&[label]).inc(); +} + /// Try to connect to the compute node once. #[tracing::instrument(name = "connect_once", skip_all)] async fn connect_to_compute_once( node_info: &console::CachedNodeInfo, ) -> Result { - // If we couldn't connect, a cached connection info might be to blame - // (e.g. the compute node's address might've changed at the wrong time). - // Invalidate the cache entry (if any) to prevent subsequent errors. - let invalidate_cache = |_: &compute::ConnectionError| { - let is_cached = node_info.cached(); - if is_cached { - warn!("invalidating stalled compute node info cache entry"); - node_info.invalidate(); - } - - let label = match is_cached { - true => "compute_cached", - false => "compute_uncached", - }; - NUM_CONNECTION_FAILURES.with_label_values(&[label]).inc(); - }; - let allow_self_signed_compute = node_info.allow_self_signed_compute; node_info .config .connect(allow_self_signed_compute) - .inspect_err(invalidate_cache) + .inspect_err(|_: &compute::ConnectionError| invalidate_cache(node_info)) .await } diff --git a/proxy/src/proxy/tests.rs b/proxy/src/proxy/tests.rs index 60acb588dc..3373c49676 100644 --- a/proxy/src/proxy/tests.rs +++ b/proxy/src/proxy/tests.rs @@ -1,4 +1,4 @@ -///! A group of high-level tests for connection establishing logic and auth. +//! A group of high-level tests for connection establishing logic and auth. use super::*; use crate::{auth, sasl, scram}; use async_trait::async_trait; diff --git a/rust-toolchain.toml b/rust-toolchain.toml index c39ba4f417..6abb435018 100644 --- a/rust-toolchain.toml +++ b/rust-toolchain.toml @@ -1,5 +1,5 @@ [toolchain] -channel = "1.68.2" +channel = "1.70.0" profile = "default" # The default profile includes rustc, rust-std, cargo, rust-docs, rustfmt and clippy. # https://rust-lang.github.io/rustup/concepts/profiles.html diff --git a/safekeeper/src/bin/safekeeper.rs b/safekeeper/src/bin/safekeeper.rs index fecbb8bd41..0625538bf3 100644 --- a/safekeeper/src/bin/safekeeper.rs +++ b/safekeeper/src/bin/safekeeper.rs @@ -3,15 +3,19 @@ // use anyhow::{bail, Context, Result}; use clap::Parser; +use futures::future::BoxFuture; +use futures::stream::FuturesUnordered; +use futures::{FutureExt, StreamExt}; use remote_storage::RemoteStorageConfig; +use tokio::runtime::Handle; +use tokio::signal::unix::{signal, SignalKind}; +use tokio::task::JoinError; use toml_edit::Document; -use utils::signals::ShutdownSignals; use std::fs::{self, File}; use std::io::{ErrorKind, Write}; use std::path::{Path, PathBuf}; use std::sync::Arc; -use std::thread; use std::time::Duration; use storage_broker::Uri; use tokio::sync::mpsc; @@ -20,22 +24,21 @@ use tracing::*; use utils::pid_file; use metrics::set_build_info_metric; -use safekeeper::broker; -use safekeeper::control_file; use safekeeper::defaults::{ DEFAULT_HEARTBEAT_TIMEOUT, DEFAULT_HTTP_LISTEN_ADDR, DEFAULT_MAX_OFFLOADER_LAG_BYTES, DEFAULT_PG_LISTEN_ADDR, }; -use safekeeper::http; -use safekeeper::remove_wal; -use safekeeper::wal_backup; use safekeeper::wal_service; use safekeeper::GlobalTimelines; use safekeeper::SafeKeeperConf; +use safekeeper::{broker, WAL_SERVICE_RUNTIME}; +use safekeeper::{control_file, BROKER_RUNTIME}; +use safekeeper::{http, WAL_REMOVER_RUNTIME}; +use safekeeper::{remove_wal, WAL_BACKUP_RUNTIME}; +use safekeeper::{wal_backup, HTTP_RUNTIME}; use storage_broker::DEFAULT_ENDPOINT; use utils::auth::JwtAuth; use utils::{ - http::endpoint, id::NodeId, logging::{self, LogFormat}, project_git_version, @@ -104,10 +107,6 @@ struct Args { /// Safekeeper won't be elected for WAL offloading if it is lagging for more than this value in bytes #[arg(long, default_value_t = DEFAULT_MAX_OFFLOADER_LAG_BYTES)] max_offloader_lag: u64, - /// Number of threads for wal backup runtime, by default number of cores - /// available to the system. - #[arg(long)] - wal_backup_threads: Option, /// Number of max parallel WAL segments to be offloaded to remote storage. #[arg(long, default_value = "5")] wal_backup_parallel_jobs: usize, @@ -121,9 +120,14 @@ struct Args { /// Format for logging, either 'plain' or 'json'. #[arg(long, default_value = "plain")] log_format: String, + /// Run everything in single threaded current thread runtime, might be + /// useful for debugging. + #[arg(long)] + current_thread_runtime: bool, } -fn main() -> anyhow::Result<()> { +#[tokio::main(flavor = "current_thread")] +async fn main() -> anyhow::Result<()> { let args = Args::parse(); if let Some(addr) = args.dump_control_file { @@ -183,10 +187,10 @@ fn main() -> anyhow::Result<()> { heartbeat_timeout: args.heartbeat_timeout, remote_storage: args.remote_storage, max_offloader_lag_bytes: args.max_offloader_lag, - backup_runtime_threads: args.wal_backup_threads, wal_backup_enabled: !args.disable_wal_backup, backup_parallel_jobs: args.wal_backup_parallel_jobs, auth, + current_thread_runtime: args.current_thread_runtime, }; // initialize sentry if SENTRY_DSN is provided @@ -194,10 +198,14 @@ fn main() -> anyhow::Result<()> { Some(GIT_VERSION.into()), &[("node_id", &conf.my_id.to_string())], ); - start_safekeeper(conf) + start_safekeeper(conf).await } -fn start_safekeeper(conf: SafeKeeperConf) -> Result<()> { +/// Result of joining any of main tasks: upper error means task failed to +/// complete, e.g. panicked, inner is error produced by task itself. +type JoinTaskRes = Result, JoinError>; + +async fn start_safekeeper(conf: SafeKeeperConf) -> Result<()> { // Prevent running multiple safekeepers on the same directory let lock_file_path = conf.workdir.join(PID_FILE_NAME); let lock_file = @@ -208,14 +216,18 @@ fn start_safekeeper(conf: SafeKeeperConf) -> Result<()> { // we need to release the lock file only when the current process is gone std::mem::forget(lock_file); - let http_listener = tcp_listener::bind(conf.listen_http_addr.clone()).map_err(|e| { - error!("failed to bind to address {}: {}", conf.listen_http_addr, e); + info!("starting safekeeper WAL service on {}", conf.listen_pg_addr); + let pg_listener = tcp_listener::bind(conf.listen_pg_addr.clone()).map_err(|e| { + error!("failed to bind to address {}: {}", conf.listen_pg_addr, e); e })?; - info!("starting safekeeper on {}", conf.listen_pg_addr); - let pg_listener = tcp_listener::bind(conf.listen_pg_addr.clone()).map_err(|e| { - error!("failed to bind to address {}: {}", conf.listen_pg_addr, e); + info!( + "starting safekeeper HTTP service on {}", + conf.listen_http_addr + ); + let http_listener = tcp_listener::bind(conf.listen_http_addr.clone()).map_err(|e| { + error!("failed to bind to address {}: {}", conf.listen_http_addr, e); e })?; @@ -224,71 +236,88 @@ fn start_safekeeper(conf: SafeKeeperConf) -> Result<()> { let timeline_collector = safekeeper::metrics::TimelineCollector::new(); metrics::register_internal(Box::new(timeline_collector))?; - let mut threads = vec![]; let (wal_backup_launcher_tx, wal_backup_launcher_rx) = mpsc::channel(100); // Load all timelines from disk to memory. GlobalTimelines::init(conf.clone(), wal_backup_launcher_tx)?; - let conf_ = conf.clone(); - threads.push( - thread::Builder::new() - .name("http_endpoint_thread".into()) - .spawn(|| { - let router = http::make_router(conf_); - endpoint::serve_thread_main( - router, - http_listener, - std::future::pending(), // never shut down - ) - .unwrap(); - })?, - ); - - let conf_cloned = conf.clone(); - let safekeeper_thread = thread::Builder::new() - .name("WAL service thread".into()) - .spawn(|| wal_service::thread_main(conf_cloned, pg_listener)) - .unwrap(); - - threads.push(safekeeper_thread); + // Keep handles to main tasks to die if any of them disappears. + let mut tasks_handles: FuturesUnordered> = + FuturesUnordered::new(); let conf_ = conf.clone(); - threads.push( - thread::Builder::new() - .name("broker thread".into()) - .spawn(|| { - broker::thread_main(conf_); - })?, - ); + // Run everything in current thread rt, if asked. + if conf.current_thread_runtime { + info!("running in current thread runtime"); + } + let current_thread_rt = conf + .current_thread_runtime + .then(|| Handle::try_current().expect("no runtime in main")); + let wal_service_handle = current_thread_rt + .as_ref() + .unwrap_or_else(|| WAL_SERVICE_RUNTIME.handle()) + .spawn(wal_service::task_main(conf_, pg_listener)) + // wrap with task name for error reporting + .map(|res| ("WAL service main".to_owned(), res)); + tasks_handles.push(Box::pin(wal_service_handle)); let conf_ = conf.clone(); - threads.push( - thread::Builder::new() - .name("WAL removal thread".into()) - .spawn(|| { - remove_wal::thread_main(conf_); - })?, - ); + let http_handle = current_thread_rt + .as_ref() + .unwrap_or_else(|| HTTP_RUNTIME.handle()) + .spawn(http::task_main(conf_, http_listener)) + .map(|res| ("HTTP service main".to_owned(), res)); + tasks_handles.push(Box::pin(http_handle)); - threads.push( - thread::Builder::new() - .name("WAL backup launcher thread".into()) - .spawn(move || { - wal_backup::wal_backup_launcher_thread_main(conf, wal_backup_launcher_rx); - })?, - ); + let conf_ = conf.clone(); + let broker_task_handle = current_thread_rt + .as_ref() + .unwrap_or_else(|| BROKER_RUNTIME.handle()) + .spawn(broker::task_main(conf_).instrument(info_span!("broker"))) + .map(|res| ("broker main".to_owned(), res)); + tasks_handles.push(Box::pin(broker_task_handle)); + + let conf_ = conf.clone(); + let wal_remover_handle = current_thread_rt + .as_ref() + .unwrap_or_else(|| WAL_REMOVER_RUNTIME.handle()) + .spawn(remove_wal::task_main(conf_)) + .map(|res| ("WAL remover".to_owned(), res)); + tasks_handles.push(Box::pin(wal_remover_handle)); + + let conf_ = conf.clone(); + let wal_backup_handle = current_thread_rt + .as_ref() + .unwrap_or_else(|| WAL_BACKUP_RUNTIME.handle()) + .spawn(wal_backup::wal_backup_launcher_task_main( + conf_, + wal_backup_launcher_rx, + )) + .map(|res| ("WAL backup launcher".to_owned(), res)); + tasks_handles.push(Box::pin(wal_backup_handle)); set_build_info_metric(GIT_VERSION); - // TODO: put more thoughts into handling of failed threads - // We should catch & die if they are in trouble. - // On any shutdown signal, log receival and exit. Additionally, handling - // SIGQUIT prevents coredump. - ShutdownSignals::handle(|signal| { - info!("received {}, terminating", signal.name()); - std::process::exit(0); - }) + // TODO: update tokio-stream, convert to real async Stream with + // SignalStream, map it to obtain missing signal name, combine streams into + // single stream we can easily sit on. + let mut sigquit_stream = signal(SignalKind::quit())?; + let mut sigint_stream = signal(SignalKind::interrupt())?; + let mut sigterm_stream = signal(SignalKind::terminate())?; + + tokio::select! { + Some((task_name, res)) = tasks_handles.next()=> { + error!("{} task failed: {:?}, exiting", task_name, res); + std::process::exit(1); + } + // On any shutdown signal, log receival and exit. Additionally, handling + // SIGQUIT prevents coredump. + _ = sigquit_stream.recv() => info!("received SIGQUIT, terminating"), + _ = sigint_stream.recv() => info!("received SIGINT, terminating"), + _ = sigterm_stream.recv() => info!("received SIGTERM, terminating") + + }; + std::process::exit(0); } /// Determine safekeeper id. diff --git a/safekeeper/src/broker.rs b/safekeeper/src/broker.rs index 5e25d22ec1..2b1db2714b 100644 --- a/safekeeper/src/broker.rs +++ b/safekeeper/src/broker.rs @@ -8,7 +8,7 @@ use anyhow::Error; use anyhow::Result; use storage_broker::parse_proto_ttid; -use storage_broker::proto::broker_service_client::BrokerServiceClient; + use storage_broker::proto::subscribe_safekeeper_info_request::SubscriptionKey as ProtoSubscriptionKey; use storage_broker::proto::SubscribeSafekeeperInfoRequest; use storage_broker::Request; @@ -16,34 +16,23 @@ use storage_broker::Request; use std::time::Duration; use std::time::Instant; use tokio::task::JoinHandle; -use tokio::{runtime, time::sleep}; +use tokio::time::sleep; use tracing::*; +use crate::metrics::BROKER_ITERATION_TIMELINES; use crate::metrics::BROKER_PULLED_UPDATES; use crate::metrics::BROKER_PUSHED_UPDATES; +use crate::metrics::BROKER_PUSH_ALL_UPDATES_SECONDS; use crate::GlobalTimelines; use crate::SafeKeeperConf; const RETRY_INTERVAL_MSEC: u64 = 1000; const PUSH_INTERVAL_MSEC: u64 = 1000; -pub fn thread_main(conf: SafeKeeperConf) { - let runtime = runtime::Builder::new_current_thread() - .enable_all() - .build() - .unwrap(); - - let _enter = info_span!("broker").entered(); - info!("started, broker endpoint {:?}", conf.broker_endpoint); - - runtime.block_on(async { - main_loop(conf).await; - }); -} - /// Push once in a while data about all active timelines to the broker. async fn push_loop(conf: SafeKeeperConf) -> anyhow::Result<()> { - let mut client = BrokerServiceClient::connect(conf.broker_endpoint.clone()).await?; + let mut client = + storage_broker::connect(conf.broker_endpoint.clone(), conf.broker_keepalive_interval)?; let push_interval = Duration::from_millis(PUSH_INTERVAL_MSEC); let outbound = async_stream::stream! { @@ -53,16 +42,29 @@ async fn push_loop(conf: SafeKeeperConf) -> anyhow::Result<()> { // sensitive and there is no risk of deadlock as we don't await while // lock is held. let now = Instant::now(); - let mut active_tlis = GlobalTimelines::get_all(); - active_tlis.retain(|tli| tli.is_active()); - for tli in &active_tlis { - let sk_info = tli.get_safekeeper_info(&conf); + let all_tlis = GlobalTimelines::get_all(); + let mut n_pushed_tlis = 0; + for tli in &all_tlis { + // filtering alternative futures::stream::iter(all_tlis) + // .filter(|tli| {let tli = tli.clone(); async move { tli.is_active().await}}).collect::>().await; + // doesn't look better, and I'm not sure how to do that without collect. + if !tli.is_active().await { + continue; + } + let sk_info = tli.get_safekeeper_info(&conf).await; yield sk_info; BROKER_PUSHED_UPDATES.inc(); + n_pushed_tlis += 1; } let elapsed = now.elapsed(); - // Log duration every second. Should be about 10MB of logs per day. - info!("pushed {} timeline updates to broker in {:?}", active_tlis.len(), elapsed); + + BROKER_PUSH_ALL_UPDATES_SECONDS.observe(elapsed.as_secs_f64()); + BROKER_ITERATION_TIMELINES.observe(n_pushed_tlis as f64); + + if elapsed > push_interval / 2 { + info!("broker push is too long, pushed {} timeline updates to broker in {:?}", n_pushed_tlis, elapsed); + } + sleep(push_interval).await; } }; @@ -117,10 +119,13 @@ async fn pull_loop(conf: SafeKeeperConf) -> Result<()> { bail!("end of stream"); } -async fn main_loop(conf: SafeKeeperConf) { +pub async fn task_main(conf: SafeKeeperConf) -> anyhow::Result<()> { + info!("started, broker endpoint {:?}", conf.broker_endpoint); + let mut ticker = tokio::time::interval(Duration::from_millis(RETRY_INTERVAL_MSEC)); let mut push_handle: Option>> = None; let mut pull_handle: Option>> = None; + // Selecting on JoinHandles requires some squats; is there a better way to // reap tasks individually? diff --git a/safekeeper/src/control_file.rs b/safekeeper/src/control_file.rs index ba5e453e41..6c4ad24323 100644 --- a/safekeeper/src/control_file.rs +++ b/safekeeper/src/control_file.rs @@ -2,11 +2,13 @@ use anyhow::{bail, ensure, Context, Result}; use byteorder::{LittleEndian, ReadBytesExt, WriteBytesExt}; +use tokio::fs::{self, File}; +use tokio::io::AsyncWriteExt; -use std::fs::{self, File, OpenOptions}; -use std::io::{Read, Write}; +use std::io::Read; use std::ops::Deref; use std::path::{Path, PathBuf}; +use std::time::Instant; use crate::control_file_upgrade::upgrade_control_file; use crate::metrics::PERSIST_CONTROL_FILE_SECONDS; @@ -25,9 +27,13 @@ pub const CHECKSUM_SIZE: usize = std::mem::size_of::(); /// Storage should keep actual state inside of it. It should implement Deref /// trait to access state fields and have persist method for updating that state. +#[async_trait::async_trait] pub trait Storage: Deref { /// Persist safekeeper state on disk and update internal state. - fn persist(&mut self, s: &SafeKeeperState) -> Result<()>; + async fn persist(&mut self, s: &SafeKeeperState) -> Result<()>; + + /// Timestamp of last persist. + fn last_persist_at(&self) -> Instant; } #[derive(Debug)] @@ -38,6 +44,8 @@ pub struct FileStorage { /// Last state persisted to disk. state: SafeKeeperState, + /// Not preserved across restarts. + last_persist_at: Instant, } impl FileStorage { @@ -51,6 +59,7 @@ impl FileStorage { timeline_dir, conf: conf.clone(), state, + last_persist_at: Instant::now(), }) } @@ -66,6 +75,7 @@ impl FileStorage { timeline_dir, conf: conf.clone(), state, + last_persist_at: Instant::now(), }; Ok(store) @@ -74,7 +84,7 @@ impl FileStorage { /// Check the magic/version in the on-disk data and deserialize it, if possible. fn deser_sk_state(buf: &mut &[u8]) -> Result { // Read the version independent part - let magic = buf.read_u32::()?; + let magic = ReadBytesExt::read_u32::(buf)?; if magic != SK_MAGIC { bail!( "bad control file magic: {:X}, expected {:X}", @@ -82,7 +92,7 @@ impl FileStorage { SK_MAGIC ); } - let version = buf.read_u32::()?; + let version = ReadBytesExt::read_u32::(buf)?; if version == SK_FORMAT_VERSION { let res = SafeKeeperState::des(buf)?; return Ok(res); @@ -102,7 +112,7 @@ impl FileStorage { /// Read in the control file. pub fn load_control_file>(control_file_path: P) -> Result { - let mut control_file = OpenOptions::new() + let mut control_file = std::fs::OpenOptions::new() .read(true) .write(true) .open(&control_file_path) @@ -151,30 +161,31 @@ impl Deref for FileStorage { } } +#[async_trait::async_trait] impl Storage for FileStorage { /// persists state durably to underlying storage /// for description see https://lwn.net/Articles/457667/ - fn persist(&mut self, s: &SafeKeeperState) -> Result<()> { + async fn persist(&mut self, s: &SafeKeeperState) -> Result<()> { let _timer = PERSIST_CONTROL_FILE_SECONDS.start_timer(); // write data to safekeeper.control.partial let control_partial_path = self.timeline_dir.join(CONTROL_FILE_NAME_PARTIAL); - let mut control_partial = File::create(&control_partial_path).with_context(|| { + let mut control_partial = File::create(&control_partial_path).await.with_context(|| { format!( "failed to create partial control file at: {}", &control_partial_path.display() ) })?; let mut buf: Vec = Vec::new(); - buf.write_u32::(SK_MAGIC)?; - buf.write_u32::(SK_FORMAT_VERSION)?; + WriteBytesExt::write_u32::(&mut buf, SK_MAGIC)?; + WriteBytesExt::write_u32::(&mut buf, SK_FORMAT_VERSION)?; s.ser_into(&mut buf)?; // calculate checksum before resize let checksum = crc32c::crc32c(&buf); buf.extend_from_slice(&checksum.to_le_bytes()); - control_partial.write_all(&buf).with_context(|| { + control_partial.write_all(&buf).await.with_context(|| { format!( "failed to write safekeeper state into control file at: {}", control_partial_path.display() @@ -183,7 +194,7 @@ impl Storage for FileStorage { // fsync the file if !self.conf.no_sync { - control_partial.sync_all().with_context(|| { + control_partial.sync_all().await.with_context(|| { format!( "failed to sync partial control file at {}", control_partial_path.display() @@ -194,21 +205,22 @@ impl Storage for FileStorage { let control_path = self.timeline_dir.join(CONTROL_FILE_NAME); // rename should be atomic - fs::rename(&control_partial_path, &control_path)?; + fs::rename(&control_partial_path, &control_path).await?; // this sync is not required by any standard but postgres does this (see durable_rename) if !self.conf.no_sync { - File::open(&control_path) - .and_then(|f| f.sync_all()) - .with_context(|| { - format!( - "failed to sync control file at: {}", - &control_path.display() - ) - })?; + let new_f = File::open(&control_path).await?; + new_f.sync_all().await.with_context(|| { + format!( + "failed to sync control file at: {}", + &control_path.display() + ) + })?; // fsync the directory (linux specific) - File::open(&self.timeline_dir) - .and_then(|f| f.sync_all()) + let tli_dir = File::open(&self.timeline_dir).await?; + tli_dir + .sync_all() + .await .context("failed to sync control file directory")?; } @@ -216,6 +228,10 @@ impl Storage for FileStorage { self.state = s.clone(); Ok(()) } + + fn last_persist_at(&self) -> Instant { + self.last_persist_at + } } #[cfg(test)] @@ -224,7 +240,6 @@ mod test { use super::*; use crate::{safekeeper::SafeKeeperState, SafeKeeperConf}; use anyhow::Result; - use std::fs; use utils::{id::TenantTimelineId, lsn::Lsn}; fn stub_conf() -> SafeKeeperConf { @@ -235,59 +250,75 @@ mod test { } } - fn load_from_control_file( + async fn load_from_control_file( conf: &SafeKeeperConf, ttid: &TenantTimelineId, ) -> Result<(FileStorage, SafeKeeperState)> { - fs::create_dir_all(conf.timeline_dir(ttid)).expect("failed to create timeline dir"); + fs::create_dir_all(conf.timeline_dir(ttid)) + .await + .expect("failed to create timeline dir"); Ok(( FileStorage::restore_new(ttid, conf)?, FileStorage::load_control_file_conf(conf, ttid)?, )) } - fn create( + async fn create( conf: &SafeKeeperConf, ttid: &TenantTimelineId, ) -> Result<(FileStorage, SafeKeeperState)> { - fs::create_dir_all(conf.timeline_dir(ttid)).expect("failed to create timeline dir"); + fs::create_dir_all(conf.timeline_dir(ttid)) + .await + .expect("failed to create timeline dir"); let state = SafeKeeperState::empty(); let storage = FileStorage::create_new(ttid, conf, state.clone())?; Ok((storage, state)) } - #[test] - fn test_read_write_safekeeper_state() { + #[tokio::test] + async fn test_read_write_safekeeper_state() { let conf = stub_conf(); let ttid = TenantTimelineId::generate(); { - let (mut storage, mut state) = create(&conf, &ttid).expect("failed to create state"); + let (mut storage, mut state) = + create(&conf, &ttid).await.expect("failed to create state"); // change something state.commit_lsn = Lsn(42); - storage.persist(&state).expect("failed to persist state"); + storage + .persist(&state) + .await + .expect("failed to persist state"); } - let (_, state) = load_from_control_file(&conf, &ttid).expect("failed to read state"); + let (_, state) = load_from_control_file(&conf, &ttid) + .await + .expect("failed to read state"); assert_eq!(state.commit_lsn, Lsn(42)); } - #[test] - fn test_safekeeper_state_checksum_mismatch() { + #[tokio::test] + async fn test_safekeeper_state_checksum_mismatch() { let conf = stub_conf(); let ttid = TenantTimelineId::generate(); { - let (mut storage, mut state) = create(&conf, &ttid).expect("failed to read state"); + let (mut storage, mut state) = + create(&conf, &ttid).await.expect("failed to read state"); // change something state.commit_lsn = Lsn(42); - storage.persist(&state).expect("failed to persist state"); + storage + .persist(&state) + .await + .expect("failed to persist state"); } let control_path = conf.timeline_dir(&ttid).join(CONTROL_FILE_NAME); - let mut data = fs::read(&control_path).unwrap(); + let mut data = fs::read(&control_path).await.unwrap(); data[0] += 1; // change the first byte of the file to fail checksum validation - fs::write(&control_path, &data).expect("failed to write control file"); + fs::write(&control_path, &data) + .await + .expect("failed to write control file"); - match load_from_control_file(&conf, &ttid) { + match load_from_control_file(&conf, &ttid).await { Err(err) => assert!(err .to_string() .contains("safekeeper control file checksum mismatch")), diff --git a/safekeeper/src/debug_dump.rs b/safekeeper/src/debug_dump.rs index f711c4429d..387b577a13 100644 --- a/safekeeper/src/debug_dump.rs +++ b/safekeeper/src/debug_dump.rs @@ -121,7 +121,7 @@ pub struct FileInfo { } /// Build debug dump response, using the provided [`Args`] filters. -pub fn build(args: Args) -> Result { +pub async fn build(args: Args) -> Result { let start_time = Utc::now(); let timelines_count = GlobalTimelines::timelines_count(); @@ -155,7 +155,7 @@ pub fn build(args: Args) -> Result { } let control_file = if args.dump_control_file { - let mut state = tli.get_state().1; + let mut state = tli.get_state().await.1; if !args.dump_term_history { state.acceptor_state.term_history = TermHistory(vec![]); } @@ -165,7 +165,7 @@ pub fn build(args: Args) -> Result { }; let memory = if args.dump_memory { - Some(tli.memory_dump()) + Some(tli.memory_dump().await) } else { None }; diff --git a/safekeeper/src/handler.rs b/safekeeper/src/handler.rs index 7d25ced449..1367d5eebb 100644 --- a/safekeeper/src/handler.rs +++ b/safekeeper/src/handler.rs @@ -256,14 +256,14 @@ impl SafekeeperPostgresHandler { let lsn = if self.is_walproposer_recovery() { // walproposer should get all local WAL until flush_lsn - tli.get_flush_lsn() + tli.get_flush_lsn().await } else { // other clients shouldn't get any uncommitted WAL - tli.get_state().0.commit_lsn + tli.get_state().await.0.commit_lsn } .to_string(); - let sysid = tli.get_state().1.server.system_id.to_string(); + let sysid = tli.get_state().await.1.server.system_id.to_string(); let lsn_bytes = lsn.as_bytes(); let tli = PG_TLI.to_string(); let tli_bytes = tli.as_bytes(); diff --git a/safekeeper/src/http/mod.rs b/safekeeper/src/http/mod.rs index 1831470007..2a9570595f 100644 --- a/safekeeper/src/http/mod.rs +++ b/safekeeper/src/http/mod.rs @@ -2,3 +2,18 @@ pub mod routes; pub use routes::make_router; pub use safekeeper_api::models; + +use crate::SafeKeeperConf; + +pub async fn task_main( + conf: SafeKeeperConf, + http_listener: std::net::TcpListener, +) -> anyhow::Result<()> { + let router = make_router(conf) + .build() + .map_err(|err| anyhow::anyhow!(err))?; + let service = utils::http::RouterService::new(router).unwrap(); + let server = hyper::Server::from_tcp(http_listener)?; + server.serve(service).await?; + Ok(()) // unreachable +} diff --git a/safekeeper/src/http/routes.rs b/safekeeper/src/http/routes.rs index a498d868af..5cd0973ad6 100644 --- a/safekeeper/src/http/routes.rs +++ b/safekeeper/src/http/routes.rs @@ -13,7 +13,7 @@ use storage_broker::proto::SafekeeperTimelineInfo; use storage_broker::proto::TenantTimelineId as ProtoTenantTimelineId; use tokio::fs::File; use tokio::io::AsyncReadExt; -use tokio::task::JoinError; +use utils::http::endpoint::request_span; use crate::safekeeper::ServerInfo; use crate::safekeeper::Term; @@ -116,8 +116,8 @@ async fn timeline_status_handler(request: Request) -> Result) -> Result timeline_id, }; - let resp = tokio::task::spawn_blocking(move || { - debug_dump::build(args).map_err(ApiError::InternalServerError) - }) - .await - .map_err(|e: JoinError| ApiError::InternalServerError(e.into()))??; + let resp = debug_dump::build(args) + .await + .map_err(ApiError::InternalServerError)?; // TODO: use streaming response json_response(StatusCode::OK, resp) @@ -386,29 +379,32 @@ pub fn make_router(conf: SafeKeeperConf) -> RouterBuilder router .data(Arc::new(conf)) .data(auth) - .get("/v1/status", status_handler) + .get("/v1/status", |r| request_span(r, status_handler)) // Will be used in the future instead of implicit timeline creation - .post("/v1/tenant/timeline", timeline_create_handler) - .get( - "/v1/tenant/:tenant_id/timeline/:timeline_id", - timeline_status_handler, - ) - .delete( - "/v1/tenant/:tenant_id/timeline/:timeline_id", - timeline_delete_force_handler, - ) - .delete("/v1/tenant/:tenant_id", tenant_delete_force_handler) - .post("/v1/pull_timeline", timeline_pull_handler) + .post("/v1/tenant/timeline", |r| { + request_span(r, timeline_create_handler) + }) + .get("/v1/tenant/:tenant_id/timeline/:timeline_id", |r| { + request_span(r, timeline_status_handler) + }) + .delete("/v1/tenant/:tenant_id/timeline/:timeline_id", |r| { + request_span(r, timeline_delete_force_handler) + }) + .delete("/v1/tenant/:tenant_id", |r| { + request_span(r, tenant_delete_force_handler) + }) + .post("/v1/pull_timeline", |r| { + request_span(r, timeline_pull_handler) + }) .get( "/v1/tenant/:tenant_id/timeline/:timeline_id/file/:filename", - timeline_files_handler, + |r| request_span(r, timeline_files_handler), ) // for tests - .post( - "/v1/record_safekeeper_info/:tenant_id/:timeline_id", - record_safekeeper_info, - ) - .get("/v1/debug_dump", dump_debug_handler) + .post("/v1/record_safekeeper_info/:tenant_id/:timeline_id", |r| { + request_span(r, record_safekeeper_info) + }) + .get("/v1/debug_dump", |r| request_span(r, dump_debug_handler)) } #[cfg(test)] diff --git a/safekeeper/src/json_ctrl.rs b/safekeeper/src/json_ctrl.rs index dc9188723e..14d0cc3653 100644 --- a/safekeeper/src/json_ctrl.rs +++ b/safekeeper/src/json_ctrl.rs @@ -73,12 +73,12 @@ pub async fn handle_json_ctrl( // if send_proposer_elected is true, we need to update local history if append_request.send_proposer_elected { - send_proposer_elected(&tli, append_request.term, append_request.epoch_start_lsn)?; + send_proposer_elected(&tli, append_request.term, append_request.epoch_start_lsn).await?; } - let inserted_wal = append_logical_message(&tli, append_request)?; + let inserted_wal = append_logical_message(&tli, append_request).await?; let response = AppendResult { - state: tli.get_state().1, + state: tli.get_state().await.1, inserted_wal, }; let response_data = serde_json::to_vec(&response) @@ -114,9 +114,9 @@ async fn prepare_safekeeper( .await } -fn send_proposer_elected(tli: &Arc, term: Term, lsn: Lsn) -> anyhow::Result<()> { +async fn send_proposer_elected(tli: &Arc, term: Term, lsn: Lsn) -> anyhow::Result<()> { // add new term to existing history - let history = tli.get_state().1.acceptor_state.term_history; + let history = tli.get_state().await.1.acceptor_state.term_history; let history = history.up_to(lsn.checked_sub(1u64).unwrap()); let mut history_entries = history.0; history_entries.push(TermSwitchEntry { term, lsn }); @@ -129,7 +129,7 @@ fn send_proposer_elected(tli: &Arc, term: Term, lsn: Lsn) -> anyhow::R timeline_start_lsn: lsn, }); - tli.process_msg(&proposer_elected_request)?; + tli.process_msg(&proposer_elected_request).await?; Ok(()) } @@ -142,12 +142,12 @@ pub struct InsertedWAL { /// Extend local WAL with new LogicalMessage record. To do that, /// create AppendRequest with new WAL and pass it to safekeeper. -pub fn append_logical_message( +pub async fn append_logical_message( tli: &Arc, msg: &AppendLogicalMessage, ) -> anyhow::Result { let wal_data = encode_logical_message(&msg.lm_prefix, &msg.lm_message); - let sk_state = tli.get_state().1; + let sk_state = tli.get_state().await.1; let begin_lsn = msg.begin_lsn; let end_lsn = begin_lsn + wal_data.len() as u64; @@ -171,7 +171,7 @@ pub fn append_logical_message( wal_data: Bytes::from(wal_data), }); - let response = tli.process_msg(&append_request)?; + let response = tli.process_msg(&append_request).await?; let append_response = match response { Some(AcceptorProposerMessage::AppendResponse(resp)) => resp, diff --git a/safekeeper/src/lib.rs b/safekeeper/src/lib.rs index 22d6d57e19..b8e1101369 100644 --- a/safekeeper/src/lib.rs +++ b/safekeeper/src/lib.rs @@ -1,4 +1,6 @@ +use once_cell::sync::Lazy; use remote_storage::RemoteStorageConfig; +use tokio::runtime::Runtime; use std::path::PathBuf; use std::time::Duration; @@ -36,7 +38,6 @@ pub mod defaults { DEFAULT_PG_LISTEN_PORT, }; - pub const DEFAULT_WAL_BACKUP_RUNTIME_THREADS: usize = 8; pub const DEFAULT_HEARTBEAT_TIMEOUT: &str = "5000ms"; pub const DEFAULT_MAX_OFFLOADER_LAG_BYTES: u64 = 128 * (1 << 20); } @@ -60,10 +61,10 @@ pub struct SafeKeeperConf { pub heartbeat_timeout: Duration, pub remote_storage: Option, pub max_offloader_lag_bytes: u64, - pub backup_runtime_threads: Option, pub backup_parallel_jobs: usize, pub wal_backup_enabled: bool, pub auth: Option>, + pub current_thread_runtime: bool, } impl SafeKeeperConf { @@ -92,12 +93,64 @@ impl SafeKeeperConf { .parse() .expect("failed to parse default broker endpoint"), broker_keepalive_interval: Duration::from_secs(5), - backup_runtime_threads: None, wal_backup_enabled: true, backup_parallel_jobs: 1, auth: None, heartbeat_timeout: Duration::new(5, 0), max_offloader_lag_bytes: defaults::DEFAULT_MAX_OFFLOADER_LAG_BYTES, + current_thread_runtime: false, } } } + +// Tokio runtimes. +pub static WAL_SERVICE_RUNTIME: Lazy = Lazy::new(|| { + tokio::runtime::Builder::new_multi_thread() + .thread_name("WAL service worker") + .enable_all() + .build() + .expect("Failed to create WAL service runtime") +}); + +pub static HTTP_RUNTIME: Lazy = Lazy::new(|| { + tokio::runtime::Builder::new_multi_thread() + .thread_name("HTTP worker") + .enable_all() + .build() + .expect("Failed to create WAL service runtime") +}); + +pub static BROKER_RUNTIME: Lazy = Lazy::new(|| { + tokio::runtime::Builder::new_multi_thread() + .thread_name("broker worker") + .worker_threads(2) // there are only 2 tasks, having more threads doesn't make sense + .enable_all() + .build() + .expect("Failed to create broker runtime") +}); + +pub static WAL_REMOVER_RUNTIME: Lazy = Lazy::new(|| { + tokio::runtime::Builder::new_multi_thread() + .thread_name("WAL remover") + .worker_threads(1) + .enable_all() + .build() + .expect("Failed to create broker runtime") +}); + +pub static WAL_BACKUP_RUNTIME: Lazy = Lazy::new(|| { + tokio::runtime::Builder::new_multi_thread() + .thread_name("WAL backup worker") + .enable_all() + .build() + .expect("Failed to create WAL backup runtime") +}); + +pub static METRICS_SHIFTER_RUNTIME: Lazy = Lazy::new(|| { + tokio::runtime::Builder::new_multi_thread() + .thread_name("metric shifter") + .worker_threads(1) + .enable_all() + .build() + .expect("Failed to create broker runtime") +}); diff --git a/safekeeper/src/metrics.rs b/safekeeper/src/metrics.rs index 189af2b044..0711beb290 100644 --- a/safekeeper/src/metrics.rs +++ b/safekeeper/src/metrics.rs @@ -7,6 +7,7 @@ use std::{ use ::metrics::{register_histogram, GaugeVec, Histogram, IntGauge, DISK_WRITE_SECONDS_BUCKETS}; use anyhow::Result; +use futures::Future; use metrics::{ core::{AtomicU64, Collector, Desc, GenericCounter, GenericGaugeVec, Opts}, proto::MetricFamily, @@ -125,6 +126,25 @@ pub static BACKUP_ERRORS: Lazy = Lazy::new(|| { ) .expect("Failed to register safekeeper_backup_errors_total counter") }); +pub static BROKER_PUSH_ALL_UPDATES_SECONDS: Lazy = Lazy::new(|| { + register_histogram!( + "safekeeper_broker_push_update_seconds", + "Seconds to push all timeline updates to the broker", + DISK_WRITE_SECONDS_BUCKETS.to_vec() + ) + .expect("Failed to register safekeeper_broker_push_update_seconds histogram vec") +}); +pub const TIMELINES_COUNT_BUCKETS: &[f64] = &[ + 1.0, 10.0, 50.0, 100.0, 200.0, 500.0, 1000.0, 2000.0, 5000.0, 10000.0, 20000.0, 50000.0, +]; +pub static BROKER_ITERATION_TIMELINES: Lazy = Lazy::new(|| { + register_histogram!( + "safekeeper_broker_iteration_timelines", + "Count of timelines pushed to the broker in a single iteration", + TIMELINES_COUNT_BUCKETS.to_vec() + ) + .expect("Failed to register safekeeper_broker_iteration_timelines histogram vec") +}); pub const LABEL_UNKNOWN: &str = "unknown"; @@ -273,14 +293,17 @@ impl WalStorageMetrics { } } -/// Accepts a closure that returns a result, and returns the duration of the closure. -pub fn time_io_closure(closure: impl FnOnce() -> Result<()>) -> Result { +/// Accepts async function that returns empty anyhow result, and returns the duration of its execution. +pub async fn time_io_closure>( + closure: impl Future>, +) -> Result { let start = std::time::Instant::now(); - closure()?; + closure.await.map_err(|e| e.into())?; Ok(start.elapsed().as_secs_f64()) } /// Metrics for a single timeline. +#[derive(Clone)] pub struct FullTimelineInfo { pub ttid: TenantTimelineId, pub ps_feedback: PageserverFeedback, @@ -556,13 +579,19 @@ impl Collector for TimelineCollector { let timelines = GlobalTimelines::get_all(); let timelines_count = timelines.len(); - for arc_tli in timelines { - let tli = arc_tli.info_for_metrics(); - if tli.is_none() { - continue; - } - let tli = tli.unwrap(); + // Prometheus Collector is sync, and data is stored under async lock. To + // bridge the gap with a crutch, collect data in spawned thread with + // local tokio runtime. + let infos = std::thread::spawn(|| { + let rt = tokio::runtime::Builder::new_current_thread() + .build() + .expect("failed to create rt"); + rt.block_on(collect_timeline_metrics()) + }) + .join() + .expect("collect_timeline_metrics thread panicked"); + for tli in &infos { let tenant_id = tli.ttid.tenant_id.to_string(); let timeline_id = tli.ttid.timeline_id.to_string(); let labels = &[tenant_id.as_str(), timeline_id.as_str()]; @@ -663,3 +692,15 @@ impl Collector for TimelineCollector { mfs } } + +async fn collect_timeline_metrics() -> Vec { + let mut res = vec![]; + let timelines = GlobalTimelines::get_all(); + + for tli in timelines { + if let Some(info) = tli.info_for_metrics().await { + res.push(info); + } + } + res +} diff --git a/safekeeper/src/pull_timeline.rs b/safekeeper/src/pull_timeline.rs index 344b760fd3..61ba37efaa 100644 --- a/safekeeper/src/pull_timeline.rs +++ b/safekeeper/src/pull_timeline.rs @@ -231,7 +231,7 @@ async fn pull_timeline(status: TimelineStatus, host: String) -> Result info!( "Loaded timeline {}, flush_lsn={}", ttid, - tli.get_flush_lsn() + tli.get_flush_lsn().await ); Ok(Response { diff --git a/safekeeper/src/receive_wal.rs b/safekeeper/src/receive_wal.rs index 195470e3ca..a5e99c5f0a 100644 --- a/safekeeper/src/receive_wal.rs +++ b/safekeeper/src/receive_wal.rs @@ -18,15 +18,14 @@ use postgres_backend::QueryError; use pq_proto::BeMessage; use std::net::SocketAddr; use std::sync::Arc; -use std::thread; -use std::thread::JoinHandle; use tokio::io::AsyncRead; use tokio::io::AsyncWrite; use tokio::sync::mpsc::channel; use tokio::sync::mpsc::error::TryRecvError; use tokio::sync::mpsc::Receiver; use tokio::sync::mpsc::Sender; -use tokio::task::spawn_blocking; +use tokio::task; +use tokio::task::JoinHandle; use tokio::time::Duration; use tokio::time::Instant; use tracing::*; @@ -97,7 +96,7 @@ impl SafekeeperPostgresHandler { Err(res.expect_err("no error with WalAcceptor not spawn")) } Some(handle) => { - let wal_acceptor_res = handle.join(); + let wal_acceptor_res = handle.await; // If there was any network error, return it. res?; @@ -107,7 +106,7 @@ impl SafekeeperPostgresHandler { Ok(Ok(_)) => Ok(()), // can't happen currently; would be if we add graceful termination Ok(Err(e)) => Err(CopyStreamHandlerEnd::Other(e.context("WAL acceptor"))), Err(_) => Err(CopyStreamHandlerEnd::Other(anyhow!( - "WalAcceptor thread panicked", + "WalAcceptor task panicked", ))), } } @@ -154,10 +153,12 @@ impl<'a, IO: AsyncRead + AsyncWrite + Unpin> NetworkReader<'a, IO> { } }; - *self.acceptor_handle = Some( - WalAcceptor::spawn(tli.clone(), msg_rx, reply_tx, self.conn_id) - .context("spawn WalAcceptor thread")?, - ); + *self.acceptor_handle = Some(WalAcceptor::spawn( + tli.clone(), + msg_rx, + reply_tx, + self.conn_id, + )); // Forward all messages to WalAcceptor read_network_loop(self.pgb_reader, msg_tx, next_msg).await @@ -226,28 +227,19 @@ impl WalAcceptor { msg_rx: Receiver, reply_tx: Sender, conn_id: ConnectionId, - ) -> anyhow::Result>> { - let thread_name = format!("WAL acceptor {}", tli.ttid); - thread::Builder::new() - .name(thread_name) - .spawn(move || -> anyhow::Result<()> { - let mut wa = WalAcceptor { - tli, - msg_rx, - reply_tx, - }; + ) -> JoinHandle> { + task::spawn(async move { + let mut wa = WalAcceptor { + tli, + msg_rx, + reply_tx, + }; - let runtime = tokio::runtime::Builder::new_current_thread() - .enable_all() - .build()?; - - let span_ttid = wa.tli.ttid; // satisfy borrow checker - runtime.block_on( - wa.run() - .instrument(info_span!("WAL acceptor", cid = %conn_id, ttid = %span_ttid)), - ) - }) - .map_err(anyhow::Error::from) + let span_ttid = wa.tli.ttid; // satisfy borrow checker + wa.run() + .instrument(info_span!("WAL acceptor", cid = %conn_id, ttid = %span_ttid)) + .await + }) } /// The main loop. Returns Ok(()) if either msg_rx or reply_tx got closed; @@ -281,7 +273,7 @@ impl WalAcceptor { while let ProposerAcceptorMessage::AppendRequest(append_request) = next_msg { let noflush_msg = ProposerAcceptorMessage::NoFlushAppendRequest(append_request); - if let Some(reply) = self.tli.process_msg(&noflush_msg)? { + if let Some(reply) = self.tli.process_msg(&noflush_msg).await? { if self.reply_tx.send(reply).await.is_err() { return Ok(()); // chan closed, streaming terminated } @@ -300,10 +292,12 @@ impl WalAcceptor { } // flush all written WAL to the disk - self.tli.process_msg(&ProposerAcceptorMessage::FlushWAL)? + self.tli + .process_msg(&ProposerAcceptorMessage::FlushWAL) + .await? } else { // process message other than AppendRequest - self.tli.process_msg(&next_msg)? + self.tli.process_msg(&next_msg).await? }; if let Some(reply) = reply_msg { @@ -326,8 +320,8 @@ impl Drop for ComputeConnectionGuard { let tli = self.timeline.clone(); // tokio forbids to call blocking_send inside the runtime, and see // comments in on_compute_disconnect why we call blocking_send. - spawn_blocking(move || { - if let Err(e) = tli.on_compute_disconnect() { + tokio::spawn(async move { + if let Err(e) = tli.on_compute_disconnect().await { error!("failed to unregister compute connection: {}", e); } }); diff --git a/safekeeper/src/remove_wal.rs b/safekeeper/src/remove_wal.rs index b6d497f34e..3306f0b63a 100644 --- a/safekeeper/src/remove_wal.rs +++ b/safekeeper/src/remove_wal.rs @@ -1,26 +1,36 @@ //! Thread removing old WAL. -use std::{thread, time::Duration}; +use std::time::Duration; +use tokio::time::sleep; use tracing::*; use crate::{GlobalTimelines, SafeKeeperConf}; -pub fn thread_main(conf: SafeKeeperConf) { +pub async fn task_main(conf: SafeKeeperConf) -> anyhow::Result<()> { let wal_removal_interval = Duration::from_millis(5000); loop { let tlis = GlobalTimelines::get_all(); for tli in &tlis { - if !tli.is_active() { + if !tli.is_active().await { continue; } let ttid = tli.ttid; - let _enter = - info_span!("", tenant = %ttid.tenant_id, timeline = %ttid.timeline_id).entered(); - if let Err(e) = tli.remove_old_wal(conf.wal_backup_enabled) { - warn!("failed to remove WAL: {}", e); + if let Err(e) = tli + .maybe_persist_control_file() + .instrument(info_span!("", tenant = %ttid.tenant_id, timeline = %ttid.timeline_id)) + .await + { + warn!("failed to persist control file: {e}"); + } + if let Err(e) = tli + .remove_old_wal(conf.wal_backup_enabled) + .instrument(info_span!("", tenant = %ttid.tenant_id, timeline = %ttid.timeline_id)) + .await + { + error!("failed to remove WAL: {}", e); } } - thread::sleep(wal_removal_interval) + sleep(wal_removal_interval).await; } } diff --git a/safekeeper/src/safekeeper.rs b/safekeeper/src/safekeeper.rs index 33da0c8e5a..d0b14a1282 100644 --- a/safekeeper/src/safekeeper.rs +++ b/safekeeper/src/safekeeper.rs @@ -10,6 +10,7 @@ use std::cmp::max; use std::cmp::min; use std::fmt; use std::io::Read; +use std::time::Duration; use storage_broker::proto::SafekeeperTimelineInfo; use tracing::*; @@ -567,25 +568,27 @@ where /// Process message from proposer and possibly form reply. Concurrent /// callers must exclude each other. - pub fn process_msg( + pub async fn process_msg( &mut self, msg: &ProposerAcceptorMessage, ) -> Result> { match msg { - ProposerAcceptorMessage::Greeting(msg) => self.handle_greeting(msg), - ProposerAcceptorMessage::VoteRequest(msg) => self.handle_vote_request(msg), - ProposerAcceptorMessage::Elected(msg) => self.handle_elected(msg), - ProposerAcceptorMessage::AppendRequest(msg) => self.handle_append_request(msg, true), - ProposerAcceptorMessage::NoFlushAppendRequest(msg) => { - self.handle_append_request(msg, false) + ProposerAcceptorMessage::Greeting(msg) => self.handle_greeting(msg).await, + ProposerAcceptorMessage::VoteRequest(msg) => self.handle_vote_request(msg).await, + ProposerAcceptorMessage::Elected(msg) => self.handle_elected(msg).await, + ProposerAcceptorMessage::AppendRequest(msg) => { + self.handle_append_request(msg, true).await } - ProposerAcceptorMessage::FlushWAL => self.handle_flush(), + ProposerAcceptorMessage::NoFlushAppendRequest(msg) => { + self.handle_append_request(msg, false).await + } + ProposerAcceptorMessage::FlushWAL => self.handle_flush().await, } } /// Handle initial message from proposer: check its sanity and send my /// current term. - fn handle_greeting( + async fn handle_greeting( &mut self, msg: &ProposerGreeting, ) -> Result> { @@ -634,7 +637,8 @@ where } // system_id will be updated on mismatch - if self.state.server.system_id != msg.system_id { + // sync-safekeepers doesn't know sysid and sends 0, ignore it + if self.state.server.system_id != msg.system_id && msg.system_id != 0 { if self.state.server.system_id != 0 { warn!( "unexpected system ID arrived, got {}, expected {}", @@ -647,7 +651,7 @@ where if msg.pg_version != UNKNOWN_SERVER_VERSION { state.server.pg_version = msg.pg_version; } - self.state.persist(&state)?; + self.state.persist(&state).await?; } info!( @@ -662,7 +666,7 @@ where } /// Give vote for the given term, if we haven't done that previously. - fn handle_vote_request( + async fn handle_vote_request( &mut self, msg: &VoteRequest, ) -> Result> { @@ -676,7 +680,7 @@ where // handle_elected instead. Currently not a big deal, as proposer is the // only source of WAL; with peer2peer recovery it would be more // important. - self.wal_store.flush_wal()?; + self.wal_store.flush_wal().await?; // initialize with refusal let mut resp = VoteResponse { term: self.state.acceptor_state.term, @@ -690,7 +694,7 @@ where let mut state = self.state.clone(); state.acceptor_state.term = msg.term; // persist vote before sending it out - self.state.persist(&state)?; + self.state.persist(&state).await?; resp.term = self.state.acceptor_state.term; resp.vote_given = true as u64; @@ -713,12 +717,15 @@ where ar } - fn handle_elected(&mut self, msg: &ProposerElected) -> Result> { + async fn handle_elected( + &mut self, + msg: &ProposerElected, + ) -> Result> { info!("received ProposerElected {:?}", msg); if self.state.acceptor_state.term < msg.term { let mut state = self.state.clone(); state.acceptor_state.term = msg.term; - self.state.persist(&state)?; + self.state.persist(&state).await?; } // If our term is higher, ignore the message (next feedback will inform the compute) @@ -748,7 +755,7 @@ where // intersection of our history and history from msg // truncate wal, update the LSNs - self.wal_store.truncate_wal(msg.start_streaming_at)?; + self.wal_store.truncate_wal(msg.start_streaming_at).await?; // and now adopt term history from proposer { @@ -782,7 +789,7 @@ where self.inmem.backup_lsn = max(self.inmem.backup_lsn, state.timeline_start_lsn); state.acceptor_state.term_history = msg.term_history.clone(); - self.persist_control_file(state)?; + self.persist_control_file(state).await?; } info!("start receiving WAL since {:?}", msg.start_streaming_at); @@ -794,7 +801,7 @@ where /// /// Note: it is assumed that 'WAL we have is from the right term' check has /// already been done outside. - fn update_commit_lsn(&mut self, mut candidate: Lsn) -> Result<()> { + async fn update_commit_lsn(&mut self, mut candidate: Lsn) -> Result<()> { // Both peers and walproposer communicate this value, we might already // have a fresher (higher) version. candidate = max(candidate, self.inmem.commit_lsn); @@ -816,29 +823,52 @@ where // that we receive new epoch_start_lsn, and we still need to sync // control file in this case. if commit_lsn == self.epoch_start_lsn && self.state.commit_lsn != commit_lsn { - self.persist_control_file(self.state.clone())?; + self.persist_control_file(self.state.clone()).await?; } Ok(()) } /// Persist control file to disk, called only after timeline creation (bootstrap). - pub fn persist(&mut self) -> Result<()> { - self.persist_control_file(self.state.clone()) + pub async fn persist(&mut self) -> Result<()> { + self.persist_control_file(self.state.clone()).await } /// Persist in-memory state to the disk, taking other data from state. - fn persist_control_file(&mut self, mut state: SafeKeeperState) -> Result<()> { + async fn persist_control_file(&mut self, mut state: SafeKeeperState) -> Result<()> { state.commit_lsn = self.inmem.commit_lsn; state.backup_lsn = self.inmem.backup_lsn; state.peer_horizon_lsn = self.inmem.peer_horizon_lsn; state.proposer_uuid = self.inmem.proposer_uuid; - self.state.persist(&state) + self.state.persist(&state).await + } + + /// Persist control file if there is something to save and enough time + /// passed after the last save. + pub async fn maybe_persist_control_file( + &mut self, + inmem_remote_consistent_lsn: Lsn, + ) -> Result<()> { + const CF_SAVE_INTERVAL: Duration = Duration::from_secs(300); + if self.state.last_persist_at().elapsed() < CF_SAVE_INTERVAL { + return Ok(()); + } + let need_persist = self.inmem.commit_lsn > self.state.commit_lsn + || self.inmem.backup_lsn > self.state.backup_lsn + || self.inmem.peer_horizon_lsn > self.state.peer_horizon_lsn + || inmem_remote_consistent_lsn > self.state.remote_consistent_lsn; + if need_persist { + let mut state = self.state.clone(); + state.remote_consistent_lsn = inmem_remote_consistent_lsn; + self.persist_control_file(state).await?; + trace!("saved control file: {CF_SAVE_INTERVAL:?} passed"); + } + Ok(()) } /// Handle request to append WAL. #[allow(clippy::comparison_chain)] - fn handle_append_request( + async fn handle_append_request( &mut self, msg: &AppendRequest, require_flush: bool, @@ -861,17 +891,19 @@ where // do the job if !msg.wal_data.is_empty() { - self.wal_store.write_wal(msg.h.begin_lsn, &msg.wal_data)?; + self.wal_store + .write_wal(msg.h.begin_lsn, &msg.wal_data) + .await?; } // flush wal to the disk, if required if require_flush { - self.wal_store.flush_wal()?; + self.wal_store.flush_wal().await?; } // Update commit_lsn. if msg.h.commit_lsn != Lsn(0) { - self.update_commit_lsn(msg.h.commit_lsn)?; + self.update_commit_lsn(msg.h.commit_lsn).await?; } // Value calculated by walproposer can always lag: // - safekeepers can forget inmem value and send to proposer lower @@ -887,7 +919,7 @@ where if self.state.peer_horizon_lsn + (self.state.server.wal_seg_size as u64) < self.inmem.peer_horizon_lsn { - self.persist_control_file(self.state.clone())?; + self.persist_control_file(self.state.clone()).await?; } trace!( @@ -909,15 +941,15 @@ where } /// Flush WAL to disk. Return AppendResponse with latest LSNs. - fn handle_flush(&mut self) -> Result> { - self.wal_store.flush_wal()?; + async fn handle_flush(&mut self) -> Result> { + self.wal_store.flush_wal().await?; Ok(Some(AcceptorProposerMessage::AppendResponse( self.append_response(), ))) } /// Update timeline state with peer safekeeper data. - pub fn record_safekeeper_info(&mut self, sk_info: &SafekeeperTimelineInfo) -> Result<()> { + pub async fn record_safekeeper_info(&mut self, sk_info: &SafekeeperTimelineInfo) -> Result<()> { let mut sync_control_file = false; if (Lsn(sk_info.commit_lsn) != Lsn::INVALID) && (sk_info.last_log_term != INVALID_TERM) { @@ -925,7 +957,7 @@ where // commit_lsn if our history matches (is part of) history of advanced // commit_lsn provider. if sk_info.last_log_term == self.get_epoch() { - self.update_commit_lsn(Lsn(sk_info.commit_lsn))?; + self.update_commit_lsn(Lsn(sk_info.commit_lsn)).await?; } } @@ -948,11 +980,10 @@ where if sync_control_file { let mut state = self.state.clone(); - // Note: we do not persist remote_consistent_lsn in other paths of - // persisting cf -- that is not much needed currently. We could do - // that by storing Arc to walsenders in Safekeeper. + // Note: we could make remote_consistent_lsn update in cf common by + // storing Arc to walsenders in Safekeeper. state.remote_consistent_lsn = new_remote_consistent_lsn; - self.persist_control_file(state)?; + self.persist_control_file(state).await?; } Ok(()) } @@ -976,22 +1007,28 @@ where #[cfg(test)] mod tests { + use futures::future::BoxFuture; use postgres_ffi::WAL_SEGMENT_SIZE; use super::*; use crate::wal_storage::Storage; - use std::ops::Deref; + use std::{ops::Deref, time::Instant}; // fake storage for tests struct InMemoryState { persisted_state: SafeKeeperState, } + #[async_trait::async_trait] impl control_file::Storage for InMemoryState { - fn persist(&mut self, s: &SafeKeeperState) -> Result<()> { + async fn persist(&mut self, s: &SafeKeeperState) -> Result<()> { self.persisted_state = s.clone(); Ok(()) } + + fn last_persist_at(&self) -> Instant { + Instant::now() + } } impl Deref for InMemoryState { @@ -1014,27 +1051,28 @@ mod tests { lsn: Lsn, } + #[async_trait::async_trait] impl wal_storage::Storage for DummyWalStore { fn flush_lsn(&self) -> Lsn { self.lsn } - fn write_wal(&mut self, startpos: Lsn, buf: &[u8]) -> Result<()> { + async fn write_wal(&mut self, startpos: Lsn, buf: &[u8]) -> Result<()> { self.lsn = startpos + buf.len() as u64; Ok(()) } - fn truncate_wal(&mut self, end_pos: Lsn) -> Result<()> { + async fn truncate_wal(&mut self, end_pos: Lsn) -> Result<()> { self.lsn = end_pos; Ok(()) } - fn flush_wal(&mut self) -> Result<()> { + async fn flush_wal(&mut self) -> Result<()> { Ok(()) } - fn remove_up_to(&self) -> Box Result<()>> { - Box::new(move |_segno_up_to: XLogSegNo| Ok(())) + fn remove_up_to(&self, _segno_up_to: XLogSegNo) -> BoxFuture<'static, anyhow::Result<()>> { + Box::pin(async { Ok(()) }) } fn get_metrics(&self) -> crate::metrics::WalStorageMetrics { @@ -1042,8 +1080,8 @@ mod tests { } } - #[test] - fn test_voting() { + #[tokio::test] + async fn test_voting() { let storage = InMemoryState { persisted_state: test_sk_state(), }; @@ -1052,7 +1090,7 @@ mod tests { // check voting for 1 is ok let vote_request = ProposerAcceptorMessage::VoteRequest(VoteRequest { term: 1 }); - let mut vote_resp = sk.process_msg(&vote_request); + let mut vote_resp = sk.process_msg(&vote_request).await; match vote_resp.unwrap() { Some(AcceptorProposerMessage::VoteResponse(resp)) => assert!(resp.vote_given != 0), r => panic!("unexpected response: {:?}", r), @@ -1067,15 +1105,15 @@ mod tests { sk = SafeKeeper::new(storage, sk.wal_store, NodeId(0)).unwrap(); // and ensure voting second time for 1 is not ok - vote_resp = sk.process_msg(&vote_request); + vote_resp = sk.process_msg(&vote_request).await; match vote_resp.unwrap() { Some(AcceptorProposerMessage::VoteResponse(resp)) => assert!(resp.vote_given == 0), r => panic!("unexpected response: {:?}", r), } } - #[test] - fn test_epoch_switch() { + #[tokio::test] + async fn test_epoch_switch() { let storage = InMemoryState { persisted_state: test_sk_state(), }; @@ -1107,10 +1145,13 @@ mod tests { timeline_start_lsn: Lsn(0), }; sk.process_msg(&ProposerAcceptorMessage::Elected(pem)) + .await .unwrap(); // check that AppendRequest before epochStartLsn doesn't switch epoch - let resp = sk.process_msg(&ProposerAcceptorMessage::AppendRequest(append_request)); + let resp = sk + .process_msg(&ProposerAcceptorMessage::AppendRequest(append_request)) + .await; assert!(resp.is_ok()); assert_eq!(sk.get_epoch(), 0); @@ -1121,9 +1162,11 @@ mod tests { h: ar_hdr, wal_data: Bytes::from_static(b"b"), }; - let resp = sk.process_msg(&ProposerAcceptorMessage::AppendRequest(append_request)); + let resp = sk + .process_msg(&ProposerAcceptorMessage::AppendRequest(append_request)) + .await; assert!(resp.is_ok()); - sk.wal_store.truncate_wal(Lsn(3)).unwrap(); // imitate the complete record at 3 %) + sk.wal_store.truncate_wal(Lsn(3)).await.unwrap(); // imitate the complete record at 3 %) assert_eq!(sk.get_epoch(), 1); } } diff --git a/safekeeper/src/send_wal.rs b/safekeeper/src/send_wal.rs index fb420cba64..abca0a86b1 100644 --- a/safekeeper/src/send_wal.rs +++ b/safekeeper/src/send_wal.rs @@ -396,7 +396,7 @@ impl SafekeeperPostgresHandler { // on this safekeeper itself. That's ok as (old) proposer will never be // able to commit such WAL. let stop_pos: Option = if self.is_walproposer_recovery() { - let wal_end = tli.get_flush_lsn(); + let wal_end = tli.get_flush_lsn().await; Some(wal_end) } else { None @@ -418,7 +418,7 @@ impl SafekeeperPostgresHandler { // switch to copy pgb.write_message(&BeMessage::CopyBothResponse).await?; - let (_, persisted_state) = tli.get_state(); + let (_, persisted_state) = tli.get_state().await; let wal_reader = WalReader::new( self.conf.workdir.clone(), self.conf.timeline_dir(&tli.ttid), @@ -562,7 +562,7 @@ impl WalSender<'_, IO> { .walsenders .get_ws_remote_consistent_lsn(self.ws_guard.id) { - if self.tli.should_walsender_stop(remote_consistent_lsn) { + if self.tli.should_walsender_stop(remote_consistent_lsn).await { // Terminate if there is nothing more to send. return Err(CopyStreamHandlerEnd::ServerInitiated(format!( "ending streaming to {:?} at {}, receiver is caughtup and there is no computes", diff --git a/safekeeper/src/timeline.rs b/safekeeper/src/timeline.rs index 2dbf215998..30036cc7f2 100644 --- a/safekeeper/src/timeline.rs +++ b/safekeeper/src/timeline.rs @@ -2,12 +2,13 @@ //! to glue together SafeKeeper and all other background services. use anyhow::{anyhow, bail, Result}; -use parking_lot::{Mutex, MutexGuard}; use postgres_ffi::XLogSegNo; +use tokio::fs; use std::cmp::max; use std::path::PathBuf; use std::sync::Arc; +use tokio::sync::{Mutex, MutexGuard}; use tokio::{ sync::{mpsc::Sender, watch}, time::Instant, @@ -234,7 +235,6 @@ impl SharedState { flush_lsn: self.sk.wal_store.flush_lsn().0, // note: this value is not flushed to control file yet and can be lost commit_lsn: self.sk.inmem.commit_lsn.0, - // TODO: rework feedbacks to avoid max here remote_consistent_lsn: remote_consistent_lsn.0, peer_horizon_lsn: self.sk.inmem.peer_horizon_lsn.0, safekeeper_connstr: conf.listen_pg_addr.clone(), @@ -266,7 +266,7 @@ impl From for ApiError { fn from(te: TimelineError) -> ApiError { match te { TimelineError::NotFound(ttid) => { - ApiError::NotFound(anyhow!("timeline {} not found", ttid)) + ApiError::NotFound(anyhow!("timeline {} not found", ttid).into()) } _ => ApiError::InternalServerError(anyhow!("{}", te)), } @@ -287,8 +287,9 @@ pub struct Timeline { commit_lsn_watch_tx: watch::Sender, commit_lsn_watch_rx: watch::Receiver, - /// Safekeeper and other state, that should remain consistent and synchronized - /// with the disk. + /// Safekeeper and other state, that should remain consistent and + /// synchronized with the disk. This is tokio mutex as we write WAL to disk + /// while holding it, ensuring that consensus checks are in order. mutex: Mutex, walsenders: Arc, @@ -362,8 +363,8 @@ impl Timeline { /// /// Bootstrap is transactional, so if it fails, created files will be deleted, /// and state on disk should remain unchanged. - pub fn bootstrap(&self, shared_state: &mut MutexGuard) -> Result<()> { - match std::fs::metadata(&self.timeline_dir) { + pub async fn bootstrap(&self, shared_state: &mut MutexGuard<'_, SharedState>) -> Result<()> { + match fs::metadata(&self.timeline_dir).await { Ok(_) => { // Timeline directory exists on disk, we should leave state unchanged // and return error. @@ -376,53 +377,51 @@ impl Timeline { } // Create timeline directory. - std::fs::create_dir_all(&self.timeline_dir)?; + fs::create_dir_all(&self.timeline_dir).await?; // Write timeline to disk and TODO: start background tasks. - match || -> Result<()> { - shared_state.sk.persist()?; - // TODO: add more initialization steps here - self.update_status(shared_state); - Ok(()) - }() { - Ok(_) => Ok(()), - Err(e) => { - // Bootstrap failed, cancel timeline and remove timeline directory. - self.cancel(shared_state); + if let Err(e) = shared_state.sk.persist().await { + // Bootstrap failed, cancel timeline and remove timeline directory. + self.cancel(shared_state); - if let Err(fs_err) = std::fs::remove_dir_all(&self.timeline_dir) { - warn!( - "failed to remove timeline {} directory after bootstrap failure: {}", - self.ttid, fs_err - ); - } - - Err(e) + if let Err(fs_err) = fs::remove_dir_all(&self.timeline_dir).await { + warn!( + "failed to remove timeline {} directory after bootstrap failure: {}", + self.ttid, fs_err + ); } + + return Err(e); } + + // TODO: add more initialization steps here + self.update_status(shared_state); + Ok(()) } /// Delete timeline from disk completely, by removing timeline directory. Background /// timeline activities will stop eventually. - pub fn delete_from_disk( + pub async fn delete_from_disk( &self, - shared_state: &mut MutexGuard, + shared_state: &mut MutexGuard<'_, SharedState>, ) -> Result<(bool, bool)> { let was_active = shared_state.active; self.cancel(shared_state); - let dir_existed = delete_dir(&self.timeline_dir)?; + let dir_existed = delete_dir(&self.timeline_dir).await?; Ok((dir_existed, was_active)) } /// Cancel timeline to prevent further usage. Background tasks will stop /// eventually after receiving cancellation signal. - fn cancel(&self, shared_state: &mut MutexGuard) { + /// + /// Note that we can't notify backup launcher here while holding + /// shared_state lock, as this is a potential deadlock: caller is + /// responsible for that. Generally we should probably make WAL backup tasks + /// to shut down on their own, checking once in a while whether it is the + /// time. + fn cancel(&self, shared_state: &mut MutexGuard<'_, SharedState>) { info!("timeline {} is cancelled", self.ttid); let _ = self.cancellation_tx.send(true); - let res = self.wal_backup_launcher_tx.blocking_send(self.ttid); - if let Err(e) = res { - error!("Failed to send stop signal to wal_backup_launcher: {}", e); - } // Close associated FDs. Nobody will be able to touch timeline data once // it is cancelled, so WAL storage won't be opened again. shared_state.sk.wal_store.close(); @@ -434,8 +433,8 @@ impl Timeline { } /// Take a writing mutual exclusive lock on timeline shared_state. - pub fn write_shared_state(&self) -> MutexGuard { - self.mutex.lock() + pub async fn write_shared_state(&self) -> MutexGuard { + self.mutex.lock().await } fn update_status(&self, shared_state: &mut SharedState) -> bool { @@ -451,7 +450,7 @@ impl Timeline { let is_wal_backup_action_pending: bool; { - let mut shared_state = self.write_shared_state(); + let mut shared_state = self.write_shared_state().await; shared_state.num_computes += 1; is_wal_backup_action_pending = self.update_status(&mut shared_state); } @@ -465,22 +464,17 @@ impl Timeline { /// De-register compute connection, shutting down timeline activity if /// pageserver doesn't need catchup. - pub fn on_compute_disconnect(&self) -> Result<()> { + pub async fn on_compute_disconnect(&self) -> Result<()> { let is_wal_backup_action_pending: bool; { - let mut shared_state = self.write_shared_state(); + let mut shared_state = self.write_shared_state().await; shared_state.num_computes -= 1; is_wal_backup_action_pending = self.update_status(&mut shared_state); } // Wake up wal backup launcher, if it is time to stop the offloading. if is_wal_backup_action_pending { // Can fail only if channel to a static thread got closed, which is not normal at all. - // - // Note: this is blocking_send because on_compute_disconnect is called in Drop, there is - // no async Drop and we use current thread runtimes. With current thread rt spawning - // task in drop impl is racy, as thread along with runtime might finish before the task. - // This should be switched send.await when/if we go to full async. - self.wal_backup_launcher_tx.blocking_send(self.ttid)?; + self.wal_backup_launcher_tx.send(self.ttid).await?; } Ok(()) } @@ -490,11 +484,11 @@ impl Timeline { /// computes. While there might be nothing to stream already, we learn about /// remote_consistent_lsn update through replication feedback, and we want /// to stop pushing to the broker if pageserver is fully caughtup. - pub fn should_walsender_stop(&self, reported_remote_consistent_lsn: Lsn) -> bool { + pub async fn should_walsender_stop(&self, reported_remote_consistent_lsn: Lsn) -> bool { if self.is_cancelled() { return true; } - let shared_state = self.write_shared_state(); + let shared_state = self.write_shared_state().await; if shared_state.num_computes == 0 { return shared_state.sk.inmem.commit_lsn == Lsn(0) || // no data at all yet reported_remote_consistent_lsn >= shared_state.sk.inmem.commit_lsn; @@ -504,12 +498,12 @@ impl Timeline { /// Returns whether s3 offloading is required and sets current status as /// matching it. - pub fn wal_backup_attend(&self) -> bool { + pub async fn wal_backup_attend(&self) -> bool { if self.is_cancelled() { return false; } - self.write_shared_state().wal_backup_attend() + self.write_shared_state().await.wal_backup_attend() } /// Returns commit_lsn watch channel. @@ -518,7 +512,7 @@ impl Timeline { } /// Pass arrived message to the safekeeper. - pub fn process_msg( + pub async fn process_msg( &self, msg: &ProposerAcceptorMessage, ) -> Result> { @@ -529,8 +523,8 @@ impl Timeline { let mut rmsg: Option; let commit_lsn: Lsn; { - let mut shared_state = self.write_shared_state(); - rmsg = shared_state.sk.process_msg(msg)?; + let mut shared_state = self.write_shared_state().await; + rmsg = shared_state.sk.process_msg(msg).await?; // if this is AppendResponse, fill in proper pageserver and hot // standby feedback. @@ -547,37 +541,37 @@ impl Timeline { } /// Returns wal_seg_size. - pub fn get_wal_seg_size(&self) -> usize { - self.write_shared_state().get_wal_seg_size() + pub async fn get_wal_seg_size(&self) -> usize { + self.write_shared_state().await.get_wal_seg_size() } /// Returns true only if the timeline is loaded and active. - pub fn is_active(&self) -> bool { + pub async fn is_active(&self) -> bool { if self.is_cancelled() { return false; } - self.write_shared_state().active + self.write_shared_state().await.active } /// Returns state of the timeline. - pub fn get_state(&self) -> (SafekeeperMemState, SafeKeeperState) { - let state = self.write_shared_state(); + pub async fn get_state(&self) -> (SafekeeperMemState, SafeKeeperState) { + let state = self.write_shared_state().await; (state.sk.inmem.clone(), state.sk.state.clone()) } /// Returns latest backup_lsn. - pub fn get_wal_backup_lsn(&self) -> Lsn { - self.write_shared_state().sk.inmem.backup_lsn + pub async fn get_wal_backup_lsn(&self) -> Lsn { + self.write_shared_state().await.sk.inmem.backup_lsn } /// Sets backup_lsn to the given value. - pub fn set_wal_backup_lsn(&self, backup_lsn: Lsn) -> Result<()> { + pub async fn set_wal_backup_lsn(&self, backup_lsn: Lsn) -> Result<()> { if self.is_cancelled() { bail!(TimelineError::Cancelled(self.ttid)); } - let mut state = self.write_shared_state(); + let mut state = self.write_shared_state().await; state.sk.inmem.backup_lsn = max(state.sk.inmem.backup_lsn, backup_lsn); // we should check whether to shut down offloader, but this will be done // soon by peer communication anyway. @@ -585,8 +579,8 @@ impl Timeline { } /// Get safekeeper info for broadcasting to broker and other peers. - pub fn get_safekeeper_info(&self, conf: &SafeKeeperConf) -> SafekeeperTimelineInfo { - let shared_state = self.write_shared_state(); + pub async fn get_safekeeper_info(&self, conf: &SafeKeeperConf) -> SafekeeperTimelineInfo { + let shared_state = self.write_shared_state().await; shared_state.get_safekeeper_info( &self.ttid, conf, @@ -605,8 +599,8 @@ impl Timeline { let is_wal_backup_action_pending: bool; let commit_lsn: Lsn; { - let mut shared_state = self.write_shared_state(); - shared_state.sk.record_safekeeper_info(&sk_info)?; + let mut shared_state = self.write_shared_state().await; + shared_state.sk.record_safekeeper_info(&sk_info).await?; let peer_info = PeerInfo::from_sk_info(&sk_info, Instant::now()); shared_state.peers_info.upsert(&peer_info); is_wal_backup_action_pending = self.update_status(&mut shared_state); @@ -623,8 +617,8 @@ impl Timeline { /// Get our latest view of alive peers status on the timeline. /// We pass our own info through the broker as well, so when we don't have connection /// to the broker returned vec is empty. - pub fn get_peers(&self, conf: &SafeKeeperConf) -> Vec { - let shared_state = self.write_shared_state(); + pub async fn get_peers(&self, conf: &SafeKeeperConf) -> Vec { + let shared_state = self.write_shared_state().await; let now = Instant::now(); shared_state .peers_info @@ -641,47 +635,60 @@ impl Timeline { } /// Returns flush_lsn. - pub fn get_flush_lsn(&self) -> Lsn { - self.write_shared_state().sk.wal_store.flush_lsn() + pub async fn get_flush_lsn(&self) -> Lsn { + self.write_shared_state().await.sk.wal_store.flush_lsn() } /// Delete WAL segments from disk that are no longer needed. This is determined /// based on pageserver's remote_consistent_lsn and local backup_lsn/peer_lsn. - pub fn remove_old_wal(&self, wal_backup_enabled: bool) -> Result<()> { + pub async fn remove_old_wal(&self, wal_backup_enabled: bool) -> Result<()> { if self.is_cancelled() { bail!(TimelineError::Cancelled(self.ttid)); } let horizon_segno: XLogSegNo; - let remover: Box Result<(), anyhow::Error>>; - { - let shared_state = self.write_shared_state(); + let remover = { + let shared_state = self.write_shared_state().await; horizon_segno = shared_state.sk.get_horizon_segno(wal_backup_enabled); - remover = shared_state.sk.wal_store.remove_up_to(); if horizon_segno <= 1 || horizon_segno <= shared_state.last_removed_segno { - return Ok(()); + return Ok(()); // nothing to do } + let remover = shared_state.sk.wal_store.remove_up_to(horizon_segno - 1); // release the lock before removing - } + remover + }; // delete old WAL files - remover(horizon_segno - 1)?; + remover.await?; // update last_removed_segno - let mut shared_state = self.write_shared_state(); + let mut shared_state = self.write_shared_state().await; shared_state.last_removed_segno = horizon_segno; Ok(()) } - /// Returns full timeline info, required for the metrics. If the timeline is - /// not active, returns None instead. - pub fn info_for_metrics(&self) -> Option { + /// Persist control file if there is something to save and enough time + /// passed after the last save. This helps to keep remote_consistent_lsn up + /// to date so that storage nodes restart doesn't cause many pageserver -> + /// safekeeper reconnections. + pub async fn maybe_persist_control_file(&self) -> Result<()> { + let remote_consistent_lsn = self.walsenders.get_remote_consistent_lsn(); + self.write_shared_state() + .await + .sk + .maybe_persist_control_file(remote_consistent_lsn) + .await + } + + /// Gather timeline data for metrics. If the timeline is not active, returns + /// None, we do not collect these. + pub async fn info_for_metrics(&self) -> Option { if self.is_cancelled() { return None; } let ps_feedback = self.walsenders.get_ps_feedback(); - let state = self.write_shared_state(); + let state = self.write_shared_state().await; if state.active { Some(FullTimelineInfo { ttid: self.ttid, @@ -703,8 +710,8 @@ impl Timeline { } /// Returns in-memory timeline state to build a full debug dump. - pub fn memory_dump(&self) -> debug_dump::Memory { - let state = self.write_shared_state(); + pub async fn memory_dump(&self) -> debug_dump::Memory { + let state = self.write_shared_state().await; let (write_lsn, write_record_lsn, flush_lsn, file_open) = state.sk.wal_store.internal_state(); @@ -728,8 +735,8 @@ impl Timeline { } /// Deletes directory and it's contents. Returns false if directory does not exist. -fn delete_dir(path: &PathBuf) -> Result { - match std::fs::remove_dir_all(path) { +async fn delete_dir(path: &PathBuf) -> Result { + match fs::remove_dir_all(path).await { Ok(_) => Ok(true), Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(false), Err(e) => Err(e.into()), diff --git a/safekeeper/src/timelines_global_map.rs b/safekeeper/src/timelines_global_map.rs index 41809794dc..f2d5df8744 100644 --- a/safekeeper/src/timelines_global_map.rs +++ b/safekeeper/src/timelines_global_map.rs @@ -113,9 +113,17 @@ impl GlobalTimelines { Ok(()) } - /// Loads all timelines for the given tenant to memory. Returns fs::read_dir errors if any. + /// Loads all timelines for the given tenant to memory. Returns fs::read_dir + /// errors if any. + /// + /// Note: This function (and all reading/loading below) is sync because + /// timelines are loaded while holding GlobalTimelinesState lock. Which is + /// fine as this is called only from single threaded main runtime on boot, + /// but clippy complains anyway, and suppressing that isn't trivial as async + /// is the keyword, ha. That only other user is pull_timeline.rs for which + /// being blocked is not that bad, and we can do spawn_blocking. fn load_tenant_timelines( - state: &mut MutexGuard, + state: &mut MutexGuard<'_, GlobalTimelinesState>, tenant_id: TenantId, ) -> Result<()> { let timelines_dir = state.get_conf().tenant_dir(&tenant_id); @@ -220,7 +228,7 @@ impl GlobalTimelines { // Take a lock and finish the initialization holding this mutex. No other threads // can interfere with creation after we will insert timeline into the map. { - let mut shared_state = timeline.write_shared_state(); + let mut shared_state = timeline.write_shared_state().await; // We can get a race condition here in case of concurrent create calls, but only // in theory. create() will return valid timeline on the next try. @@ -232,7 +240,7 @@ impl GlobalTimelines { // Write the new timeline to the disk and start background workers. // Bootstrap is transactional, so if it fails, the timeline will be deleted, // and the state on disk should remain unchanged. - if let Err(e) = timeline.bootstrap(&mut shared_state) { + if let Err(e) = timeline.bootstrap(&mut shared_state).await { // Note: the most likely reason for bootstrap failure is that the timeline // directory already exists on disk. This happens when timeline is corrupted // and wasn't loaded from disk on startup because of that. We want to preserve @@ -294,15 +302,16 @@ impl GlobalTimelines { } /// Cancels timeline, then deletes the corresponding data directory. - pub fn delete_force(ttid: &TenantTimelineId) -> Result { + pub async fn delete_force(ttid: &TenantTimelineId) -> Result { let tli_res = TIMELINES_STATE.lock().unwrap().get(ttid); match tli_res { Ok(timeline) => { // Take a lock and finish the deletion holding this mutex. - let mut shared_state = timeline.write_shared_state(); + let mut shared_state = timeline.write_shared_state().await; info!("deleting timeline {}", ttid); - let (dir_existed, was_active) = timeline.delete_from_disk(&mut shared_state)?; + let (dir_existed, was_active) = + timeline.delete_from_disk(&mut shared_state).await?; // Remove timeline from the map. // FIXME: re-enable it once we fix the issue with recreation of deleted timelines @@ -335,7 +344,7 @@ impl GlobalTimelines { /// the tenant had, `true` if a timeline was active. There may be a race if new timelines are /// created simultaneously. In that case the function will return error and the caller should /// retry tenant deletion again later. - pub fn delete_force_all_for_tenant( + pub async fn delete_force_all_for_tenant( tenant_id: &TenantId, ) -> Result> { info!("deleting all timelines for tenant {}", tenant_id); @@ -345,7 +354,7 @@ impl GlobalTimelines { let mut deleted = HashMap::new(); for tli in &to_delete { - match Self::delete_force(&tli.ttid) { + match Self::delete_force(&tli.ttid).await { Ok(result) => { deleted.insert(tli.ttid, result); } diff --git a/safekeeper/src/wal_backup.rs b/safekeeper/src/wal_backup.rs index 4d341a7ef8..eae3f3fe86 100644 --- a/safekeeper/src/wal_backup.rs +++ b/safekeeper/src/wal_backup.rs @@ -17,7 +17,6 @@ use postgres_ffi::XLogFileName; use postgres_ffi::{XLogSegNo, PG_TLI}; use remote_storage::{GenericRemoteStorage, RemotePath}; use tokio::fs::File; -use tokio::runtime::Builder; use tokio::select; use tokio::sync::mpsc::{self, Receiver, Sender}; @@ -36,30 +35,16 @@ use once_cell::sync::OnceCell; const UPLOAD_FAILURE_RETRY_MIN_MS: u64 = 10; const UPLOAD_FAILURE_RETRY_MAX_MS: u64 = 5000; -pub fn wal_backup_launcher_thread_main( - conf: SafeKeeperConf, - wal_backup_launcher_rx: Receiver, -) { - let mut builder = Builder::new_multi_thread(); - if let Some(num_threads) = conf.backup_runtime_threads { - builder.worker_threads(num_threads); - } - let rt = builder - .enable_all() - .build() - .expect("failed to create wal backup runtime"); - - rt.block_on(async { - wal_backup_launcher_main_loop(conf, wal_backup_launcher_rx).await; - }); -} - /// Check whether wal backup is required for timeline. If yes, mark that launcher is /// aware of current status and return the timeline. -fn is_wal_backup_required(ttid: TenantTimelineId) -> Option> { - GlobalTimelines::get(ttid) - .ok() - .filter(|tli| tli.wal_backup_attend()) +async fn is_wal_backup_required(ttid: TenantTimelineId) -> Option> { + match GlobalTimelines::get(ttid).ok() { + Some(tli) => { + tli.wal_backup_attend().await; + Some(tli) + } + None => None, + } } struct WalBackupTaskHandle { @@ -143,8 +128,8 @@ async fn update_task( ttid: TenantTimelineId, entry: &mut WalBackupTimelineEntry, ) { - let alive_peers = entry.timeline.get_peers(conf); - let wal_backup_lsn = entry.timeline.get_wal_backup_lsn(); + let alive_peers = entry.timeline.get_peers(conf).await; + let wal_backup_lsn = entry.timeline.get_wal_backup_lsn().await; let (offloader, election_dbg_str) = determine_offloader(&alive_peers, wal_backup_lsn, ttid, conf); let elected_me = Some(conf.my_id) == offloader; @@ -183,10 +168,10 @@ const CHECK_TASKS_INTERVAL_MSEC: u64 = 1000; /// Sits on wal_backup_launcher_rx and starts/stops per timeline wal backup /// tasks. Having this in separate task simplifies locking, allows to reap /// panics and separate elections from offloading itself. -async fn wal_backup_launcher_main_loop( +pub async fn wal_backup_launcher_task_main( conf: SafeKeeperConf, mut wal_backup_launcher_rx: Receiver, -) { +) -> anyhow::Result<()> { info!( "WAL backup launcher started, remote config {:?}", conf.remote_storage @@ -214,7 +199,7 @@ async fn wal_backup_launcher_main_loop( if conf.remote_storage.is_none() || !conf.wal_backup_enabled { continue; /* just drain the channel and do nothing */ } - let timeline = is_wal_backup_required(ttid); + let timeline = is_wal_backup_required(ttid).await; // do we need to do anything at all? if timeline.is_some() != tasks.contains_key(&ttid) { if let Some(timeline) = timeline { @@ -269,7 +254,7 @@ async fn backup_task_main( let tli = res.unwrap(); let mut wb = WalBackupTask { - wal_seg_size: tli.get_wal_seg_size(), + wal_seg_size: tli.get_wal_seg_size().await, commit_lsn_watch_rx: tli.get_commit_lsn_watch_rx(), timeline: tli, timeline_dir, @@ -326,7 +311,7 @@ impl WalBackupTask { continue; /* nothing to do, common case as we wake up on every commit_lsn bump */ } // Perhaps peers advanced the position, check shmem value. - backup_lsn = self.timeline.get_wal_backup_lsn(); + backup_lsn = self.timeline.get_wal_backup_lsn().await; if backup_lsn.segment_number(self.wal_seg_size) >= commit_lsn.segment_number(self.wal_seg_size) { @@ -402,6 +387,7 @@ pub async fn backup_lsn_range( let new_backup_lsn = segment.end_lsn; timeline .set_wal_backup_lsn(new_backup_lsn) + .await .context("setting wal_backup_lsn")?; *backup_lsn = new_backup_lsn; } else { diff --git a/safekeeper/src/wal_service.rs b/safekeeper/src/wal_service.rs index fb0d77a9f2..406132b2b0 100644 --- a/safekeeper/src/wal_service.rs +++ b/safekeeper/src/wal_service.rs @@ -4,7 +4,7 @@ //! use anyhow::{Context, Result}; use postgres_backend::QueryError; -use std::{future, thread, time::Duration}; +use std::{future, time::Duration}; use tokio::net::TcpStream; use tokio_io_timeout::TimeoutReader; use tracing::*; @@ -16,104 +16,82 @@ use crate::SafeKeeperConf; use postgres_backend::{AuthType, PostgresBackend}; /// Accept incoming TCP connections and spawn them into a background thread. -pub fn thread_main(conf: SafeKeeperConf, pg_listener: std::net::TcpListener) { - let runtime = tokio::runtime::Builder::new_current_thread() - .enable_all() - .build() - .context("create runtime") - // todo catch error in main thread - .expect("failed to create runtime"); +pub async fn task_main( + conf: SafeKeeperConf, + pg_listener: std::net::TcpListener, +) -> anyhow::Result<()> { + // Tokio's from_std won't do this for us, per its comment. + pg_listener.set_nonblocking(true)?; - runtime - .block_on(async move { - // Tokio's from_std won't do this for us, per its comment. - pg_listener.set_nonblocking(true)?; - let listener = tokio::net::TcpListener::from_std(pg_listener)?; - let mut connection_count: ConnectionCount = 0; + let listener = tokio::net::TcpListener::from_std(pg_listener)?; + let mut connection_count: ConnectionCount = 0; - loop { - match listener.accept().await { - Ok((socket, peer_addr)) => { - debug!("accepted connection from {}", peer_addr); - let conf = conf.clone(); - let conn_id = issue_connection_id(&mut connection_count); + loop { + let (socket, peer_addr) = listener.accept().await.context("accept")?; + debug!("accepted connection from {}", peer_addr); + let conf = conf.clone(); + let conn_id = issue_connection_id(&mut connection_count); - let _ = thread::Builder::new() - .name("WAL service thread".into()) - .spawn(move || { - if let Err(err) = handle_socket(socket, conf, conn_id) { - error!("connection handler exited: {}", err); - } - }) - .unwrap(); - } - Err(e) => error!("Failed to accept connection: {}", e), - } + tokio::spawn(async move { + if let Err(err) = handle_socket(socket, conf, conn_id) + .instrument(info_span!("", cid = %conn_id)) + .await + { + error!("connection handler exited: {}", err); } - #[allow(unreachable_code)] // hint compiler the closure return type - Ok::<(), anyhow::Error>(()) - }) - .expect("listener failed") + }); + } } -/// This is run by `thread_main` above, inside a background thread. +/// This is run by `task_main` above, inside a background thread. /// -fn handle_socket( +async fn handle_socket( socket: TcpStream, conf: SafeKeeperConf, conn_id: ConnectionId, ) -> Result<(), QueryError> { - let _enter = info_span!("", cid = %conn_id).entered(); - - let runtime = tokio::runtime::Builder::new_current_thread() - .enable_all() - .build()?; - socket.set_nodelay(true)?; let peer_addr = socket.peer_addr()?; - // TimeoutReader wants async runtime during creation. - runtime.block_on(async move { - // Set timeout on reading from the socket. It prevents hanged up connection - // if client suddenly disappears. Note that TCP_KEEPALIVE is not enabled by - // default, and tokio doesn't provide ability to set it out of the box. - let mut socket = TimeoutReader::new(socket); - let wal_service_timeout = Duration::from_secs(60 * 10); - socket.set_timeout(Some(wal_service_timeout)); - // pin! is here because TimeoutReader (due to storing sleep future inside) - // is not Unpin, and all pgbackend/framed/tokio dependencies require stream - // to be Unpin. Which is reasonable, as indeed something like TimeoutReader - // shouldn't be moved. - tokio::pin!(socket); + // Set timeout on reading from the socket. It prevents hanged up connection + // if client suddenly disappears. Note that TCP_KEEPALIVE is not enabled by + // default, and tokio doesn't provide ability to set it out of the box. + let mut socket = TimeoutReader::new(socket); + let wal_service_timeout = Duration::from_secs(60 * 10); + socket.set_timeout(Some(wal_service_timeout)); + // pin! is here because TimeoutReader (due to storing sleep future inside) + // is not Unpin, and all pgbackend/framed/tokio dependencies require stream + // to be Unpin. Which is reasonable, as indeed something like TimeoutReader + // shouldn't be moved. + tokio::pin!(socket); - let traffic_metrics = TrafficMetrics::new(); - if let Some(current_az) = conf.availability_zone.as_deref() { - traffic_metrics.set_sk_az(current_az); - } + let traffic_metrics = TrafficMetrics::new(); + if let Some(current_az) = conf.availability_zone.as_deref() { + traffic_metrics.set_sk_az(current_az); + } - let socket = MeasuredStream::new( - socket, - |cnt| { - traffic_metrics.observe_read(cnt); - }, - |cnt| { - traffic_metrics.observe_write(cnt); - }, - ); + let socket = MeasuredStream::new( + socket, + |cnt| { + traffic_metrics.observe_read(cnt); + }, + |cnt| { + traffic_metrics.observe_write(cnt); + }, + ); - let auth_type = match conf.auth { - None => AuthType::Trust, - Some(_) => AuthType::NeonJWT, - }; - let mut conn_handler = - SafekeeperPostgresHandler::new(conf, conn_id, Some(traffic_metrics.clone())); - let pgbackend = PostgresBackend::new_from_io(socket, peer_addr, auth_type, None)?; - // libpq protocol between safekeeper and walproposer / pageserver - // We don't use shutdown. - pgbackend - .run(&mut conn_handler, future::pending::<()>) - .await - }) + let auth_type = match conf.auth { + None => AuthType::Trust, + Some(_) => AuthType::NeonJWT, + }; + let mut conn_handler = + SafekeeperPostgresHandler::new(conf, conn_id, Some(traffic_metrics.clone())); + let pgbackend = PostgresBackend::new_from_io(socket, peer_addr, auth_type, None)?; + // libpq protocol between safekeeper and walproposer / pageserver + // We don't use shutdown. + pgbackend + .run(&mut conn_handler, future::pending::<()>) + .await } /// Unique WAL service connection ids are logged in spans for observability. diff --git a/safekeeper/src/wal_storage.rs b/safekeeper/src/wal_storage.rs index 1b82bd754e..687e1ba6b6 100644 --- a/safekeeper/src/wal_storage.rs +++ b/safekeeper/src/wal_storage.rs @@ -8,54 +8,47 @@ //! Note that last file has `.partial` suffix, that's different from postgres. use anyhow::{bail, Context, Result}; -use remote_storage::RemotePath; - -use std::io::{self, Seek, SeekFrom}; -use std::pin::Pin; -use tokio::io::AsyncRead; - +use bytes::Bytes; +use futures::future::BoxFuture; use postgres_ffi::v14::xlog_utils::{IsPartialXLogFileName, IsXLogFileName, XLogFromFileName}; use postgres_ffi::{XLogSegNo, PG_TLI}; +use remote_storage::RemotePath; use std::cmp::{max, min}; - -use bytes::Bytes; -use std::fs::{self, remove_file, File, OpenOptions}; -use std::io::Write; +use std::io::{self, SeekFrom}; use std::path::{Path, PathBuf}; - +use std::pin::Pin; +use tokio::fs::{self, remove_file, File, OpenOptions}; +use tokio::io::{AsyncRead, AsyncWriteExt}; +use tokio::io::{AsyncReadExt, AsyncSeekExt}; use tracing::*; -use utils::{id::TenantTimelineId, lsn::Lsn}; - use crate::metrics::{time_io_closure, WalStorageMetrics, REMOVED_WAL_SEGMENTS}; use crate::safekeeper::SafeKeeperState; - use crate::wal_backup::read_object; use crate::SafeKeeperConf; +use postgres_ffi::waldecoder::WalStreamDecoder; use postgres_ffi::XLogFileName; use postgres_ffi::XLOG_BLCKSZ; - -use postgres_ffi::waldecoder::WalStreamDecoder; - use pq_proto::SystemId; -use tokio::io::{AsyncReadExt, AsyncSeekExt}; +use utils::{id::TenantTimelineId, lsn::Lsn}; +#[async_trait::async_trait] pub trait Storage { /// LSN of last durably stored WAL record. fn flush_lsn(&self) -> Lsn; /// Write piece of WAL from buf to disk, but not necessarily sync it. - fn write_wal(&mut self, startpos: Lsn, buf: &[u8]) -> Result<()>; + async fn write_wal(&mut self, startpos: Lsn, buf: &[u8]) -> Result<()>; /// Truncate WAL at specified LSN, which must be the end of WAL record. - fn truncate_wal(&mut self, end_pos: Lsn) -> Result<()>; + async fn truncate_wal(&mut self, end_pos: Lsn) -> Result<()>; /// Durably store WAL on disk, up to the last written WAL record. - fn flush_wal(&mut self) -> Result<()>; + async fn flush_wal(&mut self) -> Result<()>; - /// Remove all segments <= given segno. Returns closure as we want to do - /// that without timeline lock. - fn remove_up_to(&self) -> Box Result<()>>; + /// Remove all segments <= given segno. Returns function doing that as we + /// want to perform it without timeline lock. + fn remove_up_to(&self, segno_up_to: XLogSegNo) -> BoxFuture<'static, anyhow::Result<()>>; /// Release resources associated with the storage -- technically, close FDs. /// Currently we don't remove timelines until restart (#3146), so need to @@ -105,6 +98,22 @@ pub struct PhysicalStorage { /// - points to write_lsn, so no seek is needed for writing /// - doesn't point to the end of the segment file: Option, + + /// When false, we have just initialized storage using the LSN from find_end_of_wal(). + /// In this case, [`write_lsn`] can be less than actually written WAL on disk. In particular, + /// there can be a case with unexpected .partial file. + /// + /// Imagine the following: + /// - 000000010000000000000001 + /// - it was fully written, but the last record is split between 2 segments + /// - after restart, find_end_of_wal() returned 0/1FFFFF0, which is in the end of this segment + /// - write_lsn, write_record_lsn and flush_record_lsn were initialized to 0/1FFFFF0 + /// - 000000010000000000000002.partial + /// - it has only 1 byte written, which is not enough to make a full WAL record + /// + /// Partial segment 002 has no WAL records, and it will be removed by the next truncate_wal(). + /// This flag will be set to true after the first truncate_wal() call. + is_truncated_after_restart: bool, } impl PhysicalStorage { @@ -164,6 +173,7 @@ impl PhysicalStorage { flush_record_lsn: flush_lsn, decoder: WalStreamDecoder::new(write_lsn, state.server.pg_version / 10000), file: None, + is_truncated_after_restart: false, }) } @@ -178,33 +188,37 @@ impl PhysicalStorage { } /// Call fdatasync if config requires so. - fn fdatasync_file(&mut self, file: &mut File) -> Result<()> { + async fn fdatasync_file(&mut self, file: &mut File) -> Result<()> { if !self.conf.no_sync { self.metrics - .observe_flush_seconds(time_io_closure(|| Ok(file.sync_data()?))?); + .observe_flush_seconds(time_io_closure(file.sync_data()).await?); } Ok(()) } /// Call fsync if config requires so. - fn fsync_file(&mut self, file: &mut File) -> Result<()> { + async fn fsync_file(&mut self, file: &mut File) -> Result<()> { if !self.conf.no_sync { self.metrics - .observe_flush_seconds(time_io_closure(|| Ok(file.sync_all()?))?); + .observe_flush_seconds(time_io_closure(file.sync_all()).await?); } Ok(()) } /// Open or create WAL segment file. Caller must call seek to the wanted position. /// Returns `file` and `is_partial`. - fn open_or_create(&mut self, segno: XLogSegNo) -> Result<(File, bool)> { + async fn open_or_create(&mut self, segno: XLogSegNo) -> Result<(File, bool)> { let (wal_file_path, wal_file_partial_path) = wal_file_paths(&self.timeline_dir, segno, self.wal_seg_size)?; // Try to open already completed segment - if let Ok(file) = OpenOptions::new().write(true).open(&wal_file_path) { + if let Ok(file) = OpenOptions::new().write(true).open(&wal_file_path).await { Ok((file, false)) - } else if let Ok(file) = OpenOptions::new().write(true).open(&wal_file_partial_path) { + } else if let Ok(file) = OpenOptions::new() + .write(true) + .open(&wal_file_partial_path) + .await + { // Try to open existing partial file Ok((file, true)) } else { @@ -213,35 +227,36 @@ impl PhysicalStorage { .create(true) .write(true) .open(&wal_file_partial_path) + .await .with_context(|| format!("Failed to open log file {:?}", &wal_file_path))?; - write_zeroes(&mut file, self.wal_seg_size)?; - self.fsync_file(&mut file)?; + write_zeroes(&mut file, self.wal_seg_size).await?; + self.fsync_file(&mut file).await?; Ok((file, true)) } } /// Write WAL bytes, which are known to be located in a single WAL segment. - fn write_in_segment(&mut self, segno: u64, xlogoff: usize, buf: &[u8]) -> Result<()> { + async fn write_in_segment(&mut self, segno: u64, xlogoff: usize, buf: &[u8]) -> Result<()> { let mut file = if let Some(file) = self.file.take() { file } else { - let (mut file, is_partial) = self.open_or_create(segno)?; + let (mut file, is_partial) = self.open_or_create(segno).await?; assert!(is_partial, "unexpected write into non-partial segment file"); - file.seek(SeekFrom::Start(xlogoff as u64))?; + file.seek(SeekFrom::Start(xlogoff as u64)).await?; file }; - file.write_all(buf)?; + file.write_all(buf).await?; if xlogoff + buf.len() == self.wal_seg_size { // If we reached the end of a WAL segment, flush and close it. - self.fdatasync_file(&mut file)?; + self.fdatasync_file(&mut file).await?; // Rename partial file to completed file let (wal_file_path, wal_file_partial_path) = wal_file_paths(&self.timeline_dir, segno, self.wal_seg_size)?; - fs::rename(wal_file_partial_path, wal_file_path)?; + fs::rename(wal_file_partial_path, wal_file_path).await?; } else { // otherwise, file can be reused later self.file = Some(file); @@ -255,11 +270,11 @@ impl PhysicalStorage { /// be flushed separately later. /// /// Updates `write_lsn`. - fn write_exact(&mut self, pos: Lsn, mut buf: &[u8]) -> Result<()> { + async fn write_exact(&mut self, pos: Lsn, mut buf: &[u8]) -> Result<()> { if self.write_lsn != pos { // need to flush the file before discarding it if let Some(mut file) = self.file.take() { - self.fdatasync_file(&mut file)?; + self.fdatasync_file(&mut file).await?; } self.write_lsn = pos; @@ -277,7 +292,8 @@ impl PhysicalStorage { buf.len() }; - self.write_in_segment(segno, xlogoff, &buf[..bytes_write])?; + self.write_in_segment(segno, xlogoff, &buf[..bytes_write]) + .await?; self.write_lsn += bytes_write as u64; buf = &buf[bytes_write..]; } @@ -286,6 +302,7 @@ impl PhysicalStorage { } } +#[async_trait::async_trait] impl Storage for PhysicalStorage { /// flush_lsn returns LSN of last durably stored WAL record. fn flush_lsn(&self) -> Lsn { @@ -293,7 +310,7 @@ impl Storage for PhysicalStorage { } /// Write WAL to disk. - fn write_wal(&mut self, startpos: Lsn, buf: &[u8]) -> Result<()> { + async fn write_wal(&mut self, startpos: Lsn, buf: &[u8]) -> Result<()> { // Disallow any non-sequential writes, which can result in gaps or overwrites. // If we need to move the pointer, use truncate_wal() instead. if self.write_lsn > startpos { @@ -311,7 +328,7 @@ impl Storage for PhysicalStorage { ); } - let write_seconds = time_io_closure(|| self.write_exact(startpos, buf))?; + let write_seconds = time_io_closure(self.write_exact(startpos, buf)).await?; // WAL is written, updating write metrics self.metrics.observe_write_seconds(write_seconds); self.metrics.observe_write_bytes(buf.len()); @@ -340,14 +357,14 @@ impl Storage for PhysicalStorage { Ok(()) } - fn flush_wal(&mut self) -> Result<()> { + async fn flush_wal(&mut self) -> Result<()> { if self.flush_record_lsn == self.write_record_lsn { // no need to do extra flush return Ok(()); } if let Some(mut unflushed_file) = self.file.take() { - self.fdatasync_file(&mut unflushed_file)?; + self.fdatasync_file(&mut unflushed_file).await?; self.file = Some(unflushed_file); } else { // We have unflushed data (write_lsn != flush_lsn), but no file. @@ -369,7 +386,7 @@ impl Storage for PhysicalStorage { /// Truncate written WAL by removing all WAL segments after the given LSN. /// end_pos must point to the end of the WAL record. - fn truncate_wal(&mut self, end_pos: Lsn) -> Result<()> { + async fn truncate_wal(&mut self, end_pos: Lsn) -> Result<()> { // Streaming must not create a hole, so truncate cannot be called on non-written lsn if self.write_lsn != Lsn(0) && end_pos > self.write_lsn { bail!( @@ -379,43 +396,53 @@ impl Storage for PhysicalStorage { ); } + // Quick exit if nothing to do to avoid writing up to 16 MiB of zeros on + // disk (this happens on each connect). + if self.is_truncated_after_restart + && end_pos == self.write_lsn + && end_pos == self.flush_record_lsn + { + return Ok(()); + } + // Close previously opened file, if any if let Some(mut unflushed_file) = self.file.take() { - self.fdatasync_file(&mut unflushed_file)?; + self.fdatasync_file(&mut unflushed_file).await?; } let xlogoff = end_pos.segment_offset(self.wal_seg_size); let segno = end_pos.segment_number(self.wal_seg_size); // Remove all segments after the given LSN. - remove_segments_from_disk(&self.timeline_dir, self.wal_seg_size, |x| x > segno)?; + remove_segments_from_disk(&self.timeline_dir, self.wal_seg_size, |x| x > segno).await?; - let (mut file, is_partial) = self.open_or_create(segno)?; + let (mut file, is_partial) = self.open_or_create(segno).await?; // Fill end with zeroes - file.seek(SeekFrom::Start(xlogoff as u64))?; - write_zeroes(&mut file, self.wal_seg_size - xlogoff)?; - self.fdatasync_file(&mut file)?; + file.seek(SeekFrom::Start(xlogoff as u64)).await?; + write_zeroes(&mut file, self.wal_seg_size - xlogoff).await?; + self.fdatasync_file(&mut file).await?; if !is_partial { // Make segment partial once again let (wal_file_path, wal_file_partial_path) = wal_file_paths(&self.timeline_dir, segno, self.wal_seg_size)?; - fs::rename(wal_file_path, wal_file_partial_path)?; + fs::rename(wal_file_path, wal_file_partial_path).await?; } // Update LSNs self.write_lsn = end_pos; self.write_record_lsn = end_pos; self.flush_record_lsn = end_pos; + self.is_truncated_after_restart = true; Ok(()) } - fn remove_up_to(&self) -> Box Result<()>> { + fn remove_up_to(&self, segno_up_to: XLogSegNo) -> BoxFuture<'static, anyhow::Result<()>> { let timeline_dir = self.timeline_dir.clone(); let wal_seg_size = self.wal_seg_size; - Box::new(move |segno_up_to: XLogSegNo| { - remove_segments_from_disk(&timeline_dir, wal_seg_size, |x| x <= segno_up_to) + Box::pin(async move { + remove_segments_from_disk(&timeline_dir, wal_seg_size, |x| x <= segno_up_to).await }) } @@ -430,7 +457,7 @@ impl Storage for PhysicalStorage { } /// Remove all WAL segments in timeline_dir that match the given predicate. -fn remove_segments_from_disk( +async fn remove_segments_from_disk( timeline_dir: &Path, wal_seg_size: usize, remove_predicate: impl Fn(XLogSegNo) -> bool, @@ -439,8 +466,8 @@ fn remove_segments_from_disk( let mut min_removed = u64::MAX; let mut max_removed = u64::MIN; - for entry in fs::read_dir(timeline_dir)? { - let entry = entry?; + let mut entries = fs::read_dir(timeline_dir).await?; + while let Some(entry) = entries.next_entry().await? { let entry_path = entry.path(); let fname = entry_path.file_name().unwrap(); @@ -451,7 +478,7 @@ fn remove_segments_from_disk( } let (segno, _) = XLogFromFileName(fname_str, wal_seg_size); if remove_predicate(segno) { - remove_file(entry_path)?; + remove_file(entry_path).await?; n_removed += 1; min_removed = min(min_removed, segno); max_removed = max(max_removed, segno); @@ -683,12 +710,12 @@ impl WalReader { const ZERO_BLOCK: &[u8] = &[0u8; XLOG_BLCKSZ]; /// Helper for filling file with zeroes. -fn write_zeroes(file: &mut File, mut count: usize) -> Result<()> { +async fn write_zeroes(file: &mut File, mut count: usize) -> Result<()> { while count >= XLOG_BLCKSZ { - file.write_all(ZERO_BLOCK)?; + file.write_all(ZERO_BLOCK).await?; count -= XLOG_BLCKSZ; } - file.write_all(&ZERO_BLOCK[0..count])?; + file.write_all(&ZERO_BLOCK[0..count]).await?; Ok(()) } diff --git a/scripts/comment-test-report.js b/scripts/comment-test-report.js old mode 100644 new mode 100755 index a7fd5b0bef..432c78d1af --- a/scripts/comment-test-report.js +++ b/scripts/comment-test-report.js @@ -1,3 +1,5 @@ +#! /usr/bin/env node + // // The script parses Allure reports and posts a comment with a summary of the test results to the PR or to the latest commit in the branch. // @@ -19,7 +21,7 @@ // }) // -// Analog of Python's defaultdict. +// Equivalent of Python's defaultdict. // // const dm = new DefaultMap(() => new DefaultMap(() => [])) // dm["firstKey"]["secondKey"].push("value") @@ -32,34 +34,7 @@ class DefaultMap extends Map { } } -module.exports = async ({ github, context, fetch, report }) => { - // Marker to find the comment in the subsequent runs - const startMarker = `` - // If we run the script in the PR or in the branch (main/release/...) - const isPullRequest = !!context.payload.pull_request - // Latest commit in PR or in the branch - const commitSha = isPullRequest ? context.payload.pull_request.head.sha : context.sha - // Let users know that the comment is updated automatically - const autoupdateNotice = `
The comment gets automatically updated with the latest test results
${commitSha} at ${new Date().toISOString()} :recycle:
` - // GitHub bot id taken from (https://api.github.com/users/github-actions[bot]) - const githubActionsBotId = 41898282 - // Commend body itself - let commentBody = `${startMarker}\n` - - // Common parameters for GitHub API requests - const ownerRepoParams = { - owner: context.repo.owner, - repo: context.repo.repo, - } - - const {reportUrl, reportJsonUrl} = report - - if (!reportUrl || !reportJsonUrl) { - commentBody += `#### No tests were run or test report is not available\n` - commentBody += autoupdateNotice - return - } - +const parseReportJson = async ({ reportJsonUrl, fetch }) => { const suites = await (await fetch(reportJsonUrl)).json() // Allure distinguishes "failed" (with an assertion error) and "broken" (with any other error) tests. @@ -83,7 +58,7 @@ module.exports = async ({ github, context, fetch, report }) => { let buildType, pgVersion const match = test.name.match(/[\[-](?debug|release)-pg(?\d+)[-\]]/)?.groups if (match) { - ({buildType, pgVersion} = match) + ({ buildType, pgVersion } = match) } else { // It's ok, we embed BUILD_TYPE and Postgres Version into the test name only for regress suite and do not for other suites (like performance). console.info(`Cannot get BUILD_TYPE and Postgres Version from test name: "${test.name}", defaulting to "release" and "14"`) @@ -123,37 +98,68 @@ module.exports = async ({ github, context, fetch, report }) => { } } + return { + failedTests, + failedTestsCount, + passedTests, + passedTestsCount, + skippedTests, + skippedTestsCount, + flakyTests, + flakyTestsCount, + retriedTests, + pgVersions, + } +} + +const reportSummary = async (params) => { + const { + failedTests, + failedTestsCount, + passedTests, + passedTestsCount, + skippedTests, + skippedTestsCount, + flakyTests, + flakyTestsCount, + retriedTests, + pgVersions, + reportUrl, + } = params + + let summary = "" + const totalTestsCount = failedTestsCount + passedTestsCount + skippedTestsCount - commentBody += `### ${totalTestsCount} tests run: ${passedTestsCount} passed, ${failedTestsCount} failed, ${skippedTestsCount} skipped ([full report](${reportUrl}))\n___\n` + summary += `### ${totalTestsCount} tests run: ${passedTestsCount} passed, ${failedTestsCount} failed, ${skippedTestsCount} skipped ([full report](${reportUrl}))\n___\n` // Print test resuls from the newest to the oldest Postgres version for release and debug builds. for (const pgVersion of Array.from(pgVersions).sort().reverse()) { if (Object.keys(failedTests[pgVersion]).length > 0) { - commentBody += `#### Failures on Posgres ${pgVersion}\n\n` + summary += `#### Failures on Posgres ${pgVersion}\n\n` for (const [testName, tests] of Object.entries(failedTests[pgVersion])) { const links = [] for (const test of tests) { const allureLink = `${reportUrl}#suites/${test.parentUid}/${test.uid}` links.push(`[${test.buildType}](${allureLink})`) } - commentBody += `- \`${testName}\`: ${links.join(", ")}\n` + summary += `- \`${testName}\`: ${links.join(", ")}\n` } const testsToRerun = Object.values(failedTests[pgVersion]).map(x => x[0].name) const command = `DEFAULT_PG_VERSION=${pgVersion} scripts/pytest -k "${testsToRerun.join(" or ")}"` - commentBody += "```\n" - commentBody += `# Run failed on Postgres ${pgVersion} tests locally:\n` - commentBody += `${command}\n` - commentBody += "```\n" + summary += "```\n" + summary += `# Run failed on Postgres ${pgVersion} tests locally:\n` + summary += `${command}\n` + summary += "```\n" } } if (flakyTestsCount > 0) { - commentBody += `
\nFlaky tests (${flakyTestsCount})\n\n` + summary += `
\nFlaky tests (${flakyTestsCount})\n\n` for (const pgVersion of Array.from(pgVersions).sort().reverse()) { if (Object.keys(flakyTests[pgVersion]).length > 0) { - commentBody += `#### Postgres ${pgVersion}\n\n` + summary += `#### Postgres ${pgVersion}\n\n` for (const [testName, tests] of Object.entries(flakyTests[pgVersion])) { const links = [] for (const test of tests) { @@ -161,11 +167,57 @@ module.exports = async ({ github, context, fetch, report }) => { const status = test.status === "passed" ? ":white_check_mark:" : ":x:" links.push(`[${status} ${test.buildType}](${allureLink})`) } - commentBody += `- \`${testName}\`: ${links.join(", ")}\n` + summary += `- \`${testName}\`: ${links.join(", ")}\n` } } } - commentBody += "\n
\n" + summary += "\n
\n" + } + + return summary +} + +module.exports = async ({ github, context, fetch, report }) => { + // Marker to find the comment in the subsequent runs + const startMarker = `` + // If we run the script in the PR or in the branch (main/release/...) + const isPullRequest = !!context.payload.pull_request + // Latest commit in PR or in the branch + const commitSha = isPullRequest ? context.payload.pull_request.head.sha : context.sha + // Let users know that the comment is updated automatically + const autoupdateNotice = `
The comment gets automatically updated with the latest test results
${commitSha} at ${new Date().toISOString()} :recycle:
` + // GitHub bot id taken from (https://api.github.com/users/github-actions[bot]) + const githubActionsBotId = 41898282 + // Commend body itself + let commentBody = `${startMarker}\n` + + // Common parameters for GitHub API requests + const ownerRepoParams = { + owner: context.repo.owner, + repo: context.repo.repo, + } + + const {reportUrl, reportJsonUrl} = report + + if (!reportUrl || !reportJsonUrl) { + commentBody += `#### No tests were run or test report is not available\n` + commentBody += autoupdateNotice + return + } + + try { + const parsed = await parseReportJson({ reportJsonUrl, fetch }) + commentBody += await reportSummary({ ...parsed, reportUrl }) + } catch (error) { + commentBody += `### [full report](${reportUrl})\n___\n` + commentBody += `#### Failed to create a summary for the test run: \n` + commentBody += "```\n" + commentBody += `${error.stack}\n` + commentBody += "```\n" + commentBody += "\nTo reproduce and debug the error locally run:\n" + commentBody += "```\n" + commentBody += `scripts/comment-test-report.js ${reportJsonUrl}` + commentBody += "\n```\n" } commentBody += autoupdateNotice @@ -207,3 +259,60 @@ module.exports = async ({ github, context, fetch, report }) => { }) } } + +// Equivalent of Python's `if __name__ == "__main__":` +// https://nodejs.org/docs/latest/api/modules.html#accessing-the-main-module +if (require.main === module) { + // Poor man's argument parsing: we expect the third argument is a JSON URL (0: node binary, 1: this script, 2: JSON url) + if (process.argv.length !== 3) { + console.error(`Unexpected number of arguments\nUsage: node ${process.argv[1]} `) + process.exit(1) + } + const jsonUrl = process.argv[2] + + try { + new URL(jsonUrl) + } catch (error) { + console.error(`Invalid URL: ${jsonUrl}\nUsage: node ${process.argv[1]} `) + process.exit(1) + } + + const htmlUrl = jsonUrl.replace("/data/suites.json", "/index.html") + + const githubMock = { + rest: { + issues: { + createComment: console.log, + listComments: async () => ({ data: [] }), + updateComment: console.log + }, + repos: { + createCommitComment: console.log, + listCommentsForCommit: async () => ({ data: [] }), + updateCommitComment: console.log + } + } + } + + const contextMock = { + repo: { + owner: 'testOwner', + repo: 'testRepo' + }, + payload: { + number: 42, + pull_request: null, + }, + sha: '0000000000000000000000000000000000000000', + } + + module.exports({ + github: githubMock, + context: contextMock, + fetch: fetch, + report: { + reportUrl: htmlUrl, + reportJsonUrl: jsonUrl, + } + }) +} diff --git a/scripts/coverage b/scripts/coverage index 1dc92e57cc..52a69c93b9 100755 --- a/scripts/coverage +++ b/scripts/coverage @@ -156,7 +156,9 @@ class LLVM: profdata: Path, objects: List[str], sources: List[str], - demangler: Optional[Path] = None) -> None: + demangler: Optional[Path] = None, + output_file: Optional[Path] = None, + ) -> None: cwd = self.cargo.cwd objects = list(intersperse('-object', objects)) @@ -180,14 +182,18 @@ class LLVM: *objects, *sources, ] - subprocess.check_call(cmd, cwd=cwd) + if output_file is not None: + with output_file.open('w') as outfile: + subprocess.check_call(cmd, cwd=cwd, stdout=outfile) + else: + subprocess.check_call(cmd, cwd=cwd) def cov_report(self, **kwargs) -> None: self._cov(subcommand='report', **kwargs) - def cov_export(self, *, kind: str, **kwargs) -> None: + def cov_export(self, *, kind: str, output_file: Optional[Path], **kwargs) -> None: extras = (f'-format={kind}', ) - self._cov(subcommand='export', *extras, **kwargs) + self._cov(subcommand='export', *extras, output_file=output_file, **kwargs) def cov_show(self, *, kind: str, output_dir: Optional[Path] = None, **kwargs) -> None: extras = [f'-format={kind}'] @@ -283,9 +289,12 @@ class TextReport(Report): self.llvm.cov_show(kind='text', **self._common_kwargs()) +@dataclass class LcovReport(Report): + output_file: Path + def generate(self) -> None: - self.llvm.cov_export(kind='lcov', **self._common_kwargs()) + self.llvm.cov_export(kind='lcov', output_file=self.output_file, **self._common_kwargs()) @dataclass @@ -475,7 +484,7 @@ class State: 'text': lambda: TextReport(**params), 'lcov': - lambda: LcovReport(**params), + lambda: LcovReport(**params, output_file=self.report_dir / 'lcov.info'), 'summary': lambda: SummaryReport(**params), 'github': diff --git a/scripts/ingest_perf_test_result.py b/scripts/ingest_perf_test_result.py index 1bfc907def..35a1e29720 100644 --- a/scripts/ingest_perf_test_result.py +++ b/scripts/ingest_perf_test_result.py @@ -1,12 +1,14 @@ #!/usr/bin/env python3 import argparse import json +import logging import os import sys from contextlib import contextmanager from datetime import datetime from pathlib import Path +import backoff import psycopg2 import psycopg2.extras @@ -35,9 +37,20 @@ def get_connection_cursor(): connstr = os.getenv("DATABASE_URL") if not connstr: err("DATABASE_URL environment variable is not set") - with psycopg2.connect(connstr, connect_timeout=30) as conn: + + @backoff.on_exception(backoff.expo, psycopg2.OperationalError, max_time=150) + def connect(connstr): + conn = psycopg2.connect(connstr, connect_timeout=30) + conn.autocommit = True + return conn + + conn = connect(connstr) + try: with conn.cursor() as cur: yield cur + finally: + if conn is not None: + conn.close() def create_table(cur): @@ -115,6 +128,7 @@ def main(): parser.add_argument( "--ingest", type=Path, + required=True, help="Path to perf test result file, or directory with perf test result files", ) parser.add_argument("--initdb", action="store_true", help="Initialuze database") @@ -140,4 +154,5 @@ def main(): if __name__ == "__main__": + logging.getLogger("backoff").addHandler(logging.StreamHandler()) main() diff --git a/scripts/ingest_regress_test_result.py b/scripts/ingest_regress_test_result.py index 974167483a..39c1c02941 100644 --- a/scripts/ingest_regress_test_result.py +++ b/scripts/ingest_regress_test_result.py @@ -1,11 +1,13 @@ #!/usr/bin/env python3 import argparse +import logging import os import re import sys from contextlib import contextmanager from pathlib import Path +import backoff import psycopg2 CREATE_TABLE = """ @@ -29,9 +31,20 @@ def get_connection_cursor(): connstr = os.getenv("DATABASE_URL") if not connstr: err("DATABASE_URL environment variable is not set") - with psycopg2.connect(connstr, connect_timeout=30) as conn: + + @backoff.on_exception(backoff.expo, psycopg2.OperationalError, max_time=150) + def connect(connstr): + conn = psycopg2.connect(connstr, connect_timeout=30) + conn.autocommit = True + return conn + + conn = connect(connstr) + try: with conn.cursor() as cur: yield cur + finally: + if conn is not None: + conn.close() def create_table(cur): @@ -101,4 +114,5 @@ def main(): if __name__ == "__main__": + logging.getLogger("backoff").addHandler(logging.StreamHandler()) main() diff --git a/storage_broker/src/lib.rs b/storage_broker/src/lib.rs index 4bc561449d..3f6fa35cbe 100644 --- a/storage_broker/src/lib.rs +++ b/storage_broker/src/lib.rs @@ -32,6 +32,7 @@ pub const DEFAULT_LISTEN_ADDR: &str = "127.0.0.1:50051"; pub const DEFAULT_ENDPOINT: &str = const_format::formatcp!("http://{DEFAULT_LISTEN_ADDR}"); pub const DEFAULT_KEEPALIVE_INTERVAL: &str = "5000 ms"; +pub const DEFAULT_CONNECT_TIMEOUT: Duration = Duration::from_millis(5000); // BrokerServiceClient charged with tonic provided Channel transport; helps to // avoid depending on tonic directly in user crates. @@ -58,7 +59,8 @@ where } tonic_endpoint = tonic_endpoint .http2_keep_alive_interval(keepalive_interval) - .keep_alive_while_idle(true); + .keep_alive_while_idle(true) + .connect_timeout(DEFAULT_CONNECT_TIMEOUT); // keep_alive_timeout is 20s by default on both client and server side let channel = tonic_endpoint.connect_lazy(); Ok(BrokerClientChannel::new(channel)) diff --git a/test_runner/fixtures/flaky.py b/test_runner/fixtures/flaky.py index 9d7f8ead9a..d13f3318b0 100644 --- a/test_runner/fixtures/flaky.py +++ b/test_runner/fixtures/flaky.py @@ -1,6 +1,6 @@ import json from pathlib import Path -from typing import List +from typing import Any, List, MutableMapping, cast import pytest from _pytest.config import Config @@ -56,3 +56,15 @@ def pytest_collection_modifyitems(config: Config, items: List[pytest.Item]): # Rerun 3 times = 1 original run + 2 reruns log.info(f"Marking {item.nodeid} as flaky. It will be rerun up to 3 times") item.add_marker(pytest.mark.flaky(reruns=2)) + + # pytest-rerunfailures is not compatible with pytest-timeout (timeout is not set for reruns), + # we can workaround it by setting `timeout_func_only` to True[1]. + # Unfortunately, setting `timeout_func_only = True` globally in pytest.ini is broken[2], + # but we still can do it using pytest marker. + # + # - [1] https://github.com/pytest-dev/pytest-rerunfailures/issues/99 + # - [2] https://github.com/pytest-dev/pytest-timeout/issues/142 + timeout_marker = item.get_closest_marker("timeout") + if timeout_marker is not None: + kwargs = cast(MutableMapping[str, Any], timeout_marker.kwargs) + kwargs["func_only"] = True diff --git a/test_runner/fixtures/metrics.py b/test_runner/fixtures/metrics.py index 0e958ddd06..7ee3c33f92 100644 --- a/test_runner/fixtures/metrics.py +++ b/test_runner/fixtures/metrics.py @@ -57,20 +57,28 @@ PAGESERVER_GLOBAL_METRICS: Tuple[str, ...] = ( "libmetrics_launch_timestamp", "libmetrics_build_info", "libmetrics_tracing_event_count_total", + "pageserver_materialized_cache_hits_total", + "pageserver_materialized_cache_hits_direct_total", + "pageserver_getpage_reconstruct_seconds_bucket", + "pageserver_getpage_reconstruct_seconds_count", + "pageserver_getpage_reconstruct_seconds_sum", + *[f"pageserver_basebackup_query_seconds_{x}" for x in ["bucket", "count", "sum"]], ) PAGESERVER_PER_TENANT_METRICS: Tuple[str, ...] = ( "pageserver_current_logical_size", "pageserver_resident_physical_size", - "pageserver_getpage_reconstruct_seconds_bucket", - "pageserver_getpage_reconstruct_seconds_count", - "pageserver_getpage_reconstruct_seconds_sum", + "pageserver_getpage_get_reconstruct_data_seconds_bucket", + "pageserver_getpage_get_reconstruct_data_seconds_count", + "pageserver_getpage_get_reconstruct_data_seconds_sum", "pageserver_io_operations_bytes_total", "pageserver_io_operations_seconds_bucket", "pageserver_io_operations_seconds_count", "pageserver_io_operations_seconds_sum", "pageserver_last_record_lsn", - "pageserver_materialized_cache_hits_total", + "pageserver_read_num_fs_layers_bucket", + "pageserver_read_num_fs_layers_count", + "pageserver_read_num_fs_layers_sum", "pageserver_smgr_query_seconds_bucket", "pageserver_smgr_query_seconds_count", "pageserver_smgr_query_seconds_sum", diff --git a/test_runner/fixtures/neon_fixtures.py b/test_runner/fixtures/neon_fixtures.py index 6b97c33ae4..e56bf78019 100644 --- a/test_runner/fixtures/neon_fixtures.py +++ b/test_runner/fixtures/neon_fixtures.py @@ -629,7 +629,7 @@ class NeonEnvBuilder: assert self.env is not None, "environment is not already initialized, call init() first" self.env.start() - def init_start(self) -> NeonEnv: + def init_start(self, initial_tenant_conf: Optional[Dict[str, str]] = None) -> NeonEnv: env = self.init_configs() self.start() @@ -638,7 +638,9 @@ class NeonEnvBuilder: log.info( f"Services started, creating initial tenant {env.initial_tenant} and its initial timeline" ) - initial_tenant, initial_timeline = env.neon_cli.create_tenant(tenant_id=env.initial_tenant) + initial_tenant, initial_timeline = env.neon_cli.create_tenant( + tenant_id=env.initial_tenant, conf=initial_tenant_conf + ) env.initial_timeline = initial_timeline log.info(f"Initial timeline {initial_tenant}/{initial_timeline} created successfully") @@ -661,6 +663,8 @@ class NeonEnvBuilder: else: raise RuntimeError(f"Unknown storage type: {remote_storage_kind}") + self.remote_storage_kind = remote_storage_kind + def enable_local_fs_remote_storage(self, force_enable: bool = True): """ Sets up the pageserver to use the local fs at the `test_dir/local_fs_remote_storage` path. @@ -1444,11 +1448,12 @@ class NeonCli(AbstractNeonCli): def endpoint_create( self, branch_name: str, + pg_port: int, + http_port: int, endpoint_id: Optional[str] = None, tenant_id: Optional[TenantId] = None, hot_standby: bool = False, lsn: Optional[Lsn] = None, - port: Optional[int] = None, ) -> "subprocess.CompletedProcess[str]": args = [ "endpoint", @@ -1462,8 +1467,10 @@ class NeonCli(AbstractNeonCli): ] if lsn is not None: args.extend(["--lsn", str(lsn)]) - if port is not None: - args.extend(["--port", str(port)]) + if pg_port is not None: + args.extend(["--pg-port", str(pg_port)]) + if http_port is not None: + args.extend(["--http-port", str(http_port)]) if endpoint_id is not None: args.append(endpoint_id) if hot_standby: @@ -1476,9 +1483,11 @@ class NeonCli(AbstractNeonCli): def endpoint_start( self, endpoint_id: str, + pg_port: int, + http_port: int, + safekeepers: Optional[List[int]] = None, tenant_id: Optional[TenantId] = None, lsn: Optional[Lsn] = None, - port: Optional[int] = None, ) -> "subprocess.CompletedProcess[str]": args = [ "endpoint", @@ -1490,8 +1499,10 @@ class NeonCli(AbstractNeonCli): ] if lsn is not None: args.append(f"--lsn={lsn}") - if port is not None: - args.append(f"--port={port}") + args.extend(["--pg-port", str(pg_port)]) + args.extend(["--http-port", str(http_port)]) + if safekeepers is not None: + args.extend(["--safekeepers", (",".join(map(str, safekeepers)))]) if endpoint_id is not None: args.append(endpoint_id) @@ -1583,13 +1594,11 @@ class NeonPageserver(PgProtocol): ".*serving compute connection task.*exited with error: Postgres connection error.*", ".*serving compute connection task.*exited with error: Connection reset by peer.*", ".*serving compute connection task.*exited with error: Postgres query error.*", - ".*Connection aborted: connection error: error communicating with the server: Broken pipe.*", - ".*Connection aborted: connection error: error communicating with the server: Transport endpoint is not connected.*", - ".*Connection aborted: connection error: error communicating with the server: Connection reset by peer.*", + ".*Connection aborted: error communicating with the server: Transport endpoint is not connected.*", # FIXME: replication patch for tokio_postgres regards any but CopyDone/CopyData message in CopyBoth stream as unexpected - ".*Connection aborted: connection error: unexpected message from server*", + ".*Connection aborted: unexpected message from server*", ".*kill_and_wait_impl.*: wait successful.*", - ".*Replication stream finished: db error:.*ending streaming to Some*", + ".*: db error:.*ending streaming to Some.*", ".*query handler for 'pagestream.*failed: Broken pipe.*", # pageserver notices compute shut down ".*query handler for 'pagestream.*failed: Connection reset by peer.*", # pageserver notices compute shut down # safekeeper connection can fail with this, in the window between timeline creation @@ -1606,13 +1615,12 @@ class NeonPageserver(PgProtocol): ".*manual_gc.*is_shutdown_requested\\(\\) called in an unexpected task or thread.*", ".*tenant_list: timeline is not found in remote index while it is present in the tenants registry.*", ".*Removing intermediate uninit mark file.*", - # FIXME: known race condition in TaskHandle: https://github.com/neondatabase/neon/issues/2885 - ".*sender is dropped while join handle is still alive.*", # Tenant::delete_timeline() can cause any of the four following errors. # FIXME: we shouldn't be considering it an error: https://github.com/neondatabase/neon/issues/2946 ".*could not flush frozen layer.*queue is in state Stopped", # when schedule layer upload fails because queued got closed before compaction got killed ".*wait for layer upload ops to complete.*", # .*Caused by:.*wait_completion aborted because upload queue was stopped ".*gc_loop.*Gc failed, retrying in.*timeline is Stopping", # When gc checks timeline state after acquiring layer_removal_cs + ".*gc_loop.*Gc failed, retrying in.*: Cannot run GC iteration on inactive tenant", # Tenant::gc precondition ".*compaction_loop.*Compaction failed, retrying in.*timeline is Stopping", # When compaction checks timeline state after acquiring layer_removal_cs ".*query handler for 'pagestream.*failed: Timeline .* was not found", # postgres reconnects while timeline_delete doesn't hold the tenant's timelines.lock() ".*query handler for 'pagestream.*failed: Timeline .* is not active", # timeline delete in progress @@ -1621,6 +1629,10 @@ class NeonPageserver(PgProtocol): ".*Compaction failed, retrying in [^:]+: Cannot run compaction iteration on inactive tenant", # these can happen anytime we do compactions from background task and shutdown pageserver r".*ERROR.*ancestor timeline \S+ is being stopped", + # this is expected given our collaborative shutdown approach for the UploadQueue + ".*Compaction failed, retrying in .*: queue is in state Stopped.*", + # Pageserver timeline deletion should be polled until it gets 404, so ignore it globally + ".*Error processing HTTP request: NotFound: Timeline .* was not found", ] def start( @@ -1688,6 +1700,9 @@ class NeonPageserver(PgProtocol): else: errors.append(line) + for error in errors: + log.info(f"not allowed error: {error.strip()}") + assert not errors def log_contains(self, pattern: str) -> Optional[str]: @@ -2280,17 +2295,24 @@ class Endpoint(PgProtocol): """An object representing a Postgres compute endpoint managed by the control plane.""" def __init__( - self, env: NeonEnv, tenant_id: TenantId, port: int, check_stop_result: bool = True + self, + env: NeonEnv, + tenant_id: TenantId, + pg_port: int, + http_port: int, + check_stop_result: bool = True, ): - super().__init__(host="localhost", port=port, user="cloud_admin", dbname="postgres") + super().__init__(host="localhost", port=pg_port, user="cloud_admin", dbname="postgres") self.env = env self.running = False self.branch_name: Optional[str] = None # dubious self.endpoint_id: Optional[str] = None # dubious, see asserts below self.pgdata_dir: Optional[str] = None # Path to computenode PGDATA self.tenant_id = tenant_id - self.port = port + self.pg_port = pg_port + self.http_port = http_port self.check_stop_result = check_stop_result + self.active_safekeepers: List[int] = list(map(lambda sk: sk.id, env.safekeepers)) # path to conf is /endpoints//pgdata/postgresql.conf def create( @@ -2320,7 +2342,8 @@ class Endpoint(PgProtocol): tenant_id=self.tenant_id, lsn=lsn, hot_standby=hot_standby, - port=self.port, + pg_port=self.pg_port, + http_port=self.http_port, ) path = Path("endpoints") / self.endpoint_id / "pgdata" self.pgdata_dir = os.path.join(self.env.repo_dir, path) @@ -2345,7 +2368,13 @@ class Endpoint(PgProtocol): log.info(f"Starting postgres endpoint {self.endpoint_id}") - self.env.neon_cli.endpoint_start(self.endpoint_id, tenant_id=self.tenant_id, port=self.port) + self.env.neon_cli.endpoint_start( + self.endpoint_id, + pg_port=self.pg_port, + http_port=self.http_port, + tenant_id=self.tenant_id, + safekeepers=self.active_safekeepers, + ) self.running = True return self @@ -2369,32 +2398,8 @@ class Endpoint(PgProtocol): return os.path.join(self.pg_data_dir_path(), "pg_twophase") def config_file_path(self) -> str: - """Path to postgresql.conf""" - return os.path.join(self.pg_data_dir_path(), "postgresql.conf") - - def adjust_for_safekeepers(self, safekeepers: str) -> "Endpoint": - """ - Adjust instance config for working with wal acceptors instead of - pageserver (pre-configured by CLI) directly. - """ - - # TODO: reuse config() - with open(self.config_file_path(), "r") as f: - cfg_lines = f.readlines() - with open(self.config_file_path(), "w") as f: - for cfg_line in cfg_lines: - # walproposer uses different application_name - if ( - "synchronous_standby_names" in cfg_line - or - # don't repeat safekeepers/wal_acceptors multiple times - "neon.safekeepers" in cfg_line - ): - continue - f.write(cfg_line) - f.write("synchronous_standby_names = 'walproposer'\n") - f.write("neon.safekeepers = '{}'\n".format(safekeepers)) - return self + """Path to the postgresql.conf in the endpoint directory (not the one in pgdata)""" + return os.path.join(self.endpoint_path(), "postgresql.conf") def config(self, lines: List[str]) -> "Endpoint": """ @@ -2410,6 +2415,17 @@ class Endpoint(PgProtocol): return self + def respec(self, **kwargs): + """Update the endpoint.json file used by control_plane.""" + # Read config + config_path = os.path.join(self.endpoint_path(), "endpoint.json") + with open(config_path, "r") as f: + data_dict = json.load(f) + + # Write it back updated + with open(config_path, "w") as file: + json.dump(dict(data_dict, **kwargs), file, indent=4) + def stop(self) -> "Endpoint": """ Stop the Postgres instance if it's running. @@ -2499,7 +2515,8 @@ class EndpointFactory: ep = Endpoint( self.env, tenant_id=tenant_id or self.env.initial_tenant, - port=self.env.port_distributor.get_port(), + pg_port=self.env.port_distributor.get_port(), + http_port=self.env.port_distributor.get_port(), ) self.num_instances += 1 self.endpoints.append(ep) @@ -2524,7 +2541,8 @@ class EndpointFactory: ep = Endpoint( self.env, tenant_id=tenant_id or self.env.initial_tenant, - port=self.env.port_distributor.get_port(), + pg_port=self.env.port_distributor.get_port(), + http_port=self.env.port_distributor.get_port(), ) if endpoint_id is None: @@ -2907,6 +2925,7 @@ SKIP_FILES = frozenset( "pg_internal.init", "pg.log", "zenith.signal", + "pg_hba.conf", "postgresql.conf", "postmaster.opts", "postmaster.pid", diff --git a/test_runner/fixtures/pageserver/http.py b/test_runner/fixtures/pageserver/http.py index f258a3a24d..5c4f5177d0 100644 --- a/test_runner/fixtures/pageserver/http.py +++ b/test_runner/fixtures/pageserver/http.py @@ -342,6 +342,11 @@ class PageserverHttpClient(requests.Session): return res_json def timeline_delete(self, tenant_id: TenantId, timeline_id: TimelineId, **kwargs): + """ + Note that deletion is not instant, it is scheduled and performed mostly in the background. + So if you need to wait for it to complete use `timeline_delete_wait_completed`. + For longer description consult with pageserver openapi spec. + """ res = self.delete( f"http://localhost:{self.port}/v1/tenant/{tenant_id}/timeline/{timeline_id}", **kwargs ) diff --git a/test_runner/fixtures/pageserver/utils.py b/test_runner/fixtures/pageserver/utils.py index c558387413..ad89ebad00 100644 --- a/test_runner/fixtures/pageserver/utils.py +++ b/test_runner/fixtures/pageserver/utils.py @@ -1,8 +1,8 @@ import time -from typing import Optional +from typing import Any, Dict, Optional from fixtures.log_helper import log -from fixtures.pageserver.http import PageserverHttpClient +from fixtures.pageserver.http import PageserverApiException, PageserverHttpClient from fixtures.types import Lsn, TenantId, TimelineId @@ -72,7 +72,7 @@ def wait_until_tenant_state( expected_state: str, iterations: int, period: float = 1.0, -) -> bool: +) -> Dict[str, Any]: """ Does not use `wait_until` for debugging purposes """ @@ -81,7 +81,7 @@ def wait_until_tenant_state( tenant = pageserver_http.tenant_status(tenant_id=tenant_id) log.debug(f"Tenant {tenant_id} data: {tenant}") if tenant["state"]["slug"] == expected_state: - return True + return tenant except Exception as e: log.debug(f"Tenant {tenant_id} state retrieval failure: {e}") @@ -92,6 +92,41 @@ def wait_until_tenant_state( ) +def wait_until_timeline_state( + pageserver_http: PageserverHttpClient, + tenant_id: TenantId, + timeline_id: TimelineId, + expected_state: str, + iterations: int, + period: float = 1.0, +) -> Dict[str, Any]: + """ + Does not use `wait_until` for debugging purposes + """ + for i in range(iterations): + try: + timeline = pageserver_http.timeline_detail(tenant_id=tenant_id, timeline_id=timeline_id) + log.debug(f"Timeline {tenant_id}/{timeline_id} data: {timeline}") + if isinstance(timeline["state"], str): + if timeline["state"] == expected_state: + return timeline + elif isinstance(timeline, Dict): + if timeline["state"].get(expected_state): + return timeline + + except Exception as e: + log.debug(f"Timeline {tenant_id}/{timeline_id} state retrieval failure: {e}") + + if i == iterations - 1: + # do not sleep last time, we already know that we failed + break + time.sleep(period) + + raise Exception( + f"Timeline {tenant_id}/{timeline_id} did not become {expected_state} within {iterations * period} seconds" + ) + + def wait_until_tenant_active( pageserver_http: PageserverHttpClient, tenant_id: TenantId, @@ -156,3 +191,32 @@ def wait_for_upload_queue_empty( if all(m.value == 0 for m in tl): return time.sleep(0.2) + + +def wait_timeline_detail_404( + pageserver_http: PageserverHttpClient, tenant_id: TenantId, timeline_id: TimelineId +): + last_exc = None + for _ in range(2): + time.sleep(0.250) + try: + data = pageserver_http.timeline_detail(tenant_id, timeline_id) + log.error(f"detail {data}") + except PageserverApiException as e: + log.debug(e) + if e.status_code == 404: + return + + last_exc = e + + raise last_exc or RuntimeError(f"Timeline wasnt deleted in time, state: {data['state']}") + + +def timeline_delete_wait_completed( + pageserver_http: PageserverHttpClient, + tenant_id: TenantId, + timeline_id: TimelineId, + **delete_args, +): + pageserver_http.timeline_delete(tenant_id=tenant_id, timeline_id=timeline_id, **delete_args) + wait_timeline_detail_404(pageserver_http, tenant_id, timeline_id) diff --git a/test_runner/performance/test_startup.py b/test_runner/performance/test_startup.py index fa2e058491..8babbbe132 100644 --- a/test_runner/performance/test_startup.py +++ b/test_runner/performance/test_startup.py @@ -1,10 +1,71 @@ from contextlib import closing import pytest -from fixtures.benchmark_fixture import NeonBenchmarker +import requests +from fixtures.benchmark_fixture import MetricReport, NeonBenchmarker from fixtures.neon_fixtures import NeonEnvBuilder +# Just start and measure duration. +# +# This test runs pretty quickly and can be informative when used in combination +# with emulated network delay. Some useful delay commands: +# +# 1. Add 2msec delay to all localhost traffic +# `sudo tc qdisc add dev lo root handle 1:0 netem delay 2msec` +# +# 2. Test that it works (you should see 4ms ping) +# `ping localhost` +# +# 3. Revert back to normal +# `sudo tc qdisc del dev lo root netem` +# +# NOTE this test might not represent the real startup time because the basebackup +# for a large database might be larger if there's a lof of transaction metadata, +# or safekeepers might need more syncing, or there might be more operations to +# apply during config step, like more users, databases, or extensions. By default +# we load extensions 'neon,pg_stat_statements,timescaledb,pg_cron', but in this +# test we only load neon. +def test_startup_simple(neon_env_builder: NeonEnvBuilder, zenbenchmark: NeonBenchmarker): + neon_env_builder.num_safekeepers = 3 + env = neon_env_builder.init_start() + + env.neon_cli.create_branch("test_startup") + + endpoint = None + + # We do two iterations so we can see if the second startup is faster. It should + # be because the compute node should already be configured with roles, databases, + # extensions, etc from the first run. + for i in range(2): + # Start + with zenbenchmark.record_duration(f"{i}_start_and_select"): + if endpoint: + endpoint.start() + else: + endpoint = env.endpoints.create_start("test_startup") + endpoint.safe_psql("select 1;") + + # Get metrics + metrics = requests.get(f"http://localhost:{endpoint.http_port}/metrics.json").json() + durations = { + "wait_for_spec_ms": f"{i}_wait_for_spec", + "sync_safekeepers_ms": f"{i}_sync_safekeepers", + "basebackup_ms": f"{i}_basebackup", + "config_ms": f"{i}_config", + "total_startup_ms": f"{i}_total_startup", + } + for key, name in durations.items(): + value = metrics[key] + zenbenchmark.record(name, value, "ms", report=MetricReport.LOWER_IS_BETTER) + + # Stop so we can restart + endpoint.stop() + + # Imitate optimizations that console would do for the second start + endpoint.respec(skip_pg_catalog_updates=True) + + # This test sometimes runs for longer than the global 5 minute timeout. @pytest.mark.timeout(600) def test_startup(neon_env_builder: NeonEnvBuilder, zenbenchmark: NeonBenchmarker): diff --git a/test_runner/pg_clients/python/pg8000/requirements.txt b/test_runner/pg_clients/python/pg8000/requirements.txt index 7bba8da06d..a8407c3cb0 100644 --- a/test_runner/pg_clients/python/pg8000/requirements.txt +++ b/test_runner/pg_clients/python/pg8000/requirements.txt @@ -1,2 +1,2 @@ -pg8000==1.29.4 +pg8000==1.29.8 scramp>=1.4.3 diff --git a/test_runner/pg_clients/rust/tokio-postgres/Cargo.lock b/test_runner/pg_clients/rust/tokio-postgres/Cargo.lock index 30deb3ff20..bdbbe0ad69 100644 --- a/test_runner/pg_clients/rust/tokio-postgres/Cargo.lock +++ b/test_runner/pg_clients/rust/tokio-postgres/Cargo.lock @@ -396,9 +396,9 @@ checksum = "b7e5500299e16ebb147ae15a00a942af264cf3688f47923b8fc2cd5858f23ad3" [[package]] name = "openssl" -version = "0.10.52" +version = "0.10.55" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01b8574602df80f7b85fdfc5392fa884a4e3b3f4f35402c070ab34c3d3f78d56" +checksum = "345df152bc43501c5eb9e4654ff05f794effb78d4efe3d53abc158baddc0703d" dependencies = [ "bitflags", "cfg-if", @@ -428,9 +428,9 @@ checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" [[package]] name = "openssl-sys" -version = "0.9.87" +version = "0.9.90" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e17f59264b2809d77ae94f0e1ebabc434773f370d6ca667bd223ea10e06cc7e" +checksum = "374533b0e45f3a7ced10fcaeccca020e66656bc03dac384f852e4e5a7a8104a6" dependencies = [ "cc", "libc", diff --git a/test_runner/pg_clients/rust/tokio-postgres/Dockerfile b/test_runner/pg_clients/rust/tokio-postgres/Dockerfile index 43fc6f6c92..35ae25a470 100644 --- a/test_runner/pg_clients/rust/tokio-postgres/Dockerfile +++ b/test_runner/pg_clients/rust/tokio-postgres/Dockerfile @@ -1,4 +1,4 @@ -FROM rust:1.69 +FROM rust:1.70 WORKDIR /source COPY . . diff --git a/test_runner/pg_clients/swift/PostgresNIOExample/Package.resolved b/test_runner/pg_clients/swift/PostgresNIOExample/Package.resolved index cc12acda4c..9f13106011 100644 --- a/test_runner/pg_clients/swift/PostgresNIOExample/Package.resolved +++ b/test_runner/pg_clients/swift/PostgresNIOExample/Package.resolved @@ -5,8 +5,8 @@ "kind" : "remoteSourceControl", "location" : "https://github.com/vapor/postgres-nio.git", "state" : { - "revision" : "dbf9c2eb596df39cba8ff3f74d74b2e6a31bd937", - "version" : "1.14.1" + "revision" : "061a0836d7c1887e04a975d1d2eaa2ef5fd7dfab", + "version" : "1.16.0" } }, { @@ -59,8 +59,8 @@ "kind" : "remoteSourceControl", "location" : "https://github.com/apple/swift-nio.git", "state" : { - "revision" : "d1690f85419fdac8d54e350fb6d2ab9fd95afd75", - "version" : "2.51.1" + "revision" : "6213ba7a06febe8fef60563a4a7d26a4085783cf", + "version" : "2.54.0" } }, { diff --git a/test_runner/pg_clients/swift/PostgresNIOExample/Package.swift b/test_runner/pg_clients/swift/PostgresNIOExample/Package.swift index ac32b982e2..a80590daa2 100644 --- a/test_runner/pg_clients/swift/PostgresNIOExample/Package.swift +++ b/test_runner/pg_clients/swift/PostgresNIOExample/Package.swift @@ -4,7 +4,7 @@ import PackageDescription let package = Package( name: "PostgresNIOExample", dependencies: [ - .package(url: "https://github.com/vapor/postgres-nio.git", from: "1.14.1") + .package(url: "https://github.com/vapor/postgres-nio.git", from: "1.16.0") ], targets: [ .executableTarget( diff --git a/test_runner/pg_clients/typescript/postgresql-client/package-lock.json b/test_runner/pg_clients/typescript/postgresql-client/package-lock.json index e4dfd1dd9d..4cedf56acd 100644 --- a/test_runner/pg_clients/typescript/postgresql-client/package-lock.json +++ b/test_runner/pg_clients/typescript/postgresql-client/package-lock.json @@ -5,23 +5,7 @@ "packages": { "": { "dependencies": { - "postgresql-client": "2.5.5" - } - }, - "node_modules/debug": { - "version": "4.3.4", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz", - "integrity": "sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==", - "dependencies": { - "ms": "2.1.2" - }, - "engines": { - "node": ">=6.0" - }, - "peerDependenciesMeta": { - "supports-color": { - "optional": true - } + "postgresql-client": "2.5.9" } }, "node_modules/doublylinked": { @@ -41,11 +25,6 @@ "putil-promisify": "^1.8.6" } }, - "node_modules/ms": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", - "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" - }, "node_modules/obuf": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/obuf/-/obuf-1.1.2.tgz", @@ -63,30 +42,28 @@ } }, "node_modules/postgresql-client": { - "version": "2.5.5", - "resolved": "https://registry.npmjs.org/postgresql-client/-/postgresql-client-2.5.5.tgz", - "integrity": "sha512-2Mu3i+6NQ9cnkoZNd0XeSZo9WoUpuWf4ZSiCCoDWSj82T93py2/SKXZ1aUaP8mVaU0oKpyyGe0IwLYZ1VHShnA==", + "version": "2.5.9", + "resolved": "https://registry.npmjs.org/postgresql-client/-/postgresql-client-2.5.9.tgz", + "integrity": "sha512-s+kgTN6TfWLzehEyxw4Im4odnxVRCbZ0DEJzWS6SLowPAmB2m1/DOiOvZC0+ZVoi5AfbGE6SBqFxKguSyVAXZg==", "dependencies": { - "debug": "^4.3.4", "doublylinked": "^2.5.2", "lightning-pool": "^4.2.1", "postgres-bytea": "^3.0.0", - "power-tasks": "^1.6.4", + "power-tasks": "^1.7.0", "putil-merge": "^3.10.3", "putil-promisify": "^1.10.0", "putil-varhelpers": "^1.6.5" }, "engines": { - "node": ">=14.0", + "node": ">=16.0", "npm": ">=7.0.0" } }, "node_modules/power-tasks": { - "version": "1.6.4", - "resolved": "https://registry.npmjs.org/power-tasks/-/power-tasks-1.6.4.tgz", - "integrity": "sha512-LX8GGgEIP1N7jsZqlqZ275e6f1Ehq97APCEGj8uVO0NoEoB+77QUX12BFv3LmlNKfq4fIuNSPiHhyHFjqn2gfA==", + "version": "1.7.0", + "resolved": "https://registry.npmjs.org/power-tasks/-/power-tasks-1.7.0.tgz", + "integrity": "sha512-rndZXCDxhuIDjPUJJvQwBDHaYagCkjvbPF/NA+omh/Ef4rAI9KtnvdA0k98dyiGpn1zXOpc6c2c0JWzg/xAhJg==", "dependencies": { - "debug": "^4.3.4", "doublylinked": "^2.5.2", "strict-typed-events": "^2.3.1" }, @@ -132,9 +109,9 @@ } }, "node_modules/ts-gems": { - "version": "2.3.0", - "resolved": "https://registry.npmjs.org/ts-gems/-/ts-gems-2.3.0.tgz", - "integrity": "sha512-bUvrwrzlct7vfaNvtgMhynDf6lAki/kTtrNsIGhX6l7GJGK3s6b8Ro7dazOLXabV0m2jyShBzDQ8X1+h/C2Cug==" + "version": "2.4.0", + "resolved": "https://registry.npmjs.org/ts-gems/-/ts-gems-2.4.0.tgz", + "integrity": "sha512-SdugYAXoWvbqrxLodIObzxhEKacDxh5LfAJIiIkiH7q5thvuuCzdmkdTVQYf7uEDrEpPhfx4tokDMamdO3be9A==" } } } diff --git a/test_runner/pg_clients/typescript/postgresql-client/package.json b/test_runner/pg_clients/typescript/postgresql-client/package.json index 9eaa13437a..12703ce89f 100644 --- a/test_runner/pg_clients/typescript/postgresql-client/package.json +++ b/test_runner/pg_clients/typescript/postgresql-client/package.json @@ -1,6 +1,6 @@ { "type": "module", "dependencies": { - "postgresql-client": "2.5.5" + "postgresql-client": "2.5.9" } } diff --git a/test_runner/pg_clients/typescript/serverless-driver/Dockerfile b/test_runner/pg_clients/typescript/serverless-driver/Dockerfile index a5ad832a5c..07e98c586b 100644 --- a/test_runner/pg_clients/typescript/serverless-driver/Dockerfile +++ b/test_runner/pg_clients/typescript/serverless-driver/Dockerfile @@ -1,4 +1,4 @@ -FROM node:18 +FROM node:20 WORKDIR /source COPY . . diff --git a/test_runner/pg_clients/typescript/serverless-driver/package-lock.json b/test_runner/pg_clients/typescript/serverless-driver/package-lock.json index 0fb84cf5b7..72cc452817 100644 --- a/test_runner/pg_clients/typescript/serverless-driver/package-lock.json +++ b/test_runner/pg_clients/typescript/serverless-driver/package-lock.json @@ -5,16 +5,16 @@ "packages": { "": { "dependencies": { - "@neondatabase/serverless": "0.4.3", + "@neondatabase/serverless": "0.4.18", "ws": "8.13.0" } }, "node_modules/@neondatabase/serverless": { - "version": "0.4.3", - "resolved": "https://registry.npmjs.org/@neondatabase/serverless/-/serverless-0.4.3.tgz", - "integrity": "sha512-U8tpuF5f0R5WRsciR7iaJ5S2h54DWa6Z6CEW+J4KgwyvRN3q3qDz0MibdfFXU0WqnRoi/9RSf/2XN4TfeaOCbQ==", + "version": "0.4.18", + "resolved": "https://registry.npmjs.org/@neondatabase/serverless/-/serverless-0.4.18.tgz", + "integrity": "sha512-2TZnIyRGC/+0fjZ8TKCzaSTPUD94PM7NBGuantGZbUrbWyqBwGnUoRtdZAQ95qBKVHqORLVfymlv2NE+HQMFeA==", "dependencies": { - "@types/pg": "^8.6.6" + "@types/pg": "8.6.6" } }, "node_modules/@types/node": { diff --git a/test_runner/pg_clients/typescript/serverless-driver/package.json b/test_runner/pg_clients/typescript/serverless-driver/package.json index 71ba181afc..840c7a5c4c 100644 --- a/test_runner/pg_clients/typescript/serverless-driver/package.json +++ b/test_runner/pg_clients/typescript/serverless-driver/package.json @@ -1,7 +1,7 @@ { "type": "module", "dependencies": { - "@neondatabase/serverless": "0.4.3", + "@neondatabase/serverless": "0.4.18", "ws": "8.13.0" } } diff --git a/test_runner/regress/test_attach_tenant_config.py b/test_runner/regress/test_attach_tenant_config.py index eb2ba3e9ed..4df5ae18d6 100644 --- a/test_runner/regress/test_attach_tenant_config.py +++ b/test_runner/regress/test_attach_tenant_config.py @@ -20,6 +20,11 @@ def positive_env(neon_env_builder: NeonEnvBuilder) -> NeonEnv: test_name="test_attach_tenant_config", ) env = neon_env_builder.init_start() + + # eviction might be the first one after an attach to access the layers + env.pageserver.allowed_errors.append( + ".*unexpectedly on-demand downloading remote layer remote.* for task kind Eviction" + ) assert isinstance(env.remote_storage, LocalFsStorage) return env @@ -158,6 +163,7 @@ def test_fully_custom_config(positive_env: NeonEnv): "threshold": "23h", }, "evictions_low_residence_duration_metric_threshold": "2days", + "gc_feedback": True, "gc_horizon": 23 * (1024 * 1024), "gc_period": "2h 13m", "image_creation_threshold": 7, diff --git a/test_runner/regress/test_broken_timeline.py b/test_runner/regress/test_broken_timeline.py index fb592bfbc3..0fb3b4f262 100644 --- a/test_runner/regress/test_broken_timeline.py +++ b/test_runner/regress/test_broken_timeline.py @@ -20,7 +20,7 @@ def test_broken_timeline(neon_env_builder: NeonEnvBuilder): ".*is not active. Current state: Broken.*", ".*will not become active. Current state: Broken.*", ".*failed to load metadata.*", - ".*could not load tenant.*load local timeline.*", + ".*load failed.*load local timeline.*", ] ) diff --git a/test_runner/regress/test_compatibility.py b/test_runner/regress/test_compatibility.py index fe8dc293c1..51e7b01eba 100644 --- a/test_runner/regress/test_compatibility.py +++ b/test_runner/regress/test_compatibility.py @@ -2,6 +2,7 @@ import copy import os import shutil import subprocess +import tempfile from pathlib import Path from typing import Any, Optional @@ -15,7 +16,11 @@ from fixtures.neon_fixtures import ( PortDistributor, ) from fixtures.pageserver.http import PageserverHttpClient -from fixtures.pageserver.utils import wait_for_last_record_lsn, wait_for_upload +from fixtures.pageserver.utils import ( + timeline_delete_wait_completed, + wait_for_last_record_lsn, + wait_for_upload, +) from fixtures.pg_version import PgVersion from fixtures.types import Lsn from pytest import FixtureRequest @@ -383,6 +388,9 @@ def check_neon_works( cli_target = NeonCli(config_target) # And the current binaries to launch computes + snapshot_config["neon_distrib_dir"] = str(neon_current_binpath) + with (snapshot_config_toml).open("w") as f: + toml.dump(snapshot_config, f) config_current = copy.copy(config) config_current.neon_binpath = neon_current_binpath cli_current = NeonCli(config_current) @@ -391,7 +399,8 @@ def check_neon_works( request.addfinalizer(lambda: cli_target.raw_cli(["stop"])) pg_port = port_distributor.get_port() - cli_current.endpoint_start("main", port=pg_port) + http_port = port_distributor.get_port() + cli_current.endpoint_start("main", pg_port=pg_port, http_port=http_port) request.addfinalizer(lambda: cli_current.endpoint_stop("main")) connstr = f"host=127.0.0.1 port={pg_port} user=cloud_admin dbname=postgres" @@ -413,7 +422,7 @@ def check_neon_works( ) shutil.rmtree(repo_dir / "local_fs_remote_storage") - pageserver_http.timeline_delete(tenant_id, timeline_id) + timeline_delete_wait_completed(pageserver_http, tenant_id, timeline_id) pageserver_http.timeline_create(pg_version, tenant_id, timeline_id) pg_bin.run( ["pg_dumpall", f"--dbname={connstr}", f"--file={test_output_dir / 'dump-from-wal.sql'}"] @@ -440,7 +449,7 @@ def dump_differs(first: Path, second: Path, output: Path) -> bool: """ with output.open("w") as stdout: - rv = subprocess.run( + res = subprocess.run( [ "diff", "--unified", # Make diff output more readable @@ -452,4 +461,53 @@ def dump_differs(first: Path, second: Path, output: Path) -> bool: stdout=stdout, ) - return rv.returncode != 0 + differs = res.returncode != 0 + + # TODO: Remove after https://github.com/neondatabase/neon/pull/4425 is merged, and a couple of releases are made + if differs: + with tempfile.NamedTemporaryFile(mode="w") as tmp: + tmp.write(PR4425_ALLOWED_DIFF) + tmp.flush() + + allowed = subprocess.run( + [ + "diff", + "--unified", # Make diff output more readable + r"--ignore-matching-lines=^---", # Ignore diff headers + r"--ignore-matching-lines=^\+\+\+", # Ignore diff headers + "--ignore-matching-lines=^@@", # Ignore diff blocks location + "--ignore-matching-lines=^ *$", # Ignore lines with only spaces + "--ignore-matching-lines=^ --.*", # Ignore the " --" lines for compatibility with PG14 + "--ignore-blank-lines", + str(output), + str(tmp.name), + ], + ) + + differs = allowed.returncode != 0 + + return differs + + +PR4425_ALLOWED_DIFF = """ +--- /tmp/test_output/test_backward_compatibility[release-pg15]/compatibility_snapshot/dump.sql 2023-06-08 18:12:45.000000000 +0000 ++++ /tmp/test_output/test_backward_compatibility[release-pg15]/dump.sql 2023-06-13 07:25:35.211733653 +0000 +@@ -13,12 +13,20 @@ + + CREATE ROLE cloud_admin; + ALTER ROLE cloud_admin WITH SUPERUSER INHERIT CREATEROLE CREATEDB LOGIN REPLICATION BYPASSRLS; ++CREATE ROLE neon_superuser; ++ALTER ROLE neon_superuser WITH NOSUPERUSER INHERIT CREATEROLE CREATEDB NOLOGIN NOREPLICATION NOBYPASSRLS; + + -- + -- User Configurations + -- + + ++-- ++-- Role memberships ++-- ++ ++GRANT pg_read_all_data TO neon_superuser GRANTED BY cloud_admin; ++GRANT pg_write_all_data TO neon_superuser GRANTED BY cloud_admin; +""" diff --git a/test_runner/regress/test_compute_ctl.py b/test_runner/regress/test_compute_ctl.py deleted file mode 100644 index d72ffe078d..0000000000 --- a/test_runner/regress/test_compute_ctl.py +++ /dev/null @@ -1,253 +0,0 @@ -import os -from pathlib import Path -from subprocess import TimeoutExpired - -from fixtures.log_helper import log -from fixtures.neon_fixtures import ComputeCtl, NeonEnvBuilder, PgBin - - -# Test that compute_ctl works and prints "--sync-safekeepers" logs. -def test_sync_safekeepers_logs(neon_env_builder: NeonEnvBuilder, pg_bin: PgBin): - neon_env_builder.num_safekeepers = 3 - env = neon_env_builder.init_start() - ctl = ComputeCtl(env) - - env.neon_cli.create_branch("test_compute_ctl", "main") - endpoint = env.endpoints.create_start("test_compute_ctl") - endpoint.safe_psql("CREATE TABLE t(key int primary key, value text)") - - with open(endpoint.config_file_path(), "r") as f: - cfg_lines = f.readlines() - cfg_map = {} - for line in cfg_lines: - if "=" in line: - k, v = line.split("=") - cfg_map[k] = v.strip("\n '\"") - log.info(f"postgres config: {cfg_map}") - pgdata = endpoint.pg_data_dir_path() - pg_bin_path = os.path.join(pg_bin.pg_bin_path, "postgres") - - endpoint.stop_and_destroy() - - # stop_and_destroy removes the whole endpoint directory. Recreate it. - Path(pgdata).mkdir(parents=True) - - spec = ( - """ -{ - "format_version": 1.0, - - "timestamp": "2021-05-23T18:25:43.511Z", - "operation_uuid": "0f657b36-4b0f-4a2d-9c2e-1dcd615e7d8b", - - "cluster": { - "cluster_id": "test-cluster-42", - "name": "Neon Test", - "state": "restarted", - "roles": [ - ], - "databases": [ - ], - "settings": [ - { - "name": "fsync", - "value": "off", - "vartype": "bool" - }, - { - "name": "wal_level", - "value": "replica", - "vartype": "enum" - }, - { - "name": "neon.safekeepers", - "value": """ - + f'"{cfg_map["neon.safekeepers"]}"' - + """, - "vartype": "string" - }, - { - "name": "wal_log_hints", - "value": "on", - "vartype": "bool" - }, - { - "name": "log_connections", - "value": "on", - "vartype": "bool" - }, - { - "name": "shared_buffers", - "value": "32768", - "vartype": "integer" - }, - { - "name": "port", - "value": """ - + f'"{cfg_map["port"]}"' - + """, - "vartype": "integer" - }, - { - "name": "max_connections", - "value": "100", - "vartype": "integer" - }, - { - "name": "max_wal_senders", - "value": "10", - "vartype": "integer" - }, - { - "name": "listen_addresses", - "value": "0.0.0.0", - "vartype": "string" - }, - { - "name": "wal_sender_timeout", - "value": "0", - "vartype": "integer" - }, - { - "name": "password_encryption", - "value": "md5", - "vartype": "enum" - }, - { - "name": "maintenance_work_mem", - "value": "65536", - "vartype": "integer" - }, - { - "name": "max_parallel_workers", - "value": "8", - "vartype": "integer" - }, - { - "name": "max_worker_processes", - "value": "8", - "vartype": "integer" - }, - { - "name": "neon.tenant_id", - "value": """ - + f'"{cfg_map["neon.tenant_id"]}"' - + """, - "vartype": "string" - }, - { - "name": "max_replication_slots", - "value": "10", - "vartype": "integer" - }, - { - "name": "neon.timeline_id", - "value": """ - + f'"{cfg_map["neon.timeline_id"]}"' - + """, - "vartype": "string" - }, - { - "name": "shared_preload_libraries", - "value": "neon", - "vartype": "string" - }, - { - "name": "synchronous_standby_names", - "value": "walproposer", - "vartype": "string" - }, - { - "name": "neon.pageserver_connstring", - "value": """ - + f'"{cfg_map["neon.pageserver_connstring"]}"' - + """, - "vartype": "string" - } - ] - }, - "delta_operations": [ - ] -} -""" - ) - - ps_connstr = cfg_map["neon.pageserver_connstring"] - log.info(f"ps_connstr: {ps_connstr}, pgdata: {pgdata}") - - # run compute_ctl and wait for 10s - try: - ctl.raw_cli( - [ - "--connstr", - "postgres://invalid/", - "--pgdata", - pgdata, - "--spec", - spec, - "--pgbin", - pg_bin_path, - ], - timeout=10, - ) - except TimeoutExpired as exc: - ctl_logs = (exc.stderr or b"").decode("utf-8") - log.info(f"compute_ctl stderr:\n{ctl_logs}") - - with ExternalProcessManager(Path(pgdata) / "postmaster.pid"): - start = "starting safekeepers syncing" - end = "safekeepers synced at LSN" - start_pos = ctl_logs.index(start) - assert start_pos != -1 - end_pos = ctl_logs.index(end, start_pos) - assert end_pos != -1 - sync_safekeepers_logs = ctl_logs[start_pos : end_pos + len(end)] - log.info("sync_safekeepers_logs:\n" + sync_safekeepers_logs) - - # assert that --sync-safekeepers logs are present in the output - assert "connecting with node" in sync_safekeepers_logs - assert "connected with node" in sync_safekeepers_logs - assert "proposer connected to quorum (2)" in sync_safekeepers_logs - assert "got votes from majority (2)" in sync_safekeepers_logs - assert "sending elected msg to node" in sync_safekeepers_logs - - -class ExternalProcessManager: - """ - Context manager that kills a process with a pid file on exit. - """ - - def __init__(self, pid_file: Path): - self.path = pid_file - self.pid_file = open(pid_file, "r") - self.pid = int(self.pid_file.readline().strip()) - - def __enter__(self): - return self - - def leave_alive(self): - self.pid_file.close() - - def __exit__(self, _type, _value, _traceback): - import signal - import time - - if self.pid_file.closed: - return - - with self.pid_file: - try: - os.kill(self.pid, signal.SIGTERM) - except OSError as e: - if not self.path.is_file(): - return - log.info(f"Failed to kill {self.pid}, but the pidfile remains: {e}") - return - - for _ in range(20): - if not self.path.is_file(): - return - time.sleep(0.2) - - log.info("Process failed to stop after SIGTERM: {self.pid}") - os.kill(self.pid, signal.SIGKILL) diff --git a/test_runner/regress/test_ddl_forwarding.py b/test_runner/regress/test_ddl_forwarding.py index 6bfa8fdbe7..ebd836ecbc 100644 --- a/test_runner/regress/test_ddl_forwarding.py +++ b/test_runner/regress/test_ddl_forwarding.py @@ -33,6 +33,7 @@ def handle_role(dbs, roles, operation): dbs[db] = operation["name"] if "password" in operation: roles[operation["name"]] = operation["password"] + assert "encrypted_password" in operation elif operation["op"] == "del": if "old_name" in operation: roles.pop(operation["old_name"]) diff --git a/test_runner/regress/test_disk_usage_eviction.py b/test_runner/regress/test_disk_usage_eviction.py index ab67518092..fc2d29c2c1 100644 --- a/test_runner/regress/test_disk_usage_eviction.py +++ b/test_runner/regress/test_disk_usage_eviction.py @@ -1,7 +1,6 @@ import shutil import time from dataclasses import dataclass -from pathlib import Path from typing import Dict, Tuple import pytest @@ -110,6 +109,12 @@ class EvictionEnv: overrides=( "--pageserver-config-override=disk_usage_based_eviction=" + enc.dump_inline_table(disk_usage_config).replace("\n", " "), + # Disk usage based eviction runs as a background task. + # But pageserver startup delays launch of background tasks for some time, to prioritize initial logical size calculations during startup. + # But, initial logical size calculation may not be triggered if safekeepers don't publish new broker messages. + # But, we only have a 10-second-timeout in this test. + # So, disable the delay for this test. + "--pageserver-config-override=background_task_maximum_delay='0s'", ), ) @@ -422,14 +427,14 @@ def poor_mans_du( largest_layer = 0 smallest_layer = None for tenant_id, timeline_id in timelines: - dir = Path(env.repo_dir) / "tenants" / str(tenant_id) / "timelines" / str(timeline_id) - assert dir.exists(), f"timeline dir does not exist: {dir}" - sum = 0 - for file in dir.iterdir(): + timeline_dir = env.timeline_dir(tenant_id, timeline_id) + assert timeline_dir.exists(), f"timeline dir does not exist: {timeline_dir}" + total = 0 + for file in timeline_dir.iterdir(): if "__" not in file.name: continue size = file.stat().st_size - sum += size + total += size largest_layer = max(largest_layer, size) if smallest_layer: smallest_layer = min(smallest_layer, size) @@ -437,8 +442,8 @@ def poor_mans_du( smallest_layer = size log.info(f"{tenant_id}/{timeline_id} => {file.name} {size}") - log.info(f"{tenant_id}/{timeline_id}: sum {sum}") - total_on_disk += sum + log.info(f"{tenant_id}/{timeline_id}: sum {total}") + total_on_disk += total assert smallest_layer is not None or total_on_disk == 0 and largest_layer == 0 return (total_on_disk, largest_layer, smallest_layer or 0) diff --git a/test_runner/regress/test_hot_standby.py b/test_runner/regress/test_hot_standby.py index 12e034cea2..6b003ce356 100644 --- a/test_runner/regress/test_hot_standby.py +++ b/test_runner/regress/test_hot_standby.py @@ -1,3 +1,5 @@ +import time + import pytest from fixtures.neon_fixtures import NeonEnv @@ -10,9 +12,10 @@ def test_hot_standby(neon_simple_env: NeonEnv): branch_name="main", endpoint_id="primary", ) as primary: + time.sleep(1) with env.endpoints.new_replica_start(origin=primary, endpoint_id="secondary") as secondary: primary_lsn = None - cought_up = False + caught_up = False queries = [ "SHOW neon.timeline_id", "SHOW neon.tenant_id", @@ -56,7 +59,7 @@ def test_hot_standby(neon_simple_env: NeonEnv): res = s_cur.fetchone() assert res is not None - while not cought_up: + while not caught_up: with s_con.cursor() as secondary_cursor: secondary_cursor.execute("SELECT pg_last_wal_replay_lsn()") res = secondary_cursor.fetchone() @@ -66,7 +69,7 @@ def test_hot_standby(neon_simple_env: NeonEnv): # due to e.g. autovacuum, but that shouldn't impact the content # of the tables, so we check whether we've replayed up to at # least after the commit of the `test` table. - cought_up = secondary_lsn >= primary_lsn + caught_up = secondary_lsn >= primary_lsn # Explicit commit to flush any transient transaction-level state. s_con.commit() diff --git a/test_runner/regress/test_import.py b/test_runner/regress/test_import.py index 77030288f0..141c69b230 100644 --- a/test_runner/regress/test_import.py +++ b/test_runner/regress/test_import.py @@ -14,7 +14,11 @@ from fixtures.neon_fixtures import ( NeonEnvBuilder, PgBin, ) -from fixtures.pageserver.utils import wait_for_last_record_lsn, wait_for_upload +from fixtures.pageserver.utils import ( + timeline_delete_wait_completed, + wait_for_last_record_lsn, + wait_for_upload, +) from fixtures.types import Lsn, TenantId, TimelineId from fixtures.utils import subprocess_capture @@ -79,6 +83,7 @@ def test_import_from_vanilla(test_output_dir, pg_bin, vanilla_pg, neon_env_build # Set up pageserver for import neon_env_builder.enable_local_fs_remote_storage() env = neon_env_builder.init_start() + client = env.pageserver.http_client() client.tenant_create(tenant) @@ -145,7 +150,12 @@ def test_import_from_vanilla(test_output_dir, pg_bin, vanilla_pg, neon_env_build ) # NOTE: delete can easily come before upload operations are completed - client.timeline_delete(tenant, timeline) + # https://github.com/neondatabase/neon/issues/4326 + env.pageserver.allowed_errors.append( + ".*files not bound to index_file.json, proceeding with their deletion.*" + ) + + timeline_delete_wait_completed(client, tenant, timeline) # Importing correct backup works import_tar(base_tar, wal_tar) diff --git a/test_runner/regress/test_layer_eviction.py b/test_runner/regress/test_layer_eviction.py index a96532c0d8..d4f0c94a29 100644 --- a/test_runner/regress/test_layer_eviction.py +++ b/test_runner/regress/test_layer_eviction.py @@ -24,7 +24,13 @@ def test_basic_eviction( test_name="test_download_remote_layers_api", ) - env = neon_env_builder.init_start() + env = neon_env_builder.init_start( + initial_tenant_conf={ + # disable gc and compaction background loops because they perform on-demand downloads + "gc_period": "0s", + "compaction_period": "0s", + } + ) client = env.pageserver.http_client() endpoint = env.endpoints.create_start("main") @@ -47,7 +53,12 @@ def test_basic_eviction( client.timeline_checkpoint(tenant_id, timeline_id) wait_for_upload(client, tenant_id, timeline_id, current_lsn) - timeline_path = env.repo_dir / "tenants" / str(tenant_id) / "timelines" / str(timeline_id) + # disable compute & sks to avoid on-demand downloads by walreceiver / getpage + endpoint.stop() + for sk in env.safekeepers: + sk.stop() + + timeline_path = env.timeline_dir(tenant_id, timeline_id) initial_local_layers = sorted( list(filter(lambda path: path.name != "metadata", timeline_path.glob("*"))) ) diff --git a/test_runner/regress/test_neon_local_cli.py b/test_runner/regress/test_neon_local_cli.py index f6629c54f9..3314e7fbf6 100644 --- a/test_runner/regress/test_neon_local_cli.py +++ b/test_runner/regress/test_neon_local_cli.py @@ -9,11 +9,18 @@ def test_neon_cli_basics(neon_env_builder: NeonEnvBuilder, port_distributor: Por try: env.neon_cli.start() env.neon_cli.create_tenant(tenant_id=env.initial_tenant, set_default=True) - env.neon_cli.endpoint_start(endpoint_id="ep-main", port=port_distributor.get_port()) + + pg_port = port_distributor.get_port() + http_port = port_distributor.get_port() + env.neon_cli.endpoint_start( + endpoint_id="ep-basic-main", pg_port=pg_port, http_port=http_port + ) env.neon_cli.create_branch(new_branch_name="migration_check") + pg_port = port_distributor.get_port() + http_port = port_distributor.get_port() env.neon_cli.endpoint_start( - endpoint_id="ep-migration_check", port=port_distributor.get_port() + endpoint_id="ep-migration_check", pg_port=pg_port, http_port=http_port ) finally: env.neon_cli.stop() diff --git a/test_runner/regress/test_ondemand_download.py b/test_runner/regress/test_ondemand_download.py index 1414b4ed8e..a30f0f02e1 100644 --- a/test_runner/regress/test_ondemand_download.py +++ b/test_runner/regress/test_ondemand_download.py @@ -58,11 +58,8 @@ def test_ondemand_download_large_rel( ) ##### First start, insert secret data and upload it to the remote storage - env = neon_env_builder.init_start() - - # Override defaults, to create more layers - tenant, _ = env.neon_cli.create_tenant( - conf={ + env = neon_env_builder.init_start( + initial_tenant_conf={ # disable background GC "gc_period": "0s", "gc_horizon": f"{10 * 1024 ** 3}", # 10 GB @@ -75,7 +72,6 @@ def test_ondemand_download_large_rel( "compaction_period": "0s", } ) - env.initial_tenant = tenant endpoint = env.endpoints.create_start("main") @@ -717,9 +713,7 @@ def test_ondemand_download_failure_to_replace( # error message is not useful pageserver_http.timeline_detail(tenant_id, timeline_id, True, timeout=2) - actual_message = ( - ".* ERROR .*replacing downloaded layer into layermap failed because layer was not found" - ) + actual_message = ".* ERROR .*layermap-replace-notfound" assert env.pageserver.log_contains(actual_message) is not None env.pageserver.allowed_errors.append(actual_message) diff --git a/test_runner/regress/test_pageserver_restarts_under_workload.py b/test_runner/regress/test_pageserver_restarts_under_workload.py index bc3f3f2be4..fc93dcffbb 100644 --- a/test_runner/regress/test_pageserver_restarts_under_workload.py +++ b/test_runner/regress/test_pageserver_restarts_under_workload.py @@ -17,12 +17,6 @@ def test_pageserver_restarts_under_worload(neon_simple_env: NeonEnv, pg_bin: PgB n_restarts = 10 scale = 10 - # the background task may complete the init task delay after finding an - # active tenant, but shutdown starts right before Tenant::gc_iteration - env.pageserver.allowed_errors.append( - r".*Gc failed, retrying in \S+: Cannot run GC iteration on inactive tenant" - ) - def run_pgbench(connstr: str): log.info(f"Start a pgbench workload on pg {connstr}") pg_bin.run_capture(["pgbench", "-i", f"-s{scale}", connstr]) diff --git a/test_runner/regress/test_proxy.py b/test_runner/regress/test_proxy.py index 6be3995714..24c5b42b5a 100644 --- a/test_runner/regress/test_proxy.py +++ b/test_runner/regress/test_proxy.py @@ -163,7 +163,6 @@ def test_forward_params_to_client(static_proxy: NeonProxy): assert conn.get_parameter_status(name) == value -@pytest.mark.timeout(5) def test_close_on_connections_exit(static_proxy: NeonProxy): # Open two connections, send SIGTERM, then ensure that proxy doesn't exit # until after connections close. @@ -225,3 +224,37 @@ def test_sql_over_http(static_proxy: NeonProxy): res = q("drop table t") assert res["command"] == "DROP" assert res["rowCount"] is None + + +def test_sql_over_http_output_options(static_proxy: NeonProxy): + static_proxy.safe_psql("create role http2 with login password 'http2' superuser") + + def q(sql: str, raw_text: bool, array_mode: bool, params: List[Any] = []) -> Any: + connstr = ( + f"postgresql://http2:http2@{static_proxy.domain}:{static_proxy.proxy_port}/postgres" + ) + response = requests.post( + f"https://{static_proxy.domain}:{static_proxy.external_http_port}/sql", + data=json.dumps({"query": sql, "params": params}), + headers={ + "Content-Type": "application/sql", + "Neon-Connection-String": connstr, + "Neon-Raw-Text-Output": "true" if raw_text else "false", + "Neon-Array-Mode": "true" if array_mode else "false", + }, + verify=str(static_proxy.test_output_dir / "proxy.crt"), + ) + assert response.status_code == 200 + return response.json() + + rows = q("select 1 as n, 'a' as s, '{1,2,3}'::int4[] as arr", False, False)["rows"] + assert rows == [{"arr": [1, 2, 3], "n": 1, "s": "a"}] + + rows = q("select 1 as n, 'a' as s, '{1,2,3}'::int4[] as arr", False, True)["rows"] + assert rows == [[1, "a", [1, 2, 3]]] + + rows = q("select 1 as n, 'a' as s, '{1,2,3}'::int4[] as arr", True, False)["rows"] + assert rows == [{"arr": "{1,2,3}", "n": "1", "s": "a"}] + + rows = q("select 1 as n, 'a' as s, '{1,2,3}'::int4[] as arr", True, True)["rows"] + assert rows == [["1", "a", "{1,2,3}"]] diff --git a/test_runner/regress/test_remote_storage.py b/test_runner/regress/test_remote_storage.py index 02f1aac99c..9a70266314 100644 --- a/test_runner/regress/test_remote_storage.py +++ b/test_runner/regress/test_remote_storage.py @@ -20,6 +20,7 @@ from fixtures.neon_fixtures import ( ) from fixtures.pageserver.http import PageserverApiException, PageserverHttpClient from fixtures.pageserver.utils import ( + timeline_delete_wait_completed, wait_for_last_record_lsn, wait_for_upload, wait_until_tenant_active, @@ -140,14 +141,19 @@ def test_remote_storage_backup_and_restore( # This is before the failures injected by test_remote_failures, so it's a permanent error. pageserver_http.configure_failpoints(("storage-sync-list-remote-timelines", "return")) env.pageserver.allowed_errors.append( - ".*error attaching tenant: storage-sync-list-remote-timelines", + ".*attach failed.*: storage-sync-list-remote-timelines", ) # Attach it. This HTTP request will succeed and launch a # background task to load the tenant. In that background task, # listing the remote timelines will fail because of the failpoint, # and the tenant will be marked as Broken. client.tenant_attach(tenant_id) - wait_until_tenant_state(pageserver_http, tenant_id, "Broken", 15) + + tenant_info = wait_until_tenant_state(pageserver_http, tenant_id, "Broken", 15) + assert tenant_info["attachment_status"] == { + "slug": "failed", + "data": {"reason": "storage-sync-list-remote-timelines"}, + } # Ensure that even though the tenant is broken, we can't attach it again. with pytest.raises(Exception, match=f"tenant {tenant_id} already exists, state: Broken"): @@ -177,7 +183,7 @@ def test_remote_storage_backup_and_restore( wait_until_tenant_active( pageserver_http=client, tenant_id=tenant_id, - iterations=5, + iterations=10, # make it longer for real_s3 tests when unreliable wrapper is involved ) detail = client.timeline_detail(tenant_id, timeline_id) @@ -529,7 +535,7 @@ def test_timeline_deletion_with_files_stuck_in_upload_queue( "pitr_interval": "0s", } ) - timeline_path = env.repo_dir / "tenants" / str(tenant_id) / "timelines" / str(timeline_id) + timeline_path = env.timeline_dir(tenant_id, timeline_id) client = env.pageserver.http_client() @@ -591,10 +597,22 @@ def test_timeline_deletion_with_files_stuck_in_upload_queue( env.pageserver.allowed_errors.append( ".* ERROR .*Error processing HTTP request: InternalServerError\\(timeline is Stopping" ) - client.timeline_delete(tenant_id, timeline_id) + + env.pageserver.allowed_errors.append( + ".*files not bound to index_file.json, proceeding with their deletion.*" + ) + timeline_delete_wait_completed(client, tenant_id, timeline_id) assert not timeline_path.exists() + # to please mypy + assert isinstance(env.remote_storage, LocalFsStorage) + remote_timeline_path = ( + env.remote_storage.root / "tenants" / str(tenant_id) / "timelines" / str(timeline_id) + ) + + assert not list(remote_timeline_path.iterdir()) + # timeline deletion should kill ongoing uploads, so, the metric will be gone assert get_queued_count(file_kind="index", op_kind="upload") is None @@ -693,15 +711,15 @@ def test_empty_branch_remote_storage_upload_on_restart( f".*POST.* path=/v1/tenant/{env.initial_tenant}/timeline.* request was dropped before completing" ) - # index upload is now hitting the failpoint, should not block the shutdown - env.pageserver.stop() + # index upload is now hitting the failpoint, it should block the shutdown + env.pageserver.stop(immediate=True) timeline_path = ( Path("tenants") / str(env.initial_tenant) / "timelines" / str(new_branch_timeline_id) ) local_metadata = env.repo_dir / timeline_path / "metadata" - assert local_metadata.is_file(), "timeout cancelled timeline branching, not the upload" + assert local_metadata.is_file() assert isinstance(env.remote_storage, LocalFsStorage) new_branch_on_remote_storage = env.remote_storage.root / timeline_path diff --git a/test_runner/regress/test_sni_router.py b/test_runner/regress/test_sni_router.py index f3aa429c49..9b78e8287e 100644 --- a/test_runner/regress/test_sni_router.py +++ b/test_runner/regress/test_sni_router.py @@ -37,6 +37,7 @@ class PgSniRouter(PgProtocol): destination: str, tls_cert: Path, tls_key: Path, + test_output_dir: Path, ): # Must use a hostname rather than IP here, for SNI to work host = "localhost" @@ -49,6 +50,7 @@ class PgSniRouter(PgProtocol): self.tls_cert = tls_cert self.tls_key = tls_key self._popen: Optional[subprocess.Popen[bytes]] = None + self.test_output_dir = test_output_dir def start(self) -> "PgSniRouter": assert self._popen is None @@ -60,8 +62,12 @@ class PgSniRouter(PgProtocol): *["--destination", self.destination], ] - self._popen = subprocess.Popen(args) + router_log_path = self.test_output_dir / "pg_sni_router.log" + router_log = open(router_log_path, "w") + + self._popen = subprocess.Popen(args, stderr=router_log) self._wait_until_ready() + log.info(f"pg_sni_router started, log file: {router_log_path}") return self @backoff.on_exception(backoff.expo, OSError, max_time=10) @@ -121,6 +127,7 @@ def test_pg_sni_router( destination="localtest.me", tls_cert=test_output_dir / "router.crt", tls_key=test_output_dir / "router.key", + test_output_dir=test_output_dir, ) as router: router.start() diff --git a/test_runner/regress/test_tenant_detach.py b/test_runner/regress/test_tenant_detach.py index 82664cff94..2ded79954e 100644 --- a/test_runner/regress/test_tenant_detach.py +++ b/test_runner/regress/test_tenant_detach.py @@ -59,6 +59,13 @@ def test_tenant_reattach( # create new nenant tenant_id, timeline_id = env.neon_cli.create_tenant() + # Attempts to connect from compute to pageserver while the tenant is + # temporarily detached produces these errors in the pageserver log. + env.pageserver.allowed_errors.append(f".*Tenant {tenant_id} not found.*") + env.pageserver.allowed_errors.append( + f".*Tenant {tenant_id} will not become active\\. Current state: Stopping.*" + ) + with env.endpoints.create_start("main", tenant_id=tenant_id) as endpoint: with endpoint.cursor() as cur: cur.execute("CREATE TABLE t(key int primary key, value text)") @@ -223,13 +230,6 @@ def test_tenant_reattach_while_busy( ) env = neon_env_builder.init_start() - # Attempts to connect from compute to pageserver while the tenant is - # temporarily detached produces these errors in the pageserver log. - env.pageserver.allowed_errors.append(".*Tenant .* not found.*") - env.pageserver.allowed_errors.append( - ".*Tenant .* will not become active\\. Current state: Stopping.*" - ) - pageserver_http = env.pageserver.http_client() # create new nenant @@ -238,6 +238,13 @@ def test_tenant_reattach_while_busy( conf={"checkpoint_distance": "100000"} ) + # Attempts to connect from compute to pageserver while the tenant is + # temporarily detached produces these errors in the pageserver log. + env.pageserver.allowed_errors.append(f".*Tenant {tenant_id} not found.*") + env.pageserver.allowed_errors.append( + f".*Tenant {tenant_id} will not become active\\. Current state: Stopping.*" + ) + endpoint = env.endpoints.create_start("main", tenant_id=tenant_id) cur = endpoint.connect().cursor() @@ -275,6 +282,13 @@ def test_tenant_detach_smoke(neon_env_builder: NeonEnvBuilder): # create new nenant tenant_id, timeline_id = env.neon_cli.create_tenant() + # Attempts to connect from compute to pageserver while the tenant is + # temporarily detached produces these errors in the pageserver log. + env.pageserver.allowed_errors.append(f".*Tenant {tenant_id} not found.*") + env.pageserver.allowed_errors.append( + f".*Tenant {tenant_id} will not become active\\. Current state: Stopping.*" + ) + # assert tenant exists on disk assert (env.repo_dir / "tenants" / str(tenant_id)).exists() @@ -336,6 +350,13 @@ def test_tenant_detach_ignored_tenant(neon_simple_env: NeonEnv): # create a new tenant tenant_id, _ = env.neon_cli.create_tenant() + # Attempts to connect from compute to pageserver while the tenant is + # temporarily detached produces these errors in the pageserver log. + env.pageserver.allowed_errors.append(f".*Tenant {tenant_id} not found.*") + env.pageserver.allowed_errors.append( + f".*Tenant {tenant_id} will not become active\\. Current state: Stopping.*" + ) + # assert tenant exists on disk assert (env.repo_dir / "tenants" / str(tenant_id)).exists() @@ -385,6 +406,13 @@ def test_tenant_detach_regular_tenant(neon_simple_env: NeonEnv): # create a new tenant tenant_id, _ = env.neon_cli.create_tenant() + # Attempts to connect from compute to pageserver while the tenant is + # temporarily detached produces these errors in the pageserver log. + env.pageserver.allowed_errors.append(f".*Tenant {tenant_id} not found.*") + env.pageserver.allowed_errors.append( + f".*Tenant {tenant_id} will not become active\\. Current state: Stopping.*" + ) + # assert tenant exists on disk assert (env.repo_dir / "tenants" / str(tenant_id)).exists() @@ -399,6 +427,7 @@ def test_tenant_detach_regular_tenant(neon_simple_env: NeonEnv): log.info("detaching regular tenant with detach ignored flag") client.tenant_detach(tenant_id, True) + log.info("regular tenant detached without error") # check that nothing is left on disk for deleted tenant @@ -432,6 +461,13 @@ def test_detach_while_attaching( tenant_id = TenantId(endpoint.safe_psql("show neon.tenant_id")[0][0]) timeline_id = TimelineId(endpoint.safe_psql("show neon.timeline_id")[0][0]) + # Attempts to connect from compute to pageserver while the tenant is + # temporarily detached produces these errors in the pageserver log. + env.pageserver.allowed_errors.append(f".*Tenant {tenant_id} not found.*") + env.pageserver.allowed_errors.append( + f".*Tenant {tenant_id} will not become active\\. Current state: Stopping.*" + ) + # Create table, and insert some rows. Make it big enough that it doesn't fit in # shared_buffers, otherwise the SELECT after restart will just return answer # from shared_buffers without hitting the page server, which defeats the point @@ -496,7 +532,7 @@ def test_ignored_tenant_reattach( ): neon_env_builder.enable_remote_storage( remote_storage_kind=remote_storage_kind, - test_name="test_remote_storage_backup_and_restore", + test_name="test_ignored_tenant_reattach", ) env = neon_env_builder.init_start() pageserver_http = env.pageserver.http_client() @@ -577,6 +613,13 @@ def test_ignored_tenant_download_missing_layers( tenant_id = TenantId(endpoint.safe_psql("show neon.tenant_id")[0][0]) timeline_id = TimelineId(endpoint.safe_psql("show neon.timeline_id")[0][0]) + # Attempts to connect from compute to pageserver while the tenant is + # temporarily detached produces these errors in the pageserver log. + env.pageserver.allowed_errors.append(f".*Tenant {tenant_id} not found.*") + env.pageserver.allowed_errors.append( + f".*Tenant {tenant_id} will not become active\\. Current state: Stopping.*" + ) + data_id = 1 data_secret = "very secret secret" insert_test_data(pageserver_http, tenant_id, timeline_id, data_id, data_secret, endpoint) @@ -589,14 +632,14 @@ def test_ignored_tenant_download_missing_layers( # ignore the tenant and remove its layers pageserver_http.tenant_ignore(tenant_id) - tenant_timeline_dir = env.repo_dir / "tenants" / str(tenant_id) / "timelines" / str(timeline_id) + timeline_dir = env.timeline_dir(tenant_id, timeline_id) layers_removed = False - for dir_entry in tenant_timeline_dir.iterdir(): + for dir_entry in timeline_dir.iterdir(): if dir_entry.name.startswith("00000"): # Looks like a layer file. Remove it dir_entry.unlink() layers_removed = True - assert layers_removed, f"Found no layers for tenant {tenant_timeline_dir}" + assert layers_removed, f"Found no layers for tenant {timeline_dir}" # now, load it from the local files and expect it to work due to remote storage restoration pageserver_http.tenant_load(tenant_id=tenant_id) @@ -636,18 +679,27 @@ def test_ignored_tenant_stays_broken_without_metadata( tenant_id = TenantId(endpoint.safe_psql("show neon.tenant_id")[0][0]) timeline_id = TimelineId(endpoint.safe_psql("show neon.timeline_id")[0][0]) + # Attempts to connect from compute to pageserver while the tenant is + # temporarily detached produces these errors in the pageserver log. + env.pageserver.allowed_errors.append(f".*Tenant {tenant_id} not found.*") + env.pageserver.allowed_errors.append( + f".*Tenant {tenant_id} will not become active\\. Current state: Broken.*" + ) + # ignore the tenant and remove its metadata pageserver_http.tenant_ignore(tenant_id) - tenant_timeline_dir = env.repo_dir / "tenants" / str(tenant_id) / "timelines" / str(timeline_id) + timeline_dir = env.timeline_dir(tenant_id, timeline_id) metadata_removed = False - for dir_entry in tenant_timeline_dir.iterdir(): + for dir_entry in timeline_dir.iterdir(): if dir_entry.name == "metadata": # Looks like a layer file. Remove it dir_entry.unlink() metadata_removed = True - assert metadata_removed, f"Failed to find metadata file in {tenant_timeline_dir}" + assert metadata_removed, f"Failed to find metadata file in {timeline_dir}" - env.pageserver.allowed_errors.append(".*could not load tenant .*?: failed to load metadata.*") + env.pageserver.allowed_errors.append( + f".*{tenant_id}.*: load failed.*: failed to load metadata.*" + ) # now, load it from the local files and expect it to be broken due to inability to load tenant files into memory pageserver_http.tenant_load(tenant_id=tenant_id) @@ -670,6 +722,13 @@ def test_load_attach_negatives( tenant_id = TenantId(endpoint.safe_psql("show neon.tenant_id")[0][0]) + # Attempts to connect from compute to pageserver while the tenant is + # temporarily detached produces these errors in the pageserver log. + env.pageserver.allowed_errors.append(f".*Tenant {tenant_id} not found.*") + env.pageserver.allowed_errors.append( + f".*Tenant {tenant_id} will not become active\\. Current state: Stopping.*" + ) + env.pageserver.allowed_errors.append(".*tenant .*? already exists, state:.*") with pytest.raises( expected_exception=PageserverApiException, @@ -712,6 +771,13 @@ def test_ignore_while_attaching( tenant_id = TenantId(endpoint.safe_psql("show neon.tenant_id")[0][0]) timeline_id = TimelineId(endpoint.safe_psql("show neon.timeline_id")[0][0]) + # Attempts to connect from compute to pageserver while the tenant is + # temporarily detached produces these errors in the pageserver log. + env.pageserver.allowed_errors.append(f".*Tenant {tenant_id} not found.*") + env.pageserver.allowed_errors.append( + f".*Tenant {tenant_id} will not become active\\. Current state: Stopping.*" + ) + data_id = 1 data_secret = "very secret secret" insert_test_data(pageserver_http, tenant_id, timeline_id, data_id, data_secret, endpoint) diff --git a/test_runner/regress/test_tenant_relocation.py b/test_runner/regress/test_tenant_relocation.py index 2a5b30803b..9043c29060 100644 --- a/test_runner/regress/test_tenant_relocation.py +++ b/test_runner/regress/test_tenant_relocation.py @@ -214,9 +214,7 @@ def switch_pg_to_new_pageserver( endpoint.start() - timeline_to_detach_local_path = ( - env.repo_dir / "tenants" / str(tenant_id) / "timelines" / str(timeline_id) - ) + timeline_to_detach_local_path = env.timeline_dir(tenant_id, timeline_id) files_before_detach = os.listdir(timeline_to_detach_local_path) assert ( "metadata" in files_before_detach @@ -419,8 +417,6 @@ def test_tenant_relocation( new_pageserver_http.tenant_attach(tenant_id) # wait for tenant to finish attaching - tenant_status = new_pageserver_http.tenant_status(tenant_id=tenant_id) - assert tenant_status["state"]["slug"] in ["Attaching", "Active"] wait_until( number_of_iterations=10, interval=1, diff --git a/test_runner/regress/test_tenant_size.py b/test_runner/regress/test_tenant_size.py index 60ab268882..25c6634108 100644 --- a/test_runner/regress/test_tenant_size.py +++ b/test_runner/regress/test_tenant_size.py @@ -11,10 +11,12 @@ from fixtures.neon_fixtures import ( wait_for_wal_insert_lsn, ) from fixtures.pageserver.http import PageserverHttpClient +from fixtures.pageserver.utils import timeline_delete_wait_completed from fixtures.pg_version import PgVersion, xfail_on_postgres from fixtures.types import Lsn, TenantId, TimelineId +@pytest.mark.xfail def test_empty_tenant_size(neon_simple_env: NeonEnv, test_output_dir: Path): env = neon_simple_env (tenant_id, _) = env.neon_cli.create_tenant() @@ -43,12 +45,16 @@ def test_empty_tenant_size(neon_simple_env: NeonEnv, test_output_dir: Path): # we've disabled the autovacuum and checkpoint # so background processes should not change the size. # If this test will flake we should probably loosen the check - assert size == initial_size, "starting idle compute should not change the tenant size" + assert ( + size == initial_size + ), f"starting idle compute should not change the tenant size (Currently {size}, expected {initial_size})" # the size should be the same, until we increase the size over the # gc_horizon size, inputs = http_client.tenant_size_and_modelinputs(tenant_id) - assert size == initial_size, "tenant_size should not be affected by shutdown of compute" + assert ( + size == initial_size + ), f"tenant_size should not be affected by shutdown of compute (Currently {size}, expected {initial_size})" expected_inputs = { "segments": [ @@ -317,8 +323,9 @@ def test_only_heads_within_horizon(neon_simple_env: NeonEnv, test_output_dir: Pa size_debug_file.write(size_debug) +@pytest.mark.xfail def test_single_branch_get_tenant_size_grows( - neon_env_builder: NeonEnvBuilder, test_output_dir: Path + neon_env_builder: NeonEnvBuilder, test_output_dir: Path, pg_version: PgVersion ): """ Operate on single branch reading the tenants size after each transaction. @@ -332,7 +339,14 @@ def test_single_branch_get_tenant_size_grows( # inserts is larger than gc_horizon. for example 0x20000 here hid the fact # that there next_gc_cutoff could be smaller than initdb_lsn, which will # obviously lead to issues when calculating the size. - gc_horizon = 0x38000 + gc_horizon = 0x3BA00 + + # it's a bit of a hack, but different versions of postgres have different + # amount of WAL generated for the same amount of data. so we need to + # adjust the gc_horizon accordingly. + if pg_version == PgVersion.V14: + gc_horizon = 0x4A000 + neon_env_builder.pageserver_config_override = f"tenant_config={{compaction_period='0s', gc_period='0s', pitr_interval='0sec', gc_horizon={gc_horizon}}}" env = neon_env_builder.init_start() @@ -352,11 +366,11 @@ def test_single_branch_get_tenant_size_grows( if current_lsn - initdb_lsn >= gc_horizon: assert ( size >= prev_size - ), "tenant_size may grow or not grow, because we only add gc_horizon amount of WAL to initial snapshot size" + ), f"tenant_size may grow or not grow, because we only add gc_horizon amount of WAL to initial snapshot size (Currently at: {current_lsn}, Init at: {initdb_lsn})" else: assert ( size > prev_size - ), "tenant_size should grow, because we continue to add WAL to initial snapshot size" + ), f"tenant_size should grow, because we continue to add WAL to initial snapshot size (Currently at: {current_lsn}, Init at: {initdb_lsn})" def get_current_consistent_size( env: NeonEnv, @@ -621,12 +635,12 @@ def test_get_tenant_size_with_multiple_branches( size_debug_file_before.write(size_debug) # teardown, delete branches, and the size should be going down - http_client.timeline_delete(tenant_id, first_branch_timeline_id) + timeline_delete_wait_completed(http_client, tenant_id, first_branch_timeline_id) size_after_deleting_first = http_client.tenant_size(tenant_id) assert size_after_deleting_first < size_after_thinning_branch - http_client.timeline_delete(tenant_id, second_branch_timeline_id) + timeline_delete_wait_completed(http_client, tenant_id, second_branch_timeline_id) size_after_deleting_second = http_client.tenant_size(tenant_id) assert size_after_deleting_second < size_after_deleting_first diff --git a/test_runner/regress/test_tenant_tasks.py b/test_runner/regress/test_tenant_tasks.py index 21e4af4127..75e5c2c91c 100644 --- a/test_runner/regress/test_tenant_tasks.py +++ b/test_runner/regress/test_tenant_tasks.py @@ -1,6 +1,10 @@ from fixtures.log_helper import log from fixtures.neon_fixtures import NeonEnvBuilder -from fixtures.pageserver.utils import assert_tenant_state, wait_until_tenant_active +from fixtures.pageserver.utils import ( + assert_tenant_state, + timeline_delete_wait_completed, + wait_until_tenant_active, +) from fixtures.types import TenantId, TimelineId from fixtures.utils import wait_until @@ -24,7 +28,7 @@ def test_tenant_tasks(neon_env_builder: NeonEnvBuilder): def delete_all_timelines(tenant: TenantId): timelines = [TimelineId(t["timeline_id"]) for t in client.timeline_list(tenant)] for t in timelines: - client.timeline_delete(tenant, t) + timeline_delete_wait_completed(client, tenant, t) # Create tenant, start compute tenant, _ = env.neon_cli.create_tenant() diff --git a/test_runner/regress/test_tenants.py b/test_runner/regress/test_tenants.py index 37edce6bb0..acbb8321d7 100644 --- a/test_runner/regress/test_tenants.py +++ b/test_runner/regress/test_tenants.py @@ -21,8 +21,9 @@ from fixtures.neon_fixtures import ( RemoteStorageKind, available_remote_storages, ) -from fixtures.pageserver.http import PageserverApiException +from fixtures.pageserver.utils import timeline_delete_wait_completed from fixtures.types import Lsn, TenantId, TimelineId +from fixtures.utils import wait_until from prometheus_client.samples import Sample @@ -213,7 +214,7 @@ def test_metrics_normal_work(neon_env_builder: NeonEnvBuilder): # Test (a subset of) pageserver global metrics for metric in PAGESERVER_GLOBAL_METRICS: ps_samples = ps_metrics.query_all(metric, {}) - assert len(ps_samples) > 0 + assert len(ps_samples) > 0, f"expected at least one sample for {metric}" for sample in ps_samples: labels = ",".join([f'{key}="{value}"' for key, value in sample.labels.items()]) log.info(f"{sample.name}{{{labels}}} {sample.value}") @@ -267,6 +268,7 @@ def test_pageserver_metrics_removed_after_detach( cur.execute("INSERT INTO t SELECT generate_series(1,100000), 'payload'") cur.execute("SELECT sum(key) FROM t") assert cur.fetchone() == (5000050000,) + endpoint.stop() def get_ps_metric_samples_for_tenant(tenant_id: TenantId) -> List[Sample]: ps_metrics = env.pageserver.http_client().get_metrics() @@ -309,9 +311,7 @@ def test_pageserver_with_empty_tenants( env.pageserver.allowed_errors.append( ".*marking .* as locally complete, while it doesnt exist in remote index.*" ) - env.pageserver.allowed_errors.append( - ".*could not load tenant.*Failed to list timelines directory.*" - ) + env.pageserver.allowed_errors.append(".*load failed.*list timelines directory.*") client = env.pageserver.http_client() @@ -319,9 +319,10 @@ def test_pageserver_with_empty_tenants( client.tenant_create(tenant_with_empty_timelines) temp_timelines = client.timeline_list(tenant_with_empty_timelines) for temp_timeline in temp_timelines: - client.timeline_delete( - tenant_with_empty_timelines, TimelineId(temp_timeline["timeline_id"]) + timeline_delete_wait_completed( + client, tenant_with_empty_timelines, TimelineId(temp_timeline["timeline_id"]) ) + files_in_timelines_dir = sum( 1 for _p in Path.iterdir( @@ -342,9 +343,15 @@ def test_pageserver_with_empty_tenants( env.pageserver.start() client = env.pageserver.http_client() - tenants = client.tenant_list() - assert len(tenants) == 2 + def not_loading(): + tenants = client.tenant_list() + assert len(tenants) == 2 + assert all(t["state"]["slug"] != "Loading" for t in tenants) + + wait_until(10, 0.2, not_loading) + + tenants = client.tenant_list() [broken_tenant] = [t for t in tenants if t["id"] == str(tenant_without_timelines_dir)] assert ( @@ -356,7 +363,7 @@ def test_pageserver_with_empty_tenants( broken_tenant_status["state"]["slug"] == "Broken" ), f"Tenant {tenant_without_timelines_dir} without timelines dir should be broken" - assert env.pageserver.log_contains(".*Setting tenant as Broken state, reason:.*") + assert env.pageserver.log_contains(".*load failed, setting tenant state to Broken:.*") [loaded_tenant] = [t for t in tenants if t["id"] == str(tenant_with_empty_timelines)] assert ( diff --git a/test_runner/regress/test_tenants_with_remote_storage.py b/test_runner/regress/test_tenants_with_remote_storage.py index dca2cd3d28..98f9e94276 100644 --- a/test_runner/regress/test_tenants_with_remote_storage.py +++ b/test_runner/regress/test_tenants_with_remote_storage.py @@ -257,7 +257,7 @@ def test_tenant_redownloads_truncated_file_on_startup( env.endpoints.stop_all() env.pageserver.stop() - timeline_dir = Path(env.repo_dir) / "tenants" / str(tenant_id) / "timelines" / str(timeline_id) + timeline_dir = env.timeline_dir(tenant_id, timeline_id) local_layer_truncated = None for path in Path.iterdir(timeline_dir): if path.name.startswith("00000"): diff --git a/test_runner/regress/test_timeline_delete.py b/test_runner/regress/test_timeline_delete.py index 7135b621cb..7c3424cf32 100644 --- a/test_runner/regress/test_timeline_delete.py +++ b/test_runner/regress/test_timeline_delete.py @@ -3,6 +3,7 @@ import queue import shutil import threading from pathlib import Path +from typing import Optional import pytest import requests @@ -11,13 +12,17 @@ from fixtures.neon_fixtures import ( NeonEnv, NeonEnvBuilder, RemoteStorageKind, + S3Storage, available_remote_storages, ) from fixtures.pageserver.http import PageserverApiException from fixtures.pageserver.utils import ( + timeline_delete_wait_completed, wait_for_last_record_lsn, wait_for_upload, + wait_timeline_detail_404, wait_until_tenant_active, + wait_until_timeline_state, ) from fixtures.types import Lsn, TenantId, TimelineId from fixtures.utils import query_scalar, wait_until @@ -68,7 +73,7 @@ def test_timeline_delete(neon_simple_env: NeonEnv): ps_http.timeline_delete(env.initial_tenant, parent_timeline_id) - assert exc.value.status_code == 400 + assert exc.value.status_code == 412 timeline_path = ( env.repo_dir / "tenants" / str(env.initial_tenant) / "timelines" / str(leaf_timeline_id) @@ -79,7 +84,7 @@ def test_timeline_delete(neon_simple_env: NeonEnv): wait_until( number_of_iterations=3, interval=0.2, - func=lambda: ps_http.timeline_delete(env.initial_tenant, leaf_timeline_id), + func=lambda: timeline_delete_wait_completed(ps_http, env.initial_tenant, leaf_timeline_id), ) assert not timeline_path.exists() @@ -90,16 +95,16 @@ def test_timeline_delete(neon_simple_env: NeonEnv): match=f"Timeline {env.initial_tenant}/{leaf_timeline_id} was not found", ) as exc: ps_http.timeline_detail(env.initial_tenant, leaf_timeline_id) - - # FIXME leaves tenant without timelines, should we prevent deletion of root timeline? - wait_until( - number_of_iterations=3, - interval=0.2, - func=lambda: ps_http.timeline_delete(env.initial_tenant, parent_timeline_id), - ) - assert exc.value.status_code == 404 + wait_until( + number_of_iterations=3, + interval=0.2, + func=lambda: timeline_delete_wait_completed( + ps_http, env.initial_tenant, parent_timeline_id + ), + ) + # Check that we didn't pick up the timeline again after restart. # See https://github.com/neondatabase/neon/issues/3560 env.pageserver.stop(immediate=True) @@ -130,13 +135,24 @@ def test_delete_timeline_post_rm_failure( env = neon_env_builder.init_start() assert env.initial_timeline + env.pageserver.allowed_errors.append(".*Error: failpoint: timeline-delete-after-rm") + env.pageserver.allowed_errors.append(".*Ignoring state update Stopping for broken timeline") + ps_http = env.pageserver.http_client() failpoint_name = "timeline-delete-after-rm" ps_http.configure_failpoints((failpoint_name, "return")) - with pytest.raises(PageserverApiException, match=f"failpoint: {failpoint_name}"): - ps_http.timeline_delete(env.initial_tenant, env.initial_timeline) + ps_http.timeline_delete(env.initial_tenant, env.initial_timeline) + timeline_info = wait_until_timeline_state( + pageserver_http=ps_http, + tenant_id=env.initial_tenant, + timeline_id=env.initial_timeline, + expected_state="Broken", + iterations=2, # effectively try immediately and retry once in one second + ) + + timeline_info["state"]["Broken"]["reason"] == "failpoint: timeline-delete-after-rm" at_failpoint_log_message = f".*{env.initial_timeline}.*at failpoint {failpoint_name}.*" env.pageserver.allowed_errors.append(at_failpoint_log_message) @@ -148,11 +164,8 @@ def test_delete_timeline_post_rm_failure( ps_http.configure_failpoints((failpoint_name, "off")) # this should succeed - ps_http.timeline_delete(env.initial_tenant, env.initial_timeline, timeout=2) - # the second call will try to transition the timeline into Stopping state, but it's already in that state - env.pageserver.allowed_errors.append( - f".*{env.initial_timeline}.*Ignoring new state, equal to the existing one: Stopping" - ) + # this also checks that delete can be retried even when timeline is in Broken state + timeline_delete_wait_completed(ps_http, env.initial_tenant, env.initial_timeline) env.pageserver.allowed_errors.append( f".*{env.initial_timeline}.*timeline directory not found, proceeding anyway.*" ) @@ -228,7 +241,7 @@ def test_timeline_resurrection_on_attach( pass # delete new timeline - ps_http.timeline_delete(tenant_id=tenant_id, timeline_id=branch_timeline_id) + timeline_delete_wait_completed(ps_http, tenant_id=tenant_id, timeline_id=branch_timeline_id) ##### Stop the pageserver instance, erase all its data env.endpoints.stop_all() @@ -252,12 +265,31 @@ def test_timeline_resurrection_on_attach( assert all([tl["state"] == "Active" for tl in timelines]) +def assert_prefix_empty(neon_env_builder: NeonEnvBuilder, prefix: Optional[str] = None): + # For local_fs we need to properly handle empty directories, which we currently dont, so for simplicity stick to s3 api. + assert neon_env_builder.remote_storage_kind in ( + RemoteStorageKind.MOCK_S3, + RemoteStorageKind.REAL_S3, + ) + # For mypy + assert isinstance(neon_env_builder.remote_storage, S3Storage) + + # Note that this doesnt use pagination, so list is not guaranteed to be exhaustive. + response = neon_env_builder.remote_storage_client.list_objects_v2( + Bucket=neon_env_builder.remote_storage.bucket_name, + Prefix=prefix or neon_env_builder.remote_storage.prefix_in_bucket or "", + ) + objects = response.get("Contents") + assert ( + response["KeyCount"] == 0 + ), f"remote dir with prefix {prefix} is not empty after deletion: {objects}" + + def test_timeline_delete_fail_before_local_delete(neon_env_builder: NeonEnvBuilder): """ When deleting a timeline, if we succeed in setting the deleted flag remotely but fail to delete the local state, restarting the pageserver should resume the deletion of the local state. - (Deletion of the state in S3 is not implemented yet.) """ neon_env_builder.enable_remote_storage( @@ -271,8 +303,9 @@ def test_timeline_delete_fail_before_local_delete(neon_env_builder: NeonEnvBuild env.pageserver.allowed_errors.append( ".*Ignoring new state, equal to the existing one: Stopping" ) + # this happens, because the stuck timeline is visible to shutdown env.pageserver.allowed_errors.append( - ".*during shutdown: cannot flush frozen layers when flush_loop is not running, state is Exited" + ".*freeze_and_flush_on_shutdown.+: failed to freeze and flush: cannot flush frozen layers when flush_loop is not running, state is Exited" ) ps_http = env.pageserver.http_client() @@ -292,11 +325,16 @@ def test_timeline_delete_fail_before_local_delete(neon_env_builder: NeonEnvBuild env.repo_dir / "tenants" / str(env.initial_tenant) / "timelines" / str(leaf_timeline_id) ) - with pytest.raises( - PageserverApiException, - match="failpoint: timeline-delete-before-rm", - ): - ps_http.timeline_delete(env.initial_tenant, leaf_timeline_id) + ps_http.timeline_delete(env.initial_tenant, leaf_timeline_id) + timeline_info = wait_until_timeline_state( + pageserver_http=ps_http, + tenant_id=env.initial_tenant, + timeline_id=leaf_timeline_id, + expected_state="Broken", + iterations=2, # effectively try immediately and retry once in one second + ) + + timeline_info["state"]["Broken"]["reason"] == "failpoint: timeline-delete-after-rm" assert leaf_timeline_path.exists(), "the failpoint didn't work" @@ -304,7 +342,17 @@ def test_timeline_delete_fail_before_local_delete(neon_env_builder: NeonEnvBuild env.pageserver.start() # Wait for tenant to finish loading. - wait_until_tenant_active(ps_http, tenant_id=env.initial_tenant, iterations=10, period=0.5) + wait_until_tenant_active(ps_http, tenant_id=env.initial_tenant, iterations=10, period=1) + + try: + data = ps_http.timeline_detail(env.initial_tenant, leaf_timeline_id) + log.debug(f"detail {data}") + except PageserverApiException as e: + log.debug(e) + if e.status_code != 404: + raise + else: + raise Exception("detail succeeded (it should return 404)") assert ( not leaf_timeline_path.exists() @@ -316,24 +364,67 @@ def test_timeline_delete_fail_before_local_delete(neon_env_builder: NeonEnvBuild }, "other timelines should not have been affected" assert all([tl["state"] == "Active" for tl in timelines]) + assert_prefix_empty( + neon_env_builder, + prefix="/".join( + ( + "tenants", + str(env.initial_tenant), + "timelines", + str(leaf_timeline_id), + ) + ), + ) -def test_concurrent_timeline_delete_if_first_stuck_at_index_upload( - neon_env_builder: NeonEnvBuilder, + assert env.initial_timeline is not None + + for timeline_id in (intermediate_timeline_id, env.initial_timeline): + timeline_delete_wait_completed( + ps_http, tenant_id=env.initial_tenant, timeline_id=timeline_id + ) + + assert_prefix_empty( + neon_env_builder, + prefix="/".join( + ( + "tenants", + str(env.initial_tenant), + "timelines", + str(timeline_id), + ) + ), + ) + + # for some reason the check above doesnt immediately take effect for the below. + # Assume it is mock server incosistency and check twice. + wait_until( + 2, + 0.5, + lambda: assert_prefix_empty(neon_env_builder), + ) + + +@pytest.mark.parametrize( + "stuck_failpoint", + ["persist_deleted_index_part", "in_progress_delete"], +) +def test_concurrent_timeline_delete_stuck_on( + neon_env_builder: NeonEnvBuilder, stuck_failpoint: str ): """ - If we're stuck uploading the index file with the is_delete flag, - eventually console will hand up and retry. - If we're still stuck at the retry time, ensure that the retry - fails with status 500, signalling to console that it should retry - later. - Ideally, timeline_delete should return 202 Accepted and require - console to poll for completion, but, that would require changing - the API contract. + If delete is stuck console will eventually retry deletion. + So we need to be sure that these requests wont interleave with each other. + In this tests we check two places where we can spend a lot of time. + This is a regression test because there was a bug when DeletionGuard wasnt propagated + to the background task. + + Ensure that when retry comes if we're still stuck request will get an immediate error response, + signalling to console that it should retry later. """ neon_env_builder.enable_remote_storage( remote_storage_kind=RemoteStorageKind.MOCK_S3, - test_name="test_concurrent_timeline_delete_if_first_stuck_at_index_upload", + test_name=f"concurrent_timeline_delete_stuck_on_{stuck_failpoint}", ) env = neon_env_builder.init_start() @@ -343,13 +434,14 @@ def test_concurrent_timeline_delete_if_first_stuck_at_index_upload( ps_http = env.pageserver.http_client() # make the first call sleep practically forever - failpoint_name = "persist_index_part_with_deleted_flag_after_set_before_upload_pause" - ps_http.configure_failpoints((failpoint_name, "pause")) + ps_http.configure_failpoints((stuck_failpoint, "pause")) def first_call(result_queue): try: log.info("first call start") - ps_http.timeline_delete(env.initial_tenant, child_timeline_id, timeout=10) + timeline_delete_wait_completed( + ps_http, env.initial_tenant, child_timeline_id, timeout=10 + ) log.info("first call success") result_queue.put("success") except Exception: @@ -364,17 +456,17 @@ def test_concurrent_timeline_delete_if_first_stuck_at_index_upload( def first_call_hit_failpoint(): assert env.pageserver.log_contains( - f".*{child_timeline_id}.*at failpoint {failpoint_name}" + f".*{child_timeline_id}.*at failpoint {stuck_failpoint}" ) wait_until(50, 0.1, first_call_hit_failpoint) # make the second call and assert behavior log.info("second call start") - error_msg_re = "another task is already setting the deleted_flag, started at" + error_msg_re = "Timeline deletion is already in progress" with pytest.raises(PageserverApiException, match=error_msg_re) as second_call_err: ps_http.timeline_delete(env.initial_tenant, child_timeline_id) - assert second_call_err.value.status_code == 500 + assert second_call_err.value.status_code == 409 env.pageserver.allowed_errors.append(f".*{child_timeline_id}.*{error_msg_re}.*") # the second call will try to transition the timeline into Stopping state as well env.pageserver.allowed_errors.append( @@ -382,8 +474,12 @@ def test_concurrent_timeline_delete_if_first_stuck_at_index_upload( ) log.info("second call failed as expected") + # ensure it is not 404 and stopping + detail = ps_http.timeline_detail(env.initial_tenant, child_timeline_id) + assert detail["state"] == "Stopping" + # by now we know that the second call failed, let's ensure the first call will finish - ps_http.configure_failpoints((failpoint_name, "off")) + ps_http.configure_failpoints((stuck_failpoint, "off")) result = first_call_result.get() assert result == "success" @@ -396,8 +492,10 @@ def test_concurrent_timeline_delete_if_first_stuck_at_index_upload( def test_delete_timeline_client_hangup(neon_env_builder: NeonEnvBuilder): """ - If the client hangs up before we start the index part upload but after we mark it + If the client hangs up before we start the index part upload but after deletion is scheduled + we mark it deleted in local memory, a subsequent delete_timeline call should be able to do + another delete timeline operation. This tests cancel safety up to the given failpoint. @@ -413,12 +511,18 @@ def test_delete_timeline_client_hangup(neon_env_builder: NeonEnvBuilder): ps_http = env.pageserver.http_client() - failpoint_name = "persist_index_part_with_deleted_flag_after_set_before_upload_pause" + failpoint_name = "persist_deleted_index_part" ps_http.configure_failpoints((failpoint_name, "pause")) with pytest.raises(requests.exceptions.Timeout): ps_http.timeline_delete(env.initial_tenant, child_timeline_id, timeout=2) + env.pageserver.allowed_errors.append( + f".*{child_timeline_id}.*Timeline deletion is already in progress.*" + ) + with pytest.raises(PageserverApiException, match="Timeline deletion is already in progress"): + ps_http.timeline_delete(env.initial_tenant, child_timeline_id, timeout=2) + # make sure the timeout was due to the failpoint at_failpoint_log_message = f".*{child_timeline_id}.*at failpoint {failpoint_name}.*" @@ -437,12 +541,96 @@ def test_delete_timeline_client_hangup(neon_env_builder: NeonEnvBuilder): wait_until(50, 0.1, got_hangup_log_message) - # ok, retry without failpoint, it should succeed + # check that the timeline is still present + ps_http.timeline_detail(env.initial_tenant, child_timeline_id) + + # ok, disable the failpoint to let the deletion finish ps_http.configure_failpoints((failpoint_name, "off")) - # this should succeed - ps_http.timeline_delete(env.initial_tenant, child_timeline_id, timeout=2) - # the second call will try to transition the timeline into Stopping state, but it's already in that state - env.pageserver.allowed_errors.append( - f".*{child_timeline_id}.*Ignoring new state, equal to the existing one: Stopping" + def first_request_finished(): + message = f".*DELETE.*{child_timeline_id}.*Cancelled request finished" + assert env.pageserver.log_contains(message) + + wait_until(50, 0.1, first_request_finished) + + # check that the timeline is gone + wait_timeline_detail_404(ps_http, env.initial_tenant, child_timeline_id) + + +@pytest.mark.parametrize( + "remote_storage_kind", + list( + filter( + lambda s: s in (RemoteStorageKind.MOCK_S3, RemoteStorageKind.REAL_S3), + available_remote_storages(), + ) + ), +) +def test_timeline_delete_works_for_remote_smoke( + neon_env_builder: NeonEnvBuilder, + remote_storage_kind: RemoteStorageKind, +): + neon_env_builder.enable_remote_storage( + remote_storage_kind=remote_storage_kind, + test_name="test_timeline_delete_works_for_remote_smoke", + ) + + env = neon_env_builder.init_start() + + ps_http = env.pageserver.http_client() + pg = env.endpoints.create_start("main") + + tenant_id = TenantId(pg.safe_psql("show neon.tenant_id")[0][0]) + main_timeline_id = TimelineId(pg.safe_psql("show neon.timeline_id")[0][0]) + + assert tenant_id == env.initial_tenant + assert main_timeline_id == env.initial_timeline + + timeline_ids = [env.initial_timeline] + for i in range(2): + branch_timeline_id = env.neon_cli.create_branch(f"new{i}", "main") + pg = env.endpoints.create_start(f"new{i}") + + with pg.cursor() as cur: + cur.execute("CREATE TABLE f (i integer);") + cur.execute("INSERT INTO f VALUES (generate_series(1,1000));") + current_lsn = Lsn(query_scalar(cur, "SELECT pg_current_wal_flush_lsn()")) + + # wait until pageserver receives that data + wait_for_last_record_lsn(ps_http, tenant_id, branch_timeline_id, current_lsn) + + # run checkpoint manually to be sure that data landed in remote storage + ps_http.timeline_checkpoint(tenant_id, branch_timeline_id) + + # wait until pageserver successfully uploaded a checkpoint to remote storage + log.info("waiting for checkpoint upload") + wait_for_upload(ps_http, tenant_id, branch_timeline_id, current_lsn) + log.info("upload of checkpoint is done") + timeline_id = TimelineId(pg.safe_psql("show neon.timeline_id")[0][0]) + + timeline_ids.append(timeline_id) + + for timeline_id in reversed(timeline_ids): + # note that we need to finish previous deletion before scheduling next one + # otherwise we can get an "HasChildren" error if deletion is not fast enough (real_s3) + timeline_delete_wait_completed(ps_http, tenant_id=tenant_id, timeline_id=timeline_id) + + assert_prefix_empty( + neon_env_builder, + prefix="/".join( + ( + "tenants", + str(env.initial_tenant), + "timelines", + str(timeline_id), + ) + ), + ) + + # for some reason the check above doesnt immediately take effect for the below. + # Assume it is mock server incosistency and check twice. + wait_until( + 2, + 0.5, + lambda: assert_prefix_empty(neon_env_builder), ) diff --git a/test_runner/regress/test_timeline_size.py b/test_runner/regress/test_timeline_size.py index 1460172afe..6338f4ca77 100644 --- a/test_runner/regress/test_timeline_size.py +++ b/test_runner/regress/test_timeline_size.py @@ -24,6 +24,7 @@ from fixtures.neon_fixtures import ( from fixtures.pageserver.http import PageserverApiException, PageserverHttpClient from fixtures.pageserver.utils import ( assert_tenant_state, + timeline_delete_wait_completed, wait_for_upload_queue_empty, wait_until_tenant_active, ) @@ -272,7 +273,7 @@ def test_timeline_initial_logical_size_calculation_cancellation( if deletion_method == "tenant_detach": client.tenant_detach(tenant_id) elif deletion_method == "timeline_delete": - client.timeline_delete(tenant_id, timeline_id) + timeline_delete_wait_completed(client, tenant_id, timeline_id) delete_timeline_success.put(True) except PageserverApiException: delete_timeline_success.put(False) @@ -415,6 +416,7 @@ def test_timeline_physical_size_post_compaction( wait_for_last_flush_lsn(env, endpoint, env.initial_tenant, new_timeline_id) # shutdown safekeepers to prevent new data from coming in + endpoint.stop() # We can't gracefully stop after safekeepers die for sk in env.safekeepers: sk.stop() diff --git a/test_runner/regress/test_wal_acceptor.py b/test_runner/regress/test_wal_acceptor.py index 2a4141ed30..5828d4306c 100644 --- a/test_runner/regress/test_wal_acceptor.py +++ b/test_runner/regress/test_wal_acceptor.py @@ -31,7 +31,11 @@ from fixtures.neon_fixtures import ( SafekeeperPort, available_remote_storages, ) -from fixtures.pageserver.utils import wait_for_last_record_lsn, wait_for_upload +from fixtures.pageserver.utils import ( + timeline_delete_wait_completed, + wait_for_last_record_lsn, + wait_for_upload, +) from fixtures.pg_version import PgVersion from fixtures.types import Lsn, TenantId, TimelineId from fixtures.utils import get_dir_size, query_scalar, start_in_background @@ -548,15 +552,15 @@ def test_s3_wal_replay(neon_env_builder: NeonEnvBuilder, remote_storage_kind: Re f"sk_id={sk.id} to flush {last_lsn}", ) - ps_cli = env.pageserver.http_client() - pageserver_lsn = Lsn(ps_cli.timeline_detail(tenant_id, timeline_id)["last_record_lsn"]) + ps_http = env.pageserver.http_client() + pageserver_lsn = Lsn(ps_http.timeline_detail(tenant_id, timeline_id)["last_record_lsn"]) lag = last_lsn - pageserver_lsn log.info( f"Pageserver last_record_lsn={pageserver_lsn}; flush_lsn={last_lsn}; lag before replay is {lag / 1024}kb" ) endpoint.stop_and_destroy() - ps_cli.timeline_delete(tenant_id, timeline_id) + timeline_delete_wait_completed(ps_http, tenant_id, timeline_id) # Also delete and manually create timeline on safekeepers -- this tests # scenario of manual recovery on different set of safekeepers. @@ -571,11 +575,21 @@ def test_s3_wal_replay(neon_env_builder: NeonEnvBuilder, remote_storage_kind: Re pg_version = sk.http_client().timeline_status(tenant_id, timeline_id).pg_version + # Terminate first all safekeepers to prevent communication unexpectantly + # advancing peer_horizon_lsn. for sk in env.safekeepers: cli = sk.http_client() cli.timeline_delete_force(tenant_id, timeline_id) # restart safekeeper to clear its in-memory state - sk.stop().start() + sk.stop() + # wait all potenital in flight pushes to broker arrive before starting + # safekeepers (even without sleep, it is very unlikely they are not + # delivered yet). + time.sleep(1) + + for sk in env.safekeepers: + sk.start() + cli = sk.http_client() cli.timeline_create(tenant_id, timeline_id, pg_version, last_lsn) f_partial_path = ( Path(sk.data_dir()) / str(tenant_id) / str(timeline_id) / f_partial_saved.name @@ -583,7 +597,7 @@ def test_s3_wal_replay(neon_env_builder: NeonEnvBuilder, remote_storage_kind: Re shutil.copy(f_partial_saved, f_partial_path) # recreate timeline on pageserver from scratch - ps_cli.timeline_create( + ps_http.timeline_create( pg_version=PgVersion(pg_version), tenant_id=tenant_id, new_timeline_id=timeline_id, @@ -598,7 +612,7 @@ def test_s3_wal_replay(neon_env_builder: NeonEnvBuilder, remote_storage_kind: Re if elapsed > wait_lsn_timeout: raise RuntimeError("Timed out waiting for WAL redo") - tenant_status = ps_cli.tenant_status(tenant_id) + tenant_status = ps_http.tenant_status(tenant_id) if tenant_status["state"]["slug"] == "Loading": log.debug(f"Tenant {tenant_id} is still loading, retrying") else: @@ -1001,9 +1015,6 @@ def test_safekeeper_without_pageserver( def test_replace_safekeeper(neon_env_builder: NeonEnvBuilder): - def safekeepers_guc(env: NeonEnv, sk_names: List[int]) -> str: - return ",".join([f"localhost:{sk.port.pg}" for sk in env.safekeepers if sk.id in sk_names]) - def execute_payload(endpoint: Endpoint): with closing(endpoint.connect()) as conn: with conn.cursor() as cur: @@ -1032,9 +1043,8 @@ def test_replace_safekeeper(neon_env_builder: NeonEnvBuilder): log.info("Use only first 3 safekeepers") env.safekeepers[3].stop() - active_safekeepers = [1, 2, 3] endpoint = env.endpoints.create("test_replace_safekeeper") - endpoint.adjust_for_safekeepers(safekeepers_guc(env, active_safekeepers)) + endpoint.active_safekeepers = [1, 2, 3] endpoint.start() # learn neon timeline from compute @@ -1072,9 +1082,8 @@ def test_replace_safekeeper(neon_env_builder: NeonEnvBuilder): log.info("Recreate postgres to replace failed sk1 with new sk4") endpoint.stop_and_destroy().create("test_replace_safekeeper") - active_safekeepers = [2, 3, 4] env.safekeepers[3].start() - endpoint.adjust_for_safekeepers(safekeepers_guc(env, active_safekeepers)) + endpoint.active_safekeepers = [2, 3, 4] endpoint.start() execute_payload(endpoint) @@ -1201,6 +1210,10 @@ def test_delete_force(neon_env_builder: NeonEnvBuilder, auth_enabled: bool): with conn.cursor() as cur: cur.execute("INSERT INTO t (key) VALUES (1)") + # Stop all computes gracefully before safekeepers stop responding to them + endpoint_1.stop_and_destroy() + endpoint_3.stop_and_destroy() + # Remove initial tenant's br1 (active) assert sk_http.timeline_delete_force(tenant_id, timeline_id_1)["dir_existed"] assert not (sk_data_dir / str(tenant_id) / str(timeline_id_1)).exists() @@ -1293,9 +1306,8 @@ def test_pull_timeline(neon_env_builder: NeonEnvBuilder): log.info("Use only first 3 safekeepers") env.safekeepers[3].stop() - active_safekeepers = [1, 2, 3] endpoint = env.endpoints.create("test_pull_timeline") - endpoint.adjust_for_safekeepers(safekeepers_guc(env, active_safekeepers)) + endpoint.active_safekeepers = [1, 2, 3] endpoint.start() # learn neon timeline from compute @@ -1332,10 +1344,8 @@ def test_pull_timeline(neon_env_builder: NeonEnvBuilder): show_statuses(env.safekeepers, tenant_id, timeline_id) log.info("Restarting compute with new config to verify that it works") - active_safekeepers = [1, 3, 4] - endpoint.stop_and_destroy().create("test_pull_timeline") - endpoint.adjust_for_safekeepers(safekeepers_guc(env, active_safekeepers)) + endpoint.active_safekeepers = [1, 3, 4] endpoint.start() execute_payload(endpoint) diff --git a/test_runner/regress/test_wal_acceptor_async.py b/test_runner/regress/test_wal_acceptor_async.py index 7debeed140..ce33975a0e 100644 --- a/test_runner/regress/test_wal_acceptor_async.py +++ b/test_runner/regress/test_wal_acceptor_async.py @@ -2,9 +2,11 @@ import asyncio import random import time from dataclasses import dataclass +from pathlib import Path from typing import List, Optional import asyncpg +import toml from fixtures.log_helper import getLogger from fixtures.neon_fixtures import Endpoint, NeonEnv, NeonEnvBuilder, Safekeeper from fixtures.types import Lsn, TenantId, TimelineId @@ -251,7 +253,8 @@ def endpoint_create_start(env: NeonEnv, branch: str, pgdir_name: Optional[str]): endpoint = Endpoint( env, tenant_id=env.initial_tenant, - port=env.port_distributor.get_port(), + pg_port=env.port_distributor.get_port(), + http_port=env.port_distributor.get_port(), # In these tests compute has high probability of terminating on its own # before our stop() due to lost consensus leadership. check_stop_result=False, @@ -536,15 +539,20 @@ def test_race_conditions(neon_env_builder: NeonEnvBuilder): # Check that pageserver can select safekeeper with largest commit_lsn # and switch if LSN is not updated for some time (NoWalTimeout). -async def run_wal_lagging(env: NeonEnv, endpoint: Endpoint): - def safekeepers_guc(env: NeonEnv, active_sk: List[bool]) -> str: - # use ports 10, 11 and 12 to simulate unavailable safekeepers - return ",".join( - [ - f"localhost:{sk.port.pg if active else 10 + i}" - for i, (sk, active) in enumerate(zip(env.safekeepers, active_sk)) - ] - ) +async def run_wal_lagging(env: NeonEnv, endpoint: Endpoint, test_output_dir: Path): + def adjust_safekeepers(env: NeonEnv, active_sk: List[bool]): + # Change the pg ports of the inactive safekeepers in the config file to be + # invalid, to make them unavailable to the endpoint. We use + # ports 10, 11 and 12 to simulate unavailable safekeepers. + config = toml.load(test_output_dir / "repo" / "config") + for i, (sk, active) in enumerate(zip(env.safekeepers, active_sk)): + if active: + config["safekeepers"][i]["pg_port"] = env.safekeepers[i].port.pg + else: + config["safekeepers"][i]["pg_port"] = 10 + i + + with open(test_output_dir / "repo" / "config", "w") as f: + toml.dump(config, f) conn = await endpoint.connect_async() await conn.execute("CREATE TABLE t(key int primary key, value text)") @@ -565,7 +573,7 @@ async def run_wal_lagging(env: NeonEnv, endpoint: Endpoint): it -= 1 continue - endpoint.adjust_for_safekeepers(safekeepers_guc(env, active_sk)) + adjust_safekeepers(env, active_sk) log.info(f"Iteration {it}: {active_sk}") endpoint.start() @@ -579,7 +587,7 @@ async def run_wal_lagging(env: NeonEnv, endpoint: Endpoint): await conn.close() endpoint.stop() - endpoint.adjust_for_safekeepers(safekeepers_guc(env, [True] * len(env.safekeepers))) + adjust_safekeepers(env, [True] * len(env.safekeepers)) endpoint.start() conn = await endpoint.connect_async() @@ -590,11 +598,11 @@ async def run_wal_lagging(env: NeonEnv, endpoint: Endpoint): # do inserts while restarting postgres and messing with safekeeper addresses -def test_wal_lagging(neon_env_builder: NeonEnvBuilder): +def test_wal_lagging(neon_env_builder: NeonEnvBuilder, test_output_dir: Path): neon_env_builder.num_safekeepers = 3 env = neon_env_builder.init_start() env.neon_cli.create_branch("test_wal_lagging") endpoint = env.endpoints.create_start("test_wal_lagging") - asyncio.run(run_wal_lagging(env, endpoint)) + asyncio.run(run_wal_lagging(env, endpoint, test_output_dir)) diff --git a/test_runner/regress/test_wal_receiver.py b/test_runner/regress/test_wal_receiver.py index 8e4e154be1..7ac6e6332c 100644 --- a/test_runner/regress/test_wal_receiver.py +++ b/test_runner/regress/test_wal_receiver.py @@ -1,3 +1,5 @@ +import time + from fixtures.log_helper import log from fixtures.neon_fixtures import NeonEnv, NeonEnvBuilder from fixtures.types import Lsn, TenantId @@ -40,7 +42,10 @@ def test_pageserver_lsn_wait_error_start(neon_env_builder: NeonEnvBuilder): # Kills one of the safekeepers and ensures that only the active ones are printed in the state. def test_pageserver_lsn_wait_error_safekeeper_stop(neon_env_builder: NeonEnvBuilder): # Trigger WAL wait timeout faster - neon_env_builder.pageserver_config_override = "wait_lsn_timeout = '1s'" + neon_env_builder.pageserver_config_override = """ + wait_lsn_timeout = "1s" + tenant_config={walreceiver_connect_timeout = "2s", lagging_wal_timeout = "2s"} + """ # Have notable SK ids to ensure we check logs for their presence, not some other random numbers neon_env_builder.safekeepers_id_start = 12345 neon_env_builder.num_safekeepers = 3 @@ -70,6 +75,8 @@ def test_pageserver_lsn_wait_error_safekeeper_stop(neon_env_builder: NeonEnvBuil stopped_safekeeper_id = stopped_safekeeper.id log.info(f"Stopping safekeeper {stopped_safekeeper.id}") stopped_safekeeper.stop() + # sleep until stopped safekeeper is removed from candidates + time.sleep(2) # Spend some more time inserting, to ensure SKs report updated statuses and walreceiver in PS have time to update its connection stats. insert_test_elements(env, tenant_id, start=elements_to_insert + 1, count=elements_to_insert) @@ -77,7 +84,8 @@ def test_pageserver_lsn_wait_error_safekeeper_stop(neon_env_builder: NeonEnvBuil try: trigger_wait_lsn_timeout(env, tenant_id) except Exception as e: - exception_string = str(e) + # Strip out the part before stdout, as it contains full command with the list of all safekeepers + exception_string = str(e).split("stdout", 1)[-1] assert expected_timeout_error in exception_string, "Should time out during waiting for WAL" for safekeeper in env.safekeepers: diff --git a/test_runner/regress/test_walredo_not_left_behind_on_detach.py b/test_runner/regress/test_walredo_not_left_behind_on_detach.py index 7d944bebb3..4a47898935 100644 --- a/test_runner/regress/test_walredo_not_left_behind_on_detach.py +++ b/test_runner/regress/test_walredo_not_left_behind_on_detach.py @@ -83,6 +83,9 @@ def test_walredo_not_left_behind_on_detach(neon_env_builder: NeonEnvBuilder): # XXX this is quite brittle as the lifecycle of the WAL redo process is an implementation detail assert_child_processes(pagserver_pid, wal_redo_present=True, defunct_present=False) + # Stop the compute before detaching, to avoid errors in the log. + endpoint.stop() + last_error = None for i in range(3): try: diff --git a/vendor/postgres-v14 b/vendor/postgres-v14 index a2daebc6b4..1144aee166 160000 --- a/vendor/postgres-v14 +++ b/vendor/postgres-v14 @@ -1 +1 @@ -Subproject commit a2daebc6b445dcbcca9c18e1711f47c1db7ffb04 +Subproject commit 1144aee1661c79eec65e784a8dad8bd450d9df79 diff --git a/vendor/postgres-v15 b/vendor/postgres-v15 index 2df2ce3744..1984832c74 160000 --- a/vendor/postgres-v15 +++ b/vendor/postgres-v15 @@ -1 +1 @@ -Subproject commit 2df2ce374464a7449e15dfa46c956b73b4f4098b +Subproject commit 1984832c740a7fa0e468bb720f40c525b652835d