tests: make reconcile_until_idle more aggressive by default

2026-05-17 13:10:38 +00:00 · 2025-02-04 14:47:10 +01:00
214 changed files with 4330 additions and 8729 deletions
--- a/.dockerignore
+++ b/.dockerignore
@@ -24,4 +24,3 @@
 !storage_controller/
 !vendor/postgres-*/
 !workspace_hack/
-!build_tools/patches
--- a/.github/actions/run-python-test-set/action.yml
+++ b/.github/actions/run-python-test-set/action.yml
@@ -121,8 +121,6 @@ runs:
        export DEFAULT_PG_VERSION=${PG_VERSION#v}
        export LD_LIBRARY_PATH=${POSTGRES_DISTRIB_DIR}/v${DEFAULT_PG_VERSION}/lib
        export BENCHMARK_CONNSTR=${BENCHMARK_CONNSTR:-}
-        export ASAN_OPTIONS=detect_leaks=0:detect_stack_use_after_return=0:abort_on_error=1:strict_string_checks=1:check_initialization_order=1:strict_init_order=1
-        export UBSAN_OPTIONS=abort_on_error=1:print_stacktrace=1

        if [ "${BUILD_TYPE}" = "remote" ]; then
          export REMOTE_ENV=1
--- a/.github/workflows/_build-and-test-locally.yml
+++ b/.github/workflows/_build-and-test-locally.yml
@@ -23,11 +23,6 @@ on:
        description: 'a json object of postgres versions and lfc states to run regression tests on'
        required: true
        type: string
-      sanitizers:
-        description: 'enabled or disabled'
-        required: false
-        default: 'disabled'
-        type: string

 defaults:
  run:
@@ -92,7 +87,6 @@ jobs:
      - name: Set env variables
        env:
          ARCH: ${{ inputs.arch }}
-          SANITIZERS: ${{ inputs.sanitizers }}
        run: |
          CARGO_FEATURES="--features testing"
          if [[ $BUILD_TYPE == "debug" && $ARCH == 'x64' ]]; then
@@ -105,14 +99,8 @@ jobs:
            cov_prefix=""
            CARGO_FLAGS="--locked --release"
          fi
-          if [[ $SANITIZERS == 'enabled' ]]; then
-            make_vars="WITH_SANITIZERS=yes"
-          else
-            make_vars=""
-          fi
          {
            echo "cov_prefix=${cov_prefix}"
-            echo "make_vars=${make_vars}"
            echo "CARGO_FEATURES=${CARGO_FEATURES}"
            echo "CARGO_FLAGS=${CARGO_FLAGS}"
            echo "CARGO_HOME=${GITHUB_WORKSPACE}/.cargo"
@@ -148,39 +136,35 @@ jobs:

      - name: Build postgres v14
        if: steps.cache_pg_14.outputs.cache-hit != 'true'
-        run: mold -run make ${make_vars} postgres-v14 -j$(nproc)
+        run: mold -run make postgres-v14 -j$(nproc)

      - name: Build postgres v15
        if: steps.cache_pg_15.outputs.cache-hit != 'true'
-        run: mold -run make ${make_vars} postgres-v15 -j$(nproc)
+        run: mold -run make postgres-v15 -j$(nproc)

      - name: Build postgres v16
        if: steps.cache_pg_16.outputs.cache-hit != 'true'
-        run: mold -run make ${make_vars} postgres-v16 -j$(nproc)
+        run: mold -run make postgres-v16 -j$(nproc)

      - name: Build postgres v17
        if: steps.cache_pg_17.outputs.cache-hit != 'true'
-        run: mold -run make ${make_vars} postgres-v17 -j$(nproc)
+        run: mold -run make postgres-v17 -j$(nproc)

      - name: Build neon extensions
-        run: mold -run make ${make_vars} neon-pg-ext -j$(nproc)
+        run: mold -run make neon-pg-ext -j$(nproc)

      - name: Build walproposer-lib
-        run: mold -run make ${make_vars} walproposer-lib -j$(nproc)
+        run: mold -run make walproposer-lib -j$(nproc)

      - name: Run cargo build
-        env:
-          WITH_TESTS: ${{ inputs.sanitizers != 'enabled' && '--tests' || '' }}
        run: |
-          export ASAN_OPTIONS=detect_leaks=0
-          ${cov_prefix} mold -run cargo build $CARGO_FLAGS $CARGO_FEATURES --bins ${WITH_TESTS}
+          ${cov_prefix} mold -run cargo build $CARGO_FLAGS $CARGO_FEATURES --bins --tests

      # Do install *before* running rust tests because they might recompile the
      # binaries with different features/flags.
      - name: Install rust binaries
        env:
          ARCH: ${{ inputs.arch }}
-          SANITIZERS: ${{ inputs.sanitizers }}
        run: |
          # Install target binaries
          mkdir -p /tmp/neon/bin/
@@ -195,7 +179,7 @@ jobs:
          done

          # Install test executables and write list of all binaries (for code coverage)
-          if [[ $BUILD_TYPE == "debug" && $ARCH == 'x64' && $SANITIZERS != 'enabled' ]]; then
+          if [[ $BUILD_TYPE == "debug" && $ARCH == 'x64' ]]; then
            # Keep bloated coverage data files away from the rest of the artifact
            mkdir -p /tmp/coverage/

@@ -228,7 +212,6 @@ jobs:
          role-duration-seconds: 18000 # 5 hours

      - name: Run rust tests
-        if: ${{ inputs.sanitizers != 'enabled' }}
        env:
          NEXTEST_RETRIES: 3
        run: |
@@ -290,7 +273,6 @@ jobs:
          DATABASE_URL: postgresql://localhost:1235/storage_controller
          POSTGRES_DISTRIB_DIR: /tmp/neon/pg_install
        run: |
-          export ASAN_OPTIONS=detect_leaks=0
          /tmp/neon/bin/neon_local init
          /tmp/neon/bin/neon_local storage_controller start

@@ -337,7 +319,7 @@ jobs:
      - name: Pytest regression tests
        continue-on-error: ${{ matrix.lfc_state == 'with-lfc' && inputs.build-type == 'debug' }}
        uses: ./.github/actions/run-python-test-set
-        timeout-minutes: ${{ inputs.sanitizers != 'enabled' && 60 || 180 }}
+        timeout-minutes: 60
        with:
          build_type: ${{ inputs.build-type }}
          test_selection: regress
@@ -355,7 +337,6 @@ jobs:
          PAGESERVER_VIRTUAL_FILE_IO_ENGINE: tokio-epoll-uring
          PAGESERVER_GET_VECTORED_CONCURRENT_IO: sidecar-task
          USE_LFC: ${{ matrix.lfc_state == 'with-lfc' && 'true' || 'false' }}
-          SANITIZERS: ${{ inputs.sanitizers }}

      # Temporary disable this step until we figure out why it's so flaky
      # Ref https://github.com/neondatabase/neon/issues/4540
--- a/.github/workflows/_push-to-acr.yml
+++ b/.github/workflows/_push-to-acr.yml
@@ -0,0 +1,56 @@
+name: Push images to ACR
+on:
+  workflow_call:
+    inputs:
+      client_id:
+        description: Client ID of Azure managed identity or Entra app
+        required: true
+        type: string
+      image_tag:
+        description: Tag for the container image
+        required: true
+        type: string
+      images:
+        description: Images to push
+        required: true
+        type: string
+      registry_name:
+        description: Name of the container registry
+        required: true
+        type: string
+      subscription_id:
+        description: Azure subscription ID
+        required: true
+        type: string
+      tenant_id:
+        description: Azure tenant ID
+        required: true
+        type: string
+
+jobs:
+  push-to-acr:
+    runs-on: ubuntu-22.04
+    permissions:
+      contents: read  # This is required for actions/checkout
+      id-token: write # This is required for Azure Login to work.
+
+    steps:
+      - name: Azure login
+        uses: azure/login@6c251865b4e6290e7b78be643ea2d005bc51f69a  # @v2.1.1
+        with:
+          client-id: ${{ inputs.client_id }}
+          subscription-id: ${{ inputs.subscription_id }}
+          tenant-id: ${{ inputs.tenant_id }}
+
+      - name: Login to ACR
+        run: |
+          az acr login --name=${{ inputs.registry_name }}
+
+      - name: Copy docker images to ACR ${{ inputs.registry_name }}
+        run: |
+          images='${{ inputs.images }}'
+          for image in ${images}; do
+            docker buildx imagetools create \
+              -t ${{ inputs.registry_name }}.azurecr.io/neondatabase/${image}:${{ inputs.image_tag }} \
+                                                        neondatabase/${image}:${{ inputs.image_tag }}
+          done
--- a/.github/workflows/_push-to-container-registry.yml
+++ b/.github/workflows/_push-to-container-registry.yml
@@ -1,101 +0,0 @@
-name: Push images to Container Registry
-on:
-  workflow_call:
-    inputs:
-      # Example: {"docker.io/neondatabase/neon:13196061314":["369495373322.dkr.ecr.eu-central-1.amazonaws.com/neon:13196061314","neoneastus2.azurecr.io/neondatabase/neon:13196061314"]}
-      image-map:
-        description: JSON map of images, mapping from a source image to an array of target images that should be pushed.
-        required: true
-        type: string
-      aws-region:
-        description: AWS region to log in to. Required when pushing to ECR.
-        required: false
-        type: string
-      aws-account-ids:
-        description: Comma separated AWS account IDs to log in to for pushing to ECR. Required when pushing to ECR.
-        required: false
-        type: string
-      azure-client-id:
-        description: Client ID of Azure managed identity or Entra app. Required when pushing to ACR.
-        required: false
-        type: string
-      azure-subscription-id:
-        description: Azure subscription ID. Required when pushing to ACR.
-        required: false
-        type: string
-      azure-tenant-id:
-        description: Azure tenant ID. Required when pushing to ACR.
-        required: false
-        type: string
-      acr-registry-name:
-        description: ACR registry name. Required when pushing to ACR.
-        required: false
-        type: string
-    secrets:
-      docker-hub-username:
-        description: Docker Hub username. Required when pushing to Docker Hub.
-        required: false
-      docker-hub-password:
-        description: Docker Hub password. Required when pushing to Docker Hub.
-        required: false
-      aws-role-to-assume:
-        description: AWS role to assume. Required when pushing to ECR.
-        required: false
-
-permissions: {}
-
-defaults:
-  run:
-    shell: bash -euo pipefail {0}
-
-jobs:
-  push-to-container-registry:
-    runs-on: ubuntu-22.04
-    permissions:
-      id-token: write  # Required for aws/azure login
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          sparse-checkout: scripts/push_with_image_map.py
-          sparse-checkout-cone-mode: false
-
-      - name: Print image-map
-        run: echo '${{ inputs.image-map }}' | jq
-
-      - name: Configure AWS credentials
-        if: contains(inputs.image-map, 'amazonaws.com/')
-        uses: aws-actions/configure-aws-credentials@v4
-        with:
-          aws-region: "${{ inputs.aws-region }}"
-          role-to-assume: "${{ secrets.aws-role-to-assume }}"
-          role-duration-seconds: 3600
-
-      - name: Login to ECR
-        if: contains(inputs.image-map, 'amazonaws.com/')
-        uses: aws-actions/amazon-ecr-login@v2
-        with:
-          registries: "${{ inputs.aws-account-ids }}"
-
-      - name: Configure Azure credentials
-        if: contains(inputs.image-map, 'azurecr.io/')
-        uses: azure/login@6c251865b4e6290e7b78be643ea2d005bc51f69a  # @v2.1.1
-        with:
-          client-id: ${{ inputs.azure-client-id }}
-          subscription-id: ${{ inputs.azure-subscription-id }}
-          tenant-id: ${{ inputs.azure-tenant-id }}
-
-      - name: Login to ACR
-        if: contains(inputs.image-map, 'azurecr.io/')
-        run: |
-          az acr login --name=${{ inputs.acr-registry-name }}
-
-      - name: Log in to Docker Hub
-        uses: docker/login-action@v3
-        with:
-          username: ${{ secrets.docker-hub-username }}
-          password: ${{ secrets.docker-hub-password }}
-
-      - name: Copy docker images to target registries
-        run: python scripts/push_with_image_map.py
-        env:
-          IMAGE_MAP: ${{ inputs.image-map }}
--- a/.github/workflows/approved-for-ci-run.yml
+++ b/.github/workflows/approved-for-ci-run.yml
@@ -67,9 +67,9 @@ jobs:

      - uses: actions/checkout@v4
        with:
-          ref: ${{ github.event.pull_request.head.sha }}
+          ref: main
          token: ${{ secrets.CI_ACCESS_TOKEN }}
-
+      
      - name: Look for existing PR
        id: get-pr
        env:
@@ -77,7 +77,7 @@ jobs:
        run: |
          ALREADY_CREATED="$(gh pr --repo ${GITHUB_REPOSITORY} list --head ${BRANCH} --base main --json number --jq '.[].number')"
          echo "ALREADY_CREATED=${ALREADY_CREATED}" >> ${GITHUB_OUTPUT}
-
+      
      - name: Get changed labels
        id: get-labels
        if: steps.get-pr.outputs.ALREADY_CREATED != ''
@@ -94,6 +94,10 @@ jobs:
          echo "LABELS_TO_ADD=${LABELS_TO_ADD}" >> ${GITHUB_OUTPUT}
          echo "LABELS_TO_REMOVE=${LABELS_TO_REMOVE}" >> ${GITHUB_OUTPUT}

+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event.pull_request.head.sha }}
+
      - run: git checkout -b "${BRANCH}"

      - run: git push --force origin "${BRANCH}"
@@ -101,7 +105,7 @@ jobs:

      - name: Create a Pull Request for CI run (if required)
        if: steps.get-pr.outputs.ALREADY_CREATED == ''
-        env:
+        env: 
          GH_TOKEN: ${{ secrets.CI_ACCESS_TOKEN }}
        run: |
          cat << EOF > body.md
@@ -138,7 +142,7 @@ jobs:

      - run: git push --force origin "${BRANCH}"
        if: steps.get-pr.outputs.ALREADY_CREATED != ''
-
+             
  cleanup:
    # Close PRs and delete branchs if the original PR is closed.

--- a/.github/workflows/build_and_test.yml
+++ b/.github/workflows/build_and_test.yml
@@ -263,9 +263,8 @@ jobs:
          echo "json=$(jq --compact-output '.' /tmp/benchmark_durations.json)" >> $GITHUB_OUTPUT

  benchmarks:
-    # `!failure() && !cancelled()` is required because the workflow depends on the job that can be skipped: `deploy` in PRs
-    if: github.ref_name == 'main' || (contains(github.event.pull_request.labels.*.name, 'run-benchmarks') && !failure() && !cancelled())
-    needs: [ check-permissions, build-build-tools-image, get-benchmarks-durations, deploy ]
+    if: github.ref_name == 'main' || contains(github.event.pull_request.labels.*.name, 'run-benchmarks')
+    needs: [ check-permissions, build-and-test-locally, build-build-tools-image, get-benchmarks-durations ]
    permissions:
      id-token: write # aws-actions/configure-aws-credentials
      statuses: write
@@ -498,7 +497,7 @@ jobs:

  trigger-e2e-tests:
    if: ${{ !github.event.pull_request.draft || contains( github.event.pull_request.labels.*.name, 'run-e2e-tests-in-draft') || github.ref_name == 'main' || github.ref_name == 'release' || github.ref_name == 'release-proxy' || github.ref_name == 'release-compute' }}
-    needs: [ check-permissions, push-neon-image-dev, push-compute-image-dev, tag ]
+    needs: [ check-permissions, promote-images-dev, tag ]
    uses: ./.github/workflows/trigger-e2e-tests.yml
    secrets: inherit

@@ -572,6 +571,21 @@ jobs:
                                             neondatabase/neon:${{ needs.tag.outputs.build-tag }}-bookworm-x64 \
                                             neondatabase/neon:${{ needs.tag.outputs.build-tag }}-bookworm-arm64

+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          aws-region: eu-central-1
+          role-to-assume: ${{ vars.DEV_AWS_OIDC_ROLE_ARN }}
+          role-duration-seconds: 3600
+
+      - name: Login to Amazon Dev ECR
+        uses: aws-actions/amazon-ecr-login@v2
+
+      - name: Push multi-arch image to ECR
+        run: |
+          docker buildx imagetools create -t 369495373322.dkr.ecr.eu-central-1.amazonaws.com/neon:${{ needs.tag.outputs.build-tag }} \
+                                                                                neondatabase/neon:${{ needs.tag.outputs.build-tag }}
+
  compute-node-image-arch:
    needs: [ check-permissions, build-build-tools-image, tag ]
    permissions:
@@ -618,6 +632,16 @@ jobs:
          username: ${{ secrets.NEON_DOCKERHUB_USERNAME }}
          password: ${{ secrets.NEON_DOCKERHUB_PASSWORD }}

+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          aws-region: eu-central-1
+          role-to-assume: ${{ vars.DEV_AWS_OIDC_ROLE_ARN }}
+          role-duration-seconds: 3600
+
+      - name: Login to Amazon Dev ECR
+        uses: aws-actions/amazon-ecr-login@v2
+
      - uses: docker/login-action@v3
        with:
          registry: cache.neon.build
@@ -705,6 +729,21 @@ jobs:
                                             neondatabase/neon-test-extensions-${{ matrix.version.pg }}:${{ needs.tag.outputs.build-tag }}-${{ matrix.version.debian }}-x64 \
                                             neondatabase/neon-test-extensions-${{ matrix.version.pg }}:${{ needs.tag.outputs.build-tag }}-${{ matrix.version.debian }}-arm64

+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          aws-region: eu-central-1
+          role-to-assume: ${{ vars.DEV_AWS_OIDC_ROLE_ARN }}
+          role-duration-seconds: 3600
+
+      - name: Login to Amazon Dev ECR
+        uses: aws-actions/amazon-ecr-login@v2
+
+      - name: Push multi-arch compute-node-${{ matrix.version.pg }} image to ECR
+        run: |
+          docker buildx imagetools create -t 369495373322.dkr.ecr.eu-central-1.amazonaws.com/compute-node-${{ matrix.version.pg }}:${{ needs.tag.outputs.build-tag }} \
+                                                                                neondatabase/compute-node-${{ matrix.version.pg }}:${{ needs.tag.outputs.build-tag }}
+
  vm-compute-node-image:
    needs: [ check-permissions, tag, compute-node-image ]
    runs-on: [ self-hosted, large ]
@@ -837,109 +876,133 @@ jobs:
          docker compose --profile test-extensions -f ./docker-compose/docker-compose.yml logs || true
          docker compose --profile test-extensions -f ./docker-compose/docker-compose.yml down

-  generate-image-maps:
-    needs: [ tag ]
+  promote-images-dev:
+    needs: [ check-permissions, tag, vm-compute-node-image, neon-image ]
    runs-on: ubuntu-22.04
-    outputs:
-      neon-dev: ${{ steps.generate.outputs.neon-dev }}
-      neon-prod: ${{ steps.generate.outputs.neon-prod }}
-      compute-dev: ${{ steps.generate.outputs.compute-dev }}
-      compute-prod: ${{ steps.generate.outputs.compute-prod }}
+
+    permissions:
+      id-token: write # aws-actions/configure-aws-credentials
+      statuses: write
+      contents: read
+
+    env:
+      VERSIONS: v14 v15 v16 v17
+
    steps:
-      - uses: actions/checkout@v4
+      - uses: docker/login-action@v3
        with:
-          sparse-checkout: scripts/generate_image_maps.py
-          sparse-checkout-cone-mode: false
+          username: ${{ secrets.NEON_DOCKERHUB_USERNAME }}
+          password: ${{ secrets.NEON_DOCKERHUB_PASSWORD }}

-      - name: Generate Image Maps
-        id: generate
-        run: python scripts/generate_image_maps.py
-        env:
-          BUILD_TAG: "${{ needs.tag.outputs.build-tag }}"
-          BRANCH: "${{ github.ref_name }}"
-          DEV_ACR: "${{ vars.AZURE_DEV_REGISTRY_NAME }}"
-          PROD_ACR: "${{ vars.AZURE_PROD_REGISTRY_NAME }}"
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          aws-region: eu-central-1
+          role-to-assume: ${{ vars.DEV_AWS_OIDC_ROLE_ARN }}
+          role-duration-seconds: 3600

-  push-neon-image-dev:
-    needs: [ generate-image-maps, neon-image ]
-    uses: ./.github/workflows/_push-to-container-registry.yml
-    with:
-      image-map: '${{ needs.generate-image-maps.outputs.neon-dev }}'
-      aws-region: eu-central-1
-      aws-account-ids: "369495373322"
-      azure-client-id: ${{ vars.AZURE_DEV_CLIENT_ID }}
-      azure-subscription-id: ${{ vars.AZURE_DEV_SUBSCRIPTION_ID }}
-      azure-tenant-id: ${{ vars.AZURE_TENANT_ID }}
-      acr-registry-name: ${{ vars.AZURE_DEV_REGISTRY_NAME }}
-    secrets:
-      aws-role-to-assume: "${{ vars.DEV_AWS_OIDC_ROLE_ARN }}"
-      docker-hub-username: ${{ secrets.NEON_DOCKERHUB_USERNAME }}
-      docker-hub-password: ${{ secrets.NEON_DOCKERHUB_PASSWORD }}
+      - name: Login to Amazon Dev ECR
+        uses: aws-actions/amazon-ecr-login@v2

-  push-compute-image-dev:
-    needs: [ generate-image-maps, vm-compute-node-image ]
-    uses: ./.github/workflows/_push-to-container-registry.yml
-    with:
-      image-map: '${{ needs.generate-image-maps.outputs.compute-dev }}'
-      aws-region: eu-central-1
-      aws-account-ids: "369495373322"
-      azure-client-id: ${{ vars.AZURE_DEV_CLIENT_ID }}
-      azure-subscription-id: ${{ vars.AZURE_DEV_SUBSCRIPTION_ID }}
-      azure-tenant-id: ${{ vars.AZURE_TENANT_ID }}
-      acr-registry-name: ${{ vars.AZURE_DEV_REGISTRY_NAME }}
-    secrets:
-      aws-role-to-assume: "${{ vars.DEV_AWS_OIDC_ROLE_ARN }}"
-      docker-hub-username: ${{ secrets.NEON_DOCKERHUB_USERNAME }}
-      docker-hub-password: ${{ secrets.NEON_DOCKERHUB_PASSWORD }}
+      - name: Copy vm-compute-node images to ECR
+        run: |
+          for version in ${VERSIONS}; do
+            docker buildx imagetools create -t 369495373322.dkr.ecr.eu-central-1.amazonaws.com/vm-compute-node-${version}:${{ needs.tag.outputs.build-tag }} \
+                                               neondatabase/vm-compute-node-${version}:${{ needs.tag.outputs.build-tag }}
+          done

-  push-neon-image-prod:
-    if: github.ref_name == 'release' || github.ref_name == 'release-proxy' || github.ref_name == 'release-compute'
-    needs: [ generate-image-maps, neon-image, test-images ]
-    uses: ./.github/workflows/_push-to-container-registry.yml
-    with:
-      image-map: '${{ needs.generate-image-maps.outputs.neon-prod }}'
-      aws-region: eu-central-1
-      aws-account-ids: "093970136003"
-      azure-client-id: ${{ vars.AZURE_PROD_CLIENT_ID }}
-      azure-subscription-id: ${{ vars.AZURE_PROD_SUBSCRIPTION_ID }}
-      azure-tenant-id: ${{ vars.AZURE_TENANT_ID }}
-      acr-registry-name: ${{ vars.AZURE_PROD_REGISTRY_NAME }}
-    secrets:
-      aws-role-to-assume: "${{ secrets.PROD_GHA_OIDC_ROLE }}"
-      docker-hub-username: ${{ secrets.NEON_DOCKERHUB_USERNAME }}
-      docker-hub-password: ${{ secrets.NEON_DOCKERHUB_PASSWORD }}
+  promote-images-prod:
+    needs: [ check-permissions, tag, test-images, promote-images-dev ]
+    runs-on: ubuntu-22.04
+    if: github.ref_name == 'main' || github.ref_name == 'release' || github.ref_name == 'release-proxy' || github.ref_name == 'release-compute'

-  push-compute-image-prod:
-    if: github.ref_name == 'release' || github.ref_name == 'release-proxy' || github.ref_name == 'release-compute'
-    needs: [ generate-image-maps, vm-compute-node-image, test-images ]
-    uses: ./.github/workflows/_push-to-container-registry.yml
-    with:
-      image-map: '${{ needs.generate-image-maps.outputs.compute-prod }}'
-      aws-region: eu-central-1
-      aws-account-ids: "093970136003"
-      azure-client-id: ${{ vars.AZURE_PROD_CLIENT_ID }}
-      azure-subscription-id: ${{ vars.AZURE_PROD_SUBSCRIPTION_ID }}
-      azure-tenant-id: ${{ vars.AZURE_TENANT_ID }}
-      acr-registry-name: ${{ vars.AZURE_PROD_REGISTRY_NAME }}
-    secrets:
-      aws-role-to-assume: "${{ secrets.PROD_GHA_OIDC_ROLE }}"
-      docker-hub-username: ${{ secrets.NEON_DOCKERHUB_USERNAME }}
-      docker-hub-password: ${{ secrets.NEON_DOCKERHUB_PASSWORD }}
+    permissions:
+      id-token: write # aws-actions/configure-aws-credentials
+      statuses: write
+      contents: read

-  # This is a bit of a special case so we're not using a generated image map.
-  add-latest-tag-to-neon-extensions-test-image:
+    env:
+      VERSIONS: v14 v15 v16 v17
+
+    steps:
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          aws-region: eu-central-1
+          role-to-assume: ${{ vars.DEV_AWS_OIDC_ROLE_ARN }}
+          role-duration-seconds: 3600
+
+      - name: Login to Amazon Dev ECR
+        uses: aws-actions/amazon-ecr-login@v2
+
+      - uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.NEON_DOCKERHUB_USERNAME }}
+          password: ${{ secrets.NEON_DOCKERHUB_PASSWORD }}
+
+      - name: Add latest tag to images
+        if: github.ref_name == 'main'
+        run: |
+          for repo in neondatabase 369495373322.dkr.ecr.eu-central-1.amazonaws.com; do
+            docker buildx imagetools create -t $repo/neon:latest \
+                                               $repo/neon:${{ needs.tag.outputs.build-tag }}
+
+            for version in ${VERSIONS}; do
+              docker buildx imagetools create -t $repo/compute-node-${version}:latest \
+                                                 $repo/compute-node-${version}:${{ needs.tag.outputs.build-tag }}
+
+              docker buildx imagetools create -t $repo/vm-compute-node-${version}:latest \
+                                                 $repo/vm-compute-node-${version}:${{ needs.tag.outputs.build-tag }}
+            done
+          done
+          docker buildx imagetools create -t neondatabase/neon-test-extensions-v16:latest \
+                                              neondatabase/neon-test-extensions-v16:${{ needs.tag.outputs.build-tag }}
+
+      - name: Configure AWS-prod credentials
+        if: github.ref_name == 'release'|| github.ref_name == 'release-proxy' || github.ref_name == 'release-compute'
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          aws-region: eu-central-1
+          mask-aws-account-id: true
+          role-to-assume: ${{ secrets.PROD_GHA_OIDC_ROLE }}
+
+      - name: Login to prod ECR
+        uses: docker/login-action@v3
+        if: github.ref_name == 'release'|| github.ref_name == 'release-proxy' || github.ref_name == 'release-compute'
+        with:
+          registry: 093970136003.dkr.ecr.eu-central-1.amazonaws.com
+
+      - name: Copy all images to prod ECR
+        if: github.ref_name == 'release' || github.ref_name == 'release-proxy' || github.ref_name == 'release-compute'
+        run: |
+          for image in neon {vm-,}compute-node-{v14,v15,v16,v17}; do
+            docker buildx imagetools create -t 093970136003.dkr.ecr.eu-central-1.amazonaws.com/${image}:${{ needs.tag.outputs.build-tag }} \
+                                               369495373322.dkr.ecr.eu-central-1.amazonaws.com/${image}:${{ needs.tag.outputs.build-tag }}
+          done
+
+  push-to-acr-dev:
    if: github.ref_name == 'main'
-    needs: [ tag, compute-node-image ]
-    uses: ./.github/workflows/_push-to-container-registry.yml
+    needs: [ tag, promote-images-dev ]
+    uses: ./.github/workflows/_push-to-acr.yml
    with:
-      image-map: |
-        {
-          "docker.io/neondatabase/neon-test-extensions-v16:${{ needs.tag.outputs.build-tag }}": ["docker.io/neondatabase/neon-test-extensions-v16:latest"],
-          "docker.io/neondatabase/neon-test-extensions-v17:${{ needs.tag.outputs.build-tag }}": ["docker.io/neondatabase/neon-test-extensions-v17:latest"]
-        }
-    secrets:
-      docker-hub-username: ${{ secrets.NEON_DOCKERHUB_USERNAME }}
-      docker-hub-password: ${{ secrets.NEON_DOCKERHUB_PASSWORD }}
+      client_id: ${{ vars.AZURE_DEV_CLIENT_ID }}
+      image_tag: ${{ needs.tag.outputs.build-tag }}
+      images: neon vm-compute-node-v14 vm-compute-node-v15 vm-compute-node-v16 vm-compute-node-v17 compute-node-v14 compute-node-v15 compute-node-v16 compute-node-v17
+      registry_name: ${{ vars.AZURE_DEV_REGISTRY_NAME }}
+      subscription_id: ${{ vars.AZURE_DEV_SUBSCRIPTION_ID }}
+      tenant_id: ${{ vars.AZURE_TENANT_ID }}
+
+  push-to-acr-prod:
+    if: github.ref_name == 'release' || github.ref_name == 'release-proxy' || github.ref_name == 'release-compute'
+    needs: [ tag, promote-images-prod ]
+    uses: ./.github/workflows/_push-to-acr.yml
+    with:
+      client_id: ${{ vars.AZURE_PROD_CLIENT_ID }}
+      image_tag: ${{ needs.tag.outputs.build-tag }}
+      images: neon vm-compute-node-v14 vm-compute-node-v15 vm-compute-node-v16 vm-compute-node-v17 compute-node-v14 compute-node-v15 compute-node-v16 compute-node-v17
+      registry_name: ${{ vars.AZURE_PROD_REGISTRY_NAME }}
+      subscription_id: ${{ vars.AZURE_PROD_SUBSCRIPTION_ID }}
+      tenant_id: ${{ vars.AZURE_TENANT_ID }}

  trigger-custom-extensions-build-and-wait:
    needs: [ check-permissions, tag ]
@@ -1021,7 +1084,7 @@ jobs:
          exit 1

  deploy:
-    needs: [ check-permissions, push-neon-image-prod, push-compute-image-prod, tag, build-and-test-locally, trigger-custom-extensions-build-and-wait ]
+    needs: [ check-permissions, promote-images-prod, tag, build-and-test-locally, trigger-custom-extensions-build-and-wait, push-to-acr-dev, push-to-acr-prod ]
    # `!failure() && !cancelled()` is required because the workflow depends on the job that can be skipped: `push-to-acr-dev` and `push-to-acr-prod`
    if: (github.ref_name == 'main' || github.ref_name == 'release' || github.ref_name == 'release-proxy' || github.ref_name == 'release-compute') && !failure() && !cancelled()
    permissions:
@@ -1274,7 +1337,7 @@ jobs:
          done

  pin-build-tools-image:
-    needs: [ build-build-tools-image, push-compute-image-prod, push-neon-image-prod, build-and-test-locally ]
+    needs: [ build-build-tools-image, promote-images-prod, build-and-test-locally ]
    if: github.ref_name == 'main'
    uses: ./.github/workflows/pin-build-tools-image.yml
    with:
@@ -1299,8 +1362,7 @@ jobs:
      - check-codestyle-rust
      - check-dependencies-rust
      - files-changed
-      - push-compute-image-dev
-      - push-neon-image-dev
+      - promote-images-dev
      - test-images
      - trigger-custom-extensions-build-and-wait
    runs-on: ubuntu-22.04
@@ -1317,7 +1379,6 @@ jobs:
          || needs.check-codestyle-python.result == 'skipped'
          || needs.check-codestyle-rust.result == 'skipped'
          || needs.files-changed.result == 'skipped'
-          || needs.push-compute-image-dev.result == 'skipped'
-          || needs.push-neon-image-dev.result == 'skipped'
+          || needs.promote-images-dev.result == 'skipped'
          || needs.test-images.result == 'skipped'
          || needs.trigger-custom-extensions-build-and-wait.result == 'skipped'
--- a/.github/workflows/build_and_test_with_sanitizers.yml
+++ b/.github/workflows/build_and_test_with_sanitizers.yml
@@ -1,134 +0,0 @@
-name: Build and Test with Sanitizers
-
-on:
-  schedule:
-    # * is a special character in YAML so you have to quote this string
-    #          ┌───────────── minute (0 - 59)
-    #          │ ┌───────────── hour (0 - 23)
-    #          │ │ ┌───────────── day of the month (1 - 31)
-    #          │ │ │ ┌───────────── month (1 - 12 or JAN-DEC)
-    #          │ │ │ │ ┌───────────── day of the week (0 - 6 or SUN-SAT)
-    - cron:   '0 1 * * *' # run once a day, timezone is utc
-  workflow_dispatch:
-
-defaults:
-  run:
-    shell: bash -euxo pipefail {0}
-
-concurrency:
-  # Allow only one workflow per any non-`main` branch.
-  group: ${{ github.workflow }}-${{ github.ref_name }}-${{ github.ref_name == 'main' && github.sha || 'anysha' }}
-  cancel-in-progress: true
-
-env:
-  RUST_BACKTRACE: 1
-  COPT: '-Werror'
-
-jobs:
-  tag:
-    runs-on: [ self-hosted, small ]
-    container: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/base:pinned
-    outputs:
-      build-tag: ${{steps.build-tag.outputs.tag}}
-
-    steps:
-      # Need `fetch-depth: 0` to count the number of commits in the branch
-      - uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-
-      - name: Get build tag
-        run: |
-          echo run:$GITHUB_RUN_ID
-          echo ref:$GITHUB_REF_NAME
-          echo rev:$(git rev-list --count HEAD)
-          if [[ "$GITHUB_REF_NAME" == "main" ]]; then
-            echo "tag=$(git rev-list --count HEAD)" >> $GITHUB_OUTPUT
-          elif [[ "$GITHUB_REF_NAME" == "release" ]]; then
-            echo "tag=release-$(git rev-list --count HEAD)" >> $GITHUB_OUTPUT
-          elif [[ "$GITHUB_REF_NAME" == "release-proxy" ]]; then
-            echo "tag=release-proxy-$(git rev-list --count HEAD)" >> $GITHUB_OUTPUT
-          elif [[ "$GITHUB_REF_NAME" == "release-compute" ]]; then
-            echo "tag=release-compute-$(git rev-list --count HEAD)" >> $GITHUB_OUTPUT
-          else
-            echo "GITHUB_REF_NAME (value '$GITHUB_REF_NAME') is not set to either 'main' or 'release', 'release-proxy', 'release-compute'"
-            echo "tag=$GITHUB_RUN_ID" >> $GITHUB_OUTPUT
-          fi
-        shell: bash
-        id: build-tag
-
-  build-build-tools-image:
-    uses: ./.github/workflows/build-build-tools-image.yml
-    secrets: inherit
-
-  build-and-test-locally:
-    needs: [ tag, build-build-tools-image ]
-    strategy:
-      fail-fast: false
-      matrix:
-        arch: [ x64, arm64 ]
-        build-type: [ release ]
-    uses: ./.github/workflows/_build-and-test-locally.yml
-    with:
-      arch: ${{ matrix.arch }}
-      build-tools-image: ${{ needs.build-build-tools-image.outputs.image }}-bookworm
-      build-tag: ${{ needs.tag.outputs.build-tag }}
-      build-type: ${{ matrix.build-type }}
-      test-cfg: '[{"pg_version":"v17"}]'
-      sanitizers: enabled
-    secrets: inherit
-
-
-  create-test-report:
-    needs: [ build-and-test-locally, build-build-tools-image ]
-    if: ${{ !cancelled() }}
-    permissions:
-      id-token: write # aws-actions/configure-aws-credentials
-      statuses: write
-      contents: write
-      pull-requests: write
-    outputs:
-      report-url: ${{ steps.create-allure-report.outputs.report-url }}
-
-    runs-on: [ self-hosted, small ]
-    container:
-      image: ${{ needs.build-build-tools-image.outputs.image }}-bookworm
-      credentials:
-        username: ${{ secrets.NEON_DOCKERHUB_USERNAME }}
-        password: ${{ secrets.NEON_DOCKERHUB_PASSWORD }}
-      options: --init
-
-    steps:
-      - uses: actions/checkout@v4
-
-      - name: Create Allure report
-        if: ${{ !cancelled() }}
-        id: create-allure-report
-        uses: ./.github/actions/allure-report-generate
-        with:
-          store-test-results-into-db: true
-          aws-oicd-role-arn: ${{ vars.DEV_AWS_OIDC_ROLE_ARN }}
-        env:
-          REGRESS_TEST_RESULT_CONNSTR_NEW: ${{ secrets.REGRESS_TEST_RESULT_CONNSTR_NEW }}
-
-      - uses: actions/github-script@v7
-        if: ${{ !cancelled() }}
-        with:
-          # Retry script for 5XX server errors: https://github.com/actions/github-script#retries
-          retries: 5
-          script: |
-            const report = {
-              reportUrl:     "${{ steps.create-allure-report.outputs.report-url }}",
-              reportJsonUrl: "${{ steps.create-allure-report.outputs.report-json-url }}",
-            }
-
-            const coverage = {}
-
-            const script = require("./scripts/comment-test-report.js")
-            await script({
-              github,
-              context,
-              fetch,
-              report,
-              coverage,
-            })
--- a/.github/workflows/trigger-e2e-tests.yml
+++ b/.github/workflows/trigger-e2e-tests.yml
@@ -15,14 +15,7 @@ env:
  E2E_CONCURRENCY_GROUP: ${{ github.repository }}-e2e-tests-${{ github.ref_name }}-${{ github.ref_name == 'main' && github.sha || 'anysha' }}

 jobs:
-  check-permissions:
-    if: ${{ !contains(github.event.pull_request.labels.*.name, 'run-no-ci') }}
-    uses: ./.github/workflows/check-permissions.yml
-    with:
-      github-event-name: ${{ github.event_name }}
-
  cancel-previous-e2e-tests:
-    needs: [ check-permissions ]
    if: github.event_name == 'pull_request'
    runs-on: ubuntu-22.04

@@ -36,7 +29,6 @@ jobs:
              --field concurrency_group="${{ env.E2E_CONCURRENCY_GROUP }}"

  tag:
-    needs: [ check-permissions ]
    runs-on: ubuntu-22.04
    outputs:
      build-tag: ${{ steps.build-tag.outputs.tag }}
@@ -76,7 +68,7 @@ jobs:
      GH_TOKEN: ${{ secrets.CI_ACCESS_TOKEN }}
      TAG: ${{ needs.tag.outputs.build-tag }}
    steps:
-      - name: Wait for `push-{neon,compute}-image-dev` job to finish
+      - name: Wait for `promote-images-dev` job to finish
        # It's important to have a timeout here, the script in the step can run infinitely
        timeout-minutes: 60
        run: |
@@ -87,20 +79,20 @@ jobs:
          # For PRs we use the run id as the tag
          BUILD_AND_TEST_RUN_ID=${TAG}
          while true; do
-            gh run --repo ${GITHUB_REPOSITORY} view ${BUILD_AND_TEST_RUN_ID} --json jobs --jq '[.jobs[] | select((.name | startswith("push-neon-image-dev")) or (.name | startswith("push-compute-image-dev"))) | {"name": .name, "conclusion": .conclusion, "url": .url}]' > jobs.json
-            if [ $(jq '[.[] | select(.conclusion == "success")] | length' jobs.json) -eq 2 ]; then
-              break
-            fi
-            jq -c '.[]' jobs.json | while read -r job; do
-              case $(echo $job | jq .conclusion) in
-                failure | cancelled | skipped)
-                  echo "The '$(echo $job | jq .name)' job didn't succeed: '$(echo $job | jq .conclusion)'. See log in '$(echo $job | jq .url)' Exiting..."
-                  exit 1
-                  ;;
-              esac
-            done
-            echo "The 'push-{neon,compute}-image-dev' jobs haven't succeeded yet. Waiting..."
-            sleep 60
+            conclusion=$(gh run --repo ${GITHUB_REPOSITORY} view ${BUILD_AND_TEST_RUN_ID} --json jobs --jq '.jobs[] | select(.name == "promote-images-dev") | .conclusion')
+            case "$conclusion" in
+              success)
+                break
+                ;;
+              failure | cancelled | skipped)
+                echo "The 'promote-images-dev' job didn't succeed: '${conclusion}'. Exiting..."
+                exit 1
+                ;;
+              *)
+                echo "The 'promote-images-dev' hasn't succeed yet. Waiting..."
+                sleep 60
+                ;;
+            esac
          done

      - name: Set e2e-platforms
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -206,16 +206,6 @@ dependencies = [
 "syn 2.0.90",
 ]

-[[package]]
-name = "assert-json-diff"
-version = "2.0.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "47e4f2b81832e72834d7518d8487a0396a28cc408186a2e8854c0f98011faf12"
-dependencies = [
- "serde",
- "serde_json",
-]
-
 [[package]]
 name = "async-channel"
 version = "1.9.0"
@@ -300,9 +290,9 @@ checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa"

 [[package]]
 name = "aws-config"
-version = "1.5.10"
+version = "1.5.15"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9b49afaa341e8dd8577e1a2200468f98956d6eda50bcf4a53246cc00174ba924"
+checksum = "dc47e70fc35d054c8fcd296d47a61711f043ac80534a10b4f741904f81e73a90"
 dependencies = [
 "aws-credential-types",
 "aws-runtime",
@@ -311,7 +301,7 @@ dependencies = [
 "aws-sdk-sts",
 "aws-smithy-async",
 "aws-smithy-http",
- "aws-smithy-json 0.60.7",
+ "aws-smithy-json",
 "aws-smithy-runtime",
 "aws-smithy-runtime-api",
 "aws-smithy-types",
@@ -342,9 +332,9 @@ dependencies = [

 [[package]]
 name = "aws-runtime"
-version = "1.4.4"
+version = "1.5.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b5ac934720fbb46206292d2c75b57e67acfc56fe7dfd34fb9a02334af08409ea"
+checksum = "bee7643696e7fdd74c10f9eb42848a87fe469d35eae9c3323f80aa98f350baac"
 dependencies = [
 "aws-credential-types",
 "aws-sigv4",
@@ -376,7 +366,7 @@ dependencies = [
 "aws-runtime",
 "aws-smithy-async",
 "aws-smithy-http",
- "aws-smithy-json 0.61.1",
+ "aws-smithy-json",
 "aws-smithy-query",
 "aws-smithy-runtime",
 "aws-smithy-runtime-api",
@@ -399,7 +389,7 @@ dependencies = [
 "aws-runtime",
 "aws-smithy-async",
 "aws-smithy-http",
- "aws-smithy-json 0.61.1",
+ "aws-smithy-json",
 "aws-smithy-runtime",
 "aws-smithy-runtime-api",
 "aws-smithy-types",
@@ -424,7 +414,7 @@ dependencies = [
 "aws-smithy-checksums",
 "aws-smithy-eventstream",
 "aws-smithy-http",
- "aws-smithy-json 0.61.1",
+ "aws-smithy-json",
 "aws-smithy-runtime",
 "aws-smithy-runtime-api",
 "aws-smithy-types",
@@ -447,15 +437,15 @@ dependencies = [

 [[package]]
 name = "aws-sdk-sso"
-version = "1.50.0"
+version = "1.57.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "05ca43a4ef210894f93096039ef1d6fa4ad3edfabb3be92b80908b9f2e4b4eab"
+checksum = "c54bab121fe1881a74c338c5f723d1592bf3b53167f80268a1274f404e1acc38"
 dependencies = [
 "aws-credential-types",
 "aws-runtime",
 "aws-smithy-async",
 "aws-smithy-http",
- "aws-smithy-json 0.61.1",
+ "aws-smithy-json",
 "aws-smithy-runtime",
 "aws-smithy-runtime-api",
 "aws-smithy-types",
@@ -469,15 +459,15 @@ dependencies = [

 [[package]]
 name = "aws-sdk-ssooidc"
-version = "1.51.0"
+version = "1.58.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "abaf490c2e48eed0bb8e2da2fb08405647bd7f253996e0f93b981958ea0f73b0"
+checksum = "8c8234fd024f7ac61c4e44ea008029bde934250f371efe7d4a39708397b1080c"
 dependencies = [
 "aws-credential-types",
 "aws-runtime",
 "aws-smithy-async",
 "aws-smithy-http",
- "aws-smithy-json 0.61.1",
+ "aws-smithy-json",
 "aws-smithy-runtime",
 "aws-smithy-runtime-api",
 "aws-smithy-types",
@@ -491,15 +481,15 @@ dependencies = [

 [[package]]
 name = "aws-sdk-sts"
-version = "1.51.0"
+version = "1.58.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b68fde0d69c8bfdc1060ea7da21df3e39f6014da316783336deff0a9ec28f4bf"
+checksum = "ba60e1d519d6f23a9df712c04fdeadd7872ac911c84b2f62a8bda92e129b7962"
 dependencies = [
 "aws-credential-types",
 "aws-runtime",
 "aws-smithy-async",
 "aws-smithy-http",
- "aws-smithy-json 0.61.1",
+ "aws-smithy-json",
 "aws-smithy-query",
 "aws-smithy-runtime",
 "aws-smithy-runtime-api",
@@ -514,9 +504,9 @@ dependencies = [

 [[package]]
 name = "aws-sigv4"
-version = "1.2.6"
+version = "1.2.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7d3820e0c08d0737872ff3c7c1f21ebbb6693d832312d6152bf18ef50a5471c2"
+checksum = "690118821e46967b3c4501d67d7d52dd75106a9c54cf36cefa1985cedbe94e05"
 dependencies = [
 "aws-credential-types",
 "aws-smithy-eventstream",
@@ -543,9 +533,9 @@ dependencies = [

 [[package]]
 name = "aws-smithy-async"
-version = "1.2.1"
+version = "1.2.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "62220bc6e97f946ddd51b5f1361f78996e704677afc518a4ff66b7a72ea1378c"
+checksum = "fa59d1327d8b5053c54bf2eaae63bf629ba9e904434d0835a28ed3c0ed0a614e"
 dependencies = [
 "futures-util",
 "pin-project-lite",
@@ -575,9 +565,9 @@ dependencies = [

 [[package]]
 name = "aws-smithy-eventstream"
-version = "0.60.5"
+version = "0.60.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cef7d0a272725f87e51ba2bf89f8c21e4df61b9e49ae1ac367a6d69916ef7c90"
+checksum = "8b18559a41e0c909b77625adf2b8c50de480a8041e5e4a3f5f7d177db70abc5a"
 dependencies = [
 "aws-smithy-types",
 "bytes",
@@ -586,9 +576,9 @@ dependencies = [

 [[package]]
 name = "aws-smithy-http"
-version = "0.60.11"
+version = "0.60.12"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5c8bc3e8fdc6b8d07d976e301c02fe553f72a39b7a9fea820e023268467d7ab6"
+checksum = "7809c27ad8da6a6a68c454e651d4962479e81472aa19ae99e59f9aba1f9713cc"
 dependencies = [
 "aws-smithy-eventstream",
 "aws-smithy-runtime-api",
@@ -607,18 +597,9 @@ dependencies = [

 [[package]]
 name = "aws-smithy-json"
-version = "0.60.7"
+version = "0.61.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4683df9469ef09468dad3473d129960119a0d3593617542b7d52086c8486f2d6"
-dependencies = [
- "aws-smithy-types",
-]
-
-[[package]]
-name = "aws-smithy-json"
-version = "0.61.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ee4e69cc50921eb913c6b662f8d909131bb3e6ad6cb6090d3a39b66fc5c52095"
+checksum = "623a51127f24c30776c8b374295f2df78d92517386f77ba30773f15a30ce1422"
 dependencies = [
 "aws-smithy-types",
 ]
@@ -635,9 +616,9 @@ dependencies = [

 [[package]]
 name = "aws-smithy-runtime"
-version = "1.7.4"
+version = "1.7.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9f20685047ca9d6f17b994a07f629c813f08b5bce65523e47124879e60103d45"
+checksum = "865f7050bbc7107a6c98a397a9fcd9413690c27fa718446967cf03b2d3ac517e"
 dependencies = [
 "aws-smithy-async",
 "aws-smithy-http",
@@ -679,9 +660,9 @@ dependencies = [

 [[package]]
 name = "aws-smithy-types"
-version = "1.2.9"
+version = "1.2.12"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4fbd94a32b3a7d55d3806fe27d98d3ad393050439dd05eb53ece36ec5e3d3510"
+checksum = "a28f6feb647fb5e0d5b50f0472c19a7db9462b74e2fec01bb0b44eedcc834e97"
 dependencies = [
 "base64-simd",
 "bytes",
@@ -714,9 +695,9 @@ dependencies = [

 [[package]]
 name = "aws-types"
-version = "1.3.3"
+version = "1.3.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5221b91b3e441e6675310829fd8984801b772cb1546ef6c0e54dec9f1ac13fef"
+checksum = "b0df5a18c4f951c645300d365fec53a61418bcf4650f604f85fe2a665bfaa0c2"
 dependencies = [
 "aws-credential-types",
 "aws-smithy-async",
@@ -786,7 +767,7 @@ dependencies = [
 [[package]]
 name = "azure_core"
 version = "0.21.0"
-source = "git+https://github.com/neondatabase/azure-sdk-for-rust.git?branch=neon#c36ed4c039bb3d59b5a1705f2cc337636c73b541"
+source = "git+https://github.com/neondatabase/azure-sdk-for-rust.git?branch=neon#66e77bdd87bf87e773acf3b0c84b532c1124367d"
 dependencies = [
 "async-trait",
 "base64 0.22.1",
@@ -815,7 +796,7 @@ dependencies = [
 [[package]]
 name = "azure_identity"
 version = "0.21.0"
-source = "git+https://github.com/neondatabase/azure-sdk-for-rust.git?branch=neon#c36ed4c039bb3d59b5a1705f2cc337636c73b541"
+source = "git+https://github.com/neondatabase/azure-sdk-for-rust.git?branch=neon#66e77bdd87bf87e773acf3b0c84b532c1124367d"
 dependencies = [
 "async-lock",
 "async-trait",
@@ -834,7 +815,7 @@ dependencies = [
 [[package]]
 name = "azure_storage"
 version = "0.21.0"
-source = "git+https://github.com/neondatabase/azure-sdk-for-rust.git?branch=neon#c36ed4c039bb3d59b5a1705f2cc337636c73b541"
+source = "git+https://github.com/neondatabase/azure-sdk-for-rust.git?branch=neon#66e77bdd87bf87e773acf3b0c84b532c1124367d"
 dependencies = [
 "RustyXML",
 "async-lock",
@@ -852,7 +833,7 @@ dependencies = [
 [[package]]
 name = "azure_storage_blobs"
 version = "0.21.0"
-source = "git+https://github.com/neondatabase/azure-sdk-for-rust.git?branch=neon#c36ed4c039bb3d59b5a1705f2cc337636c73b541"
+source = "git+https://github.com/neondatabase/azure-sdk-for-rust.git?branch=neon#66e77bdd87bf87e773acf3b0c84b532c1124367d"
 dependencies = [
 "RustyXML",
 "azure_core",
@@ -872,7 +853,7 @@ dependencies = [
 [[package]]
 name = "azure_svc_blobstorage"
 version = "0.21.0"
-source = "git+https://github.com/neondatabase/azure-sdk-for-rust.git?branch=neon#c36ed4c039bb3d59b5a1705f2cc337636c73b541"
+source = "git+https://github.com/neondatabase/azure-sdk-for-rust.git?branch=neon#66e77bdd87bf87e773acf3b0c84b532c1124367d"
 dependencies = [
 "azure_core",
 "bytes",
@@ -1029,12 +1010,6 @@ dependencies = [
 "generic-array",
 ]

-[[package]]
-name = "boxcar"
-version = "0.2.8"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2721c3c5a6f0e7f7e607125d963fedeb765f545f67adc9d71ed934693881eb42"
-
 [[package]]
 name = "bstr"
 version = "1.5.0"
@@ -1293,7 +1268,6 @@ version = "0.1.0"
 dependencies = [
 "anyhow",
 "chrono",
- "jsonwebtoken",
 "regex",
 "remote_storage",
 "serde",
@@ -1321,7 +1295,6 @@ dependencies = [
 "flate2",
 "futures",
 "http 1.1.0",
- "jsonwebtoken",
 "metrics",
 "nix 0.27.1",
 "notify",
@@ -1435,7 +1408,6 @@ dependencies = [
 "comfy-table",
 "compute_api",
 "futures",
- "http-utils",
 "humantime",
 "humantime-serde",
 "hyper 0.14.30",
@@ -2461,16 +2433,6 @@ dependencies = [
 "wasm-bindgen",
 ]

-[[package]]
-name = "gettid"
-version = "0.1.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "397256552fed4a9e577850498071831ec8f18ea83368aecc114cab469dcb43e5"
-dependencies = [
- "libc",
- "winapi",
-]
-
 [[package]]
 name = "gimli"
 version = "0.31.1"
@@ -2760,38 +2722,6 @@ dependencies = [
 "url",
 ]

-[[package]]
-name = "http-utils"
-version = "0.1.0"
-dependencies = [
- "anyhow",
- "backtrace",
- "bytes",
- "fail",
- "flate2",
- "hyper 0.14.30",
- "inferno 0.12.0",
- "itertools 0.10.5",
- "jemalloc_pprof",
- "metrics",
- "once_cell",
- "pprof",
- "regex",
- "routerify",
- "serde",
- "serde_json",
- "serde_path_to_error",
- "thiserror 1.0.69",
- "tokio",
- "tokio-stream",
- "tokio-util",
- "tracing",
- "url",
- "utils",
- "uuid",
- "workspace_hack",
-]
-
 [[package]]
 name = "httparse"
 version = "1.8.0"
@@ -4146,7 +4076,6 @@ dependencies = [
 "futures",
 "hex",
 "hex-literal",
- "http-utils",
 "humantime",
 "humantime-serde",
 "hyper 0.14.30",
@@ -4247,7 +4176,6 @@ dependencies = [
 "anyhow",
 "bytes",
 "futures",
- "http-utils",
 "pageserver_api",
 "postgres",
 "reqwest",
@@ -4284,16 +4212,6 @@ dependencies = [
 "workspace_hack",
 ]

-[[package]]
-name = "papaya"
-version = "0.1.8"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dc7c76487f7eaa00a0fc1d7f88dc6b295aec478d11b0fc79f857b62c2874124c"
-dependencies = [
- "equivalent",
- "seize",
-]
-
 [[package]]
 name = "parking"
 version = "2.1.1"
@@ -4921,7 +4839,6 @@ dependencies = [
 "ahash",
 "anyhow",
 "arc-swap",
- "assert-json-diff",
 "async-compression",
 "async-trait",
 "atomic-take",
@@ -4929,7 +4846,6 @@ dependencies = [
 "aws-sdk-iam",
 "aws-sigv4",
 "base64 0.13.1",
- "boxcar",
 "bstr",
 "bytes",
 "camino",
@@ -4946,7 +4862,6 @@ dependencies = [
 "flate2",
 "framed-websockets",
 "futures",
- "gettid",
 "hashbrown 0.14.5",
 "hashlink",
 "hex",
@@ -4954,7 +4869,6 @@ dependencies = [
 "hostname",
 "http 1.1.0",
 "http-body-util",
- "http-utils",
 "humantime",
 "humantime-serde",
 "hyper 0.14.30",
@@ -4970,9 +4884,7 @@ dependencies = [
 "measured",
 "metrics",
 "once_cell",
- "opentelemetry",
 "p256 0.13.2",
- "papaya",
 "parking_lot 0.12.1",
 "parquet",
 "parquet_derive",
@@ -5019,9 +4931,6 @@ dependencies = [
 "tokio-tungstenite 0.21.0",
 "tokio-util",
 "tracing",
- "tracing-log",
- "tracing-opentelemetry",
- "tracing-serde",
 "tracing-subscriber",
 "tracing-utils",
 "try-lock",
@@ -5802,7 +5711,6 @@ dependencies = [
 "futures",
 "hex",
 "http 1.1.0",
- "http-utils",
 "humantime",
 "hyper 0.14.30",
 "itertools 0.10.5",
@@ -5867,7 +5775,6 @@ dependencies = [
 name = "safekeeper_client"
 version = "0.1.0"
 dependencies = [
- "http-utils",
 "reqwest",
 "safekeeper_api",
 "serde",
@@ -5977,16 +5884,6 @@ dependencies = [
 "libc",
 ]

-[[package]]
-name = "seize"
-version = "0.4.9"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d84b0c858bdd30cb56f5597f8b3bf702ec23829e652cc636a1e5a7b9de46ae93"
-dependencies = [
- "libc",
- "windows-sys 0.52.0",
-]
-
 [[package]]
 name = "semver"
 version = "1.0.17"
@@ -6450,7 +6347,6 @@ dependencies = [
 "fail",
 "futures",
 "hex",
- "http-utils",
 "humantime",
 "hyper 0.14.30",
 "itertools 0.10.5",
@@ -6462,13 +6358,10 @@ dependencies = [
 "pageserver_client",
 "postgres_connection",
 "rand 0.8.5",
- "regex",
 "reqwest",
 "routerify",
 "rustls 0.23.18",
 "rustls-native-certs 0.8.0",
- "safekeeper_api",
- "safekeeper_client",
 "scoped-futures",
 "scopeguard",
 "serde",
@@ -7618,38 +7511,48 @@ dependencies = [
 "criterion",
 "diatomic-waker",
 "fail",
+ "flate2",
 "futures",
 "git-version",
 "hex",
 "hex-literal",
 "humantime",
+ "hyper 0.14.30",
 "inferno 0.12.0",
+ "itertools 0.10.5",
+ "jemalloc_pprof",
 "jsonwebtoken",
 "metrics",
 "nix 0.27.1",
 "once_cell",
 "pin-project-lite",
 "postgres_connection",
+ "pprof",
 "pq_proto",
 "rand 0.8.5",
 "regex",
+ "routerify",
 "scopeguard",
 "sentry",
 "serde",
 "serde_assert",
 "serde_json",
+ "serde_path_to_error",
 "serde_with",
 "signal-hook",
 "strum",
 "strum_macros",
 "thiserror 1.0.69",
 "tokio",
+ "tokio-stream",
 "tokio-tar",
 "tokio-util",
 "toml_edit",
 "tracing",
 "tracing-error",
 "tracing-subscriber",
+ "url",
+ "uuid",
 "walkdir",
 ]

@@ -8242,9 +8145,7 @@ dependencies = [
 "tower 0.4.13",
 "tracing",
 "tracing-core",
- "tracing-log",
 "url",
- "uuid",
 "zerocopy",
 "zeroize",
 "zstd",
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -18,7 +18,6 @@ members = [
    "storage_scrubber",
    "workspace_hack",
    "libs/compute_api",
-    "libs/http-utils",
    "libs/pageserver_api",
    "libs/postgres_ffi",
    "libs/safekeeper_api",
@@ -55,7 +54,6 @@ async-compression = { version = "0.4.0", features = ["tokio", "gzip", "zstd"] }
 atomic-take = "1.1.0"
 backtrace = "0.3.74"
 flate2 = "1.0.26"
-assert-json-diff = "2"
 async-stream = "0.3"
 async-trait = "0.1"
 aws-config = { version = "1.5", default-features = false, features=["rustls", "sso"] }
@@ -195,9 +193,7 @@ tower-http = { version = "0.6.2", features = ["request-id", "trace"] }
 tower-service = "0.3.3"
 tracing = "0.1"
 tracing-error = "0.2"
-tracing-log = "0.2"
 tracing-opentelemetry = "0.28"
-tracing-serde = "0.2.0"
 tracing-subscriber = { version = "0.3", default-features = false, features = ["smallvec", "fmt", "tracing-log", "std", "env-filter", "json"] }
 try-lock = "0.2.5"
 twox-hash = { version = "1.6.3", default-features = false }
@@ -230,7 +226,6 @@ azure_storage_blobs = { git = "https://github.com/neondatabase/azure-sdk-for-rus
 ## Local libraries
 compute_api = { version = "0.1", path = "./libs/compute_api/" }
 consumption_metrics = { version = "0.1", path = "./libs/consumption_metrics/" }
-http-utils = { version = "0.1", path = "./libs/http-utils/" }
 metrics = { version = "0.1", path = "./libs/metrics/" }
 pageserver = { path = "./pageserver" }
 pageserver_api = { version = "0.1", path = "./libs/pageserver_api/" }
--- a/41
+++ b/41
@@ -10,28 +10,6 @@ ARG STABLE_PG_VERSION=16
 ARG DEBIAN_VERSION=bookworm
 ARG DEBIAN_FLAVOR=${DEBIAN_VERSION}-slim

-# Here are the INDEX DIGESTS for the images we use.
-# You can get them following next steps for now:
-# 1. Get an authentication token from DockerHub:
-#    TOKEN=$(curl -s "https://auth.docker.io/token?service=registry.docker.io&scope=repository:library/debian:pull" | jq -r .token)
-# 2. Using that token, query index for the given tag:
-#    curl -s -H "Authorization: Bearer $TOKEN" \
-#       -H "Accept: application/vnd.docker.distribution.manifest.list.v2+json" \
-#       "https://registry.hub.docker.com/v2/library/debian/manifests/bullseye-slim" \
-#       -I | grep -i docker-content-digest
-# 3. As a next step, TODO(fedordikarev): create script and schedule workflow to run these checks
-#    and updates on regular bases and in automated way.
-ARG BOOKWORM_SLIM_SHA=sha256:40b107342c492725bc7aacbe93a49945445191ae364184a6d24fedb28172f6f7
-ARG BULLSEYE_SLIM_SHA=sha256:e831d9a884d63734fe3dd9c491ed9a5a3d4c6a6d32c5b14f2067357c49b0b7e1
-
-# Here we use ${var/search/replace} syntax, to check
-# if base image is one of the images, we pin image index for.
-# If var will match one the known images, we will replace it with the known sha.
-# If no match, than value will be unaffected, and will process with no-pinned image.
-ARG BASE_IMAGE_SHA=debian:${DEBIAN_FLAVOR}
-ARG BASE_IMAGE_SHA=${BASE_IMAGE_SHA/debian:bookworm-slim/debian@$BOOKWORM_SLIM_SHA}
-ARG BASE_IMAGE_SHA=${BASE_IMAGE_SHA/debian:bullseye-slim/debian@$BULLSEYE_SLIM_SHA}
-
 # Build Postgres
 FROM $REPOSITORY/$IMAGE:$TAG AS pg-build
 WORKDIR /home/nonroot
@@ -50,14 +28,6 @@ RUN set -e \
    && rm -rf pg_install/build \
    && tar -C pg_install -czf /home/nonroot/postgres_install.tar.gz .

-# Prepare cargo-chef recipe
-FROM $REPOSITORY/$IMAGE:$TAG AS plan
-WORKDIR /home/nonroot
-
-COPY --chown=nonroot . .
-
-RUN cargo chef prepare --recipe-path recipe.json
-
 # Build neon binaries
 FROM $REPOSITORY/$IMAGE:$TAG AS build
 WORKDIR /home/nonroot
@@ -71,15 +41,9 @@ COPY --from=pg-build /home/nonroot/pg_install/v16/include/postgresql/server pg_i
 COPY --from=pg-build /home/nonroot/pg_install/v17/include/postgresql/server pg_install/v17/include/postgresql/server
 COPY --from=pg-build /home/nonroot/pg_install/v16/lib                       pg_install/v16/lib
 COPY --from=pg-build /home/nonroot/pg_install/v17/lib                       pg_install/v17/lib
-COPY --from=plan     /home/nonroot/recipe.json                              recipe.json
-
-ARG ADDITIONAL_RUSTFLAGS=""
-
-RUN set -e \
-    && RUSTFLAGS="-Clinker=clang -Clink-arg=-fuse-ld=mold -Clink-arg=-Wl,--no-rosegment -Cforce-frame-pointers=yes ${ADDITIONAL_RUSTFLAGS}" cargo chef cook --locked --release --recipe-path recipe.json
-
 COPY --chown=nonroot . .

+ARG ADDITIONAL_RUSTFLAGS
 RUN set -e \
    && RUSTFLAGS="-Clinker=clang -Clink-arg=-fuse-ld=mold -Clink-arg=-Wl,--no-rosegment -Cforce-frame-pointers=yes ${ADDITIONAL_RUSTFLAGS}" cargo build \
      --bin pg_sni_router  \
@@ -95,7 +59,7 @@ RUN set -e \

 # Build final image
 #
-FROM $BASE_IMAGE_SHA
+FROM debian:${DEBIAN_FLAVOR}
 ARG DEFAULT_PG_VERSION
 WORKDIR /data

@@ -148,3 +112,4 @@ EXPOSE 6400
 EXPOSE 9898

 CMD ["/usr/local/bin/pageserver", "-D", "/data/.neon"]
+
--- a/17
+++ b/17
@@ -10,29 +10,18 @@ ICU_PREFIX_DIR := /usr/local/icu
 # environment variable.
 #
 BUILD_TYPE ?= debug
-WITH_SANITIZERS ?= no
 ifeq ($(BUILD_TYPE),release)
 	PG_CONFIGURE_OPTS = --enable-debug --with-openssl
 	PG_CFLAGS = -O2 -g3 $(CFLAGS)
-	PG_LDFLAGS = $(LDFLAGS)
 	# Unfortunately, `--profile=...` is a nightly feature
 	CARGO_BUILD_FLAGS += --release
 else ifeq ($(BUILD_TYPE),debug)
 	PG_CONFIGURE_OPTS = --enable-debug --with-openssl --enable-cassert --enable-depend
 	PG_CFLAGS = -O0 -g3 $(CFLAGS)
-	PG_LDFLAGS = $(LDFLAGS)
 else
 	$(error Bad build type '$(BUILD_TYPE)', see Makefile for options)
 endif

-ifeq ($(WITH_SANITIZERS),yes)
-	PG_CFLAGS += -fsanitize=address -fsanitize=undefined -fno-sanitize-recover
-	COPT += -Wno-error # to avoid failing on warnings induced by sanitizers
-	PG_LDFLAGS = -fsanitize=address -fsanitize=undefined -static-libasan -static-libubsan $(LDFLAGS)
-	export CC := gcc
-	export ASAN_OPTIONS := detect_leaks=0
-endif
-
 ifeq ($(shell test -e /home/nonroot/.docker_build && echo -n yes),yes)
 	# Exclude static build openssl, icu for local build (MacOS, Linux)
 	# Only keep for build type release and debug
@@ -44,9 +33,7 @@ endif
 UNAME_S := $(shell uname -s)
 ifeq ($(UNAME_S),Linux)
 	# Seccomp BPF is only available for Linux
-	ifneq ($(WITH_SANITIZERS),yes)
-		PG_CONFIGURE_OPTS += --with-libseccomp
-	endif
+	PG_CONFIGURE_OPTS += --with-libseccomp
 else ifeq ($(UNAME_S),Darwin)
 	PG_CFLAGS += -DUSE_PREFETCH
 	ifndef DISABLE_HOMEBREW
@@ -119,7 +106,7 @@ $(POSTGRES_INSTALL_DIR)/build/%/config.status:
 	EXTRA_VERSION=$$(cd $(ROOT_PROJECT_DIR)/vendor/postgres-$$VERSION && git rev-parse HEAD); \
 	(cd $(POSTGRES_INSTALL_DIR)/build/$$VERSION && \
 	env PATH="$(EXTRA_PATH_OVERRIDES):$$PATH" $(ROOT_PROJECT_DIR)/vendor/postgres-$$VERSION/configure \
-		CFLAGS='$(PG_CFLAGS)' LDFLAGS='$(PG_LDFLAGS)' \
+		CFLAGS='$(PG_CFLAGS)' \
 		$(PG_CONFIGURE_OPTS) --with-extra-version=" ($$EXTRA_VERSION)" \
 		--prefix=$(abspath $(POSTGRES_INSTALL_DIR))/$$VERSION > configure.log)

--- a/build-tools.Dockerfile
+++ b/build-tools.Dockerfile
@@ -1,42 +1,12 @@
 ARG DEBIAN_VERSION=bookworm
-ARG DEBIAN_FLAVOR=${DEBIAN_VERSION}-slim

-# Here are the INDEX DIGESTS for the images we use.
-# You can get them following next steps for now:
-# 1. Get an authentication token from DockerHub:
-#    TOKEN=$(curl -s "https://auth.docker.io/token?service=registry.docker.io&scope=repository:library/debian:pull" | jq -r .token)
-# 2. Using that token, query index for the given tag:
-#    curl -s -H "Authorization: Bearer $TOKEN" \
-#       -H "Accept: application/vnd.docker.distribution.manifest.list.v2+json" \
-#       "https://registry.hub.docker.com/v2/library/debian/manifests/bullseye-slim" \
-#       -I | grep -i docker-content-digest
-# 3. As a next step, TODO(fedordikarev): create script and schedule workflow to run these checks
-#    and updates on regular bases and in automated way.
-ARG BOOKWORM_SLIM_SHA=sha256:40b107342c492725bc7aacbe93a49945445191ae364184a6d24fedb28172f6f7
-ARG BULLSEYE_SLIM_SHA=sha256:e831d9a884d63734fe3dd9c491ed9a5a3d4c6a6d32c5b14f2067357c49b0b7e1
-
-# Here we use ${var/search/replace} syntax, to check
-# if base image is one of the images, we pin image index for.
-# If var will match one the known images, we will replace it with the known sha.
-# If no match, than value will be unaffected, and will process with no-pinned image.
-ARG BASE_IMAGE_SHA=debian:${DEBIAN_FLAVOR}
-ARG BASE_IMAGE_SHA=${BASE_IMAGE_SHA/debian:bookworm-slim/debian@$BOOKWORM_SLIM_SHA}
-ARG BASE_IMAGE_SHA=${BASE_IMAGE_SHA/debian:bullseye-slim/debian@$BULLSEYE_SLIM_SHA}
-
-FROM $BASE_IMAGE_SHA AS pgcopydb_builder
+FROM debian:bookworm-slim AS pgcopydb_builder
 ARG DEBIAN_VERSION

-# Use strict mode for bash to catch errors early
-SHELL ["/bin/bash", "-euo", "pipefail", "-c"]
-
-# By default, /bin/sh used in debian images will treat '\n' as eol,
-# but as we use bash as SHELL, and built-in echo in bash requires '-e' flag for that.
 RUN echo 'Acquire::Retries "5";' > /etc/apt/apt.conf.d/80-retries && \
-    echo -e "retry_connrefused=on\ntimeout=15\ntries=5\nretry-on-host-error=on\n" > /root/.wgetrc && \
+    echo -e "retry_connrefused = on\ntimeout=15\ntries=5\n" > /root/.wgetrc \
    echo -e "--retry-connrefused\n--connect-timeout 15\n--retry 5\n--max-time 300\n" > /root/.curlrc

-COPY build_tools/patches/pgcopydbv017.patch /pgcopydbv017.patch
-
 RUN if [ "${DEBIAN_VERSION}" = "bookworm" ]; then \
        set -e && \
        apt update && \
@@ -69,7 +39,6 @@ RUN if [ "${DEBIAN_VERSION}" = "bookworm" ]; then \
        mkdir /tmp/pgcopydb && \
        tar -xzf /tmp/pgcopydb.tar.gz -C /tmp/pgcopydb --strip-components=1 && \
        cd /tmp/pgcopydb && \
-        patch -p1 < /pgcopydbv017.patch && \
        make -s clean && \
        make -s -j12 install && \
        libpq_path=$(find /lib /usr/lib -name "libpq.so.5" | head -n 1) && \
@@ -81,13 +50,12 @@ RUN if [ "${DEBIAN_VERSION}" = "bookworm" ]; then \
        mkdir -p mkdir -p /pgcopydb/lib && touch /pgcopydb/lib/libpq.so.5; \
    fi

-FROM $BASE_IMAGE_SHA AS build_tools
+FROM debian:${DEBIAN_VERSION}-slim AS build_tools
 ARG DEBIAN_VERSION

 # Add nonroot user
 RUN useradd -ms /bin/bash nonroot -b /home
-# Use strict mode for bash to catch errors early
-SHELL ["/bin/bash", "-euo", "pipefail", "-c"]
+SHELL ["/bin/bash", "-c"]

 RUN mkdir -p /pgcopydb/bin && \
    mkdir -p /pgcopydb/lib && \
@@ -98,7 +66,7 @@ COPY --from=pgcopydb_builder /usr/lib/postgresql/16/bin/pgcopydb /pgcopydb/bin/p
 COPY --from=pgcopydb_builder /pgcopydb/lib/libpq.so.5 /pgcopydb/lib/libpq.so.5

 RUN echo 'Acquire::Retries "5";' > /etc/apt/apt.conf.d/80-retries && \
-    echo -e "retry_connrefused=on\ntimeout=15\ntries=5\nretry-on-host-error=on\n" > /root/.wgetrc && \
+    echo -e "retry_connrefused = on\ntimeout=15\ntries=5\n" > /root/.wgetrc \
    echo -e "--retry-connrefused\n--connect-timeout 15\n--retry 5\n--max-time 300\n" > /root/.curlrc

 # System deps
@@ -161,8 +129,7 @@ RUN curl -fsSL \
    --output sql_exporter.tar.gz \
    && mkdir /tmp/sql_exporter \
    && tar xzvf sql_exporter.tar.gz -C /tmp/sql_exporter --strip-components=1 \
-    && mv /tmp/sql_exporter/sql_exporter /usr/local/bin/sql_exporter \
-    && rm sql_exporter.tar.gz
+    && mv /tmp/sql_exporter/sql_exporter /usr/local/bin/sql_exporter

 # protobuf-compiler (protoc)
 ENV PROTOC_VERSION=25.1
@@ -223,14 +190,8 @@ RUN set -e \
 # It includes several bug fixes on top on v2.0 release (https://github.com/linux-test-project/lcov/compare/v2.0...master)
 # And patches from us:
 # - Generates json file with code coverage summary (https://github.com/neondatabase/lcov/commit/426e7e7a22f669da54278e9b55e6d8caabd00af0.tar.gz)
-RUN set +o pipefail && \
-	 for package in Capture::Tiny DateTime Devel::Cover Digest::MD5 File::Spec JSON::XS Memory::Process Time::HiRes JSON; do \
-		yes | perl -MCPAN -e "CPAN::Shell->notest('install', '$package')";\
-	 done && \
-	set -o pipefail
-# Split into separate step to debug flaky failures here
-RUN wget https://github.com/neondatabase/lcov/archive/426e7e7a22f669da54278e9b55e6d8caabd00af0.tar.gz -O lcov.tar.gz \
-    && ls -laht lcov.tar.gz && sha256sum lcov.tar.gz \
+RUN for package in Capture::Tiny DateTime Devel::Cover Digest::MD5 File::Spec JSON::XS Memory::Process Time::HiRes JSON; do yes | perl -MCPAN -e "CPAN::Shell->notest('install', '$package')"; done \
+    && wget https://github.com/neondatabase/lcov/archive/426e7e7a22f669da54278e9b55e6d8caabd00af0.tar.gz -O lcov.tar.gz \
    && echo "61a22a62e20908b8b9e27d890bd0ea31f567a7b9668065589266371dcbca0992  lcov.tar.gz" | sha256sum --check \
    && mkdir -p lcov && tar -xzf lcov.tar.gz -C lcov --strip-components=1 \
    && cd lcov \
@@ -300,7 +261,6 @@ ARG CARGO_HAKARI_VERSION=0.9.33
 ARG CARGO_DENY_VERSION=0.16.2
 ARG CARGO_HACK_VERSION=0.6.33
 ARG CARGO_NEXTEST_VERSION=0.9.85
-ARG CARGO_CHEF_VERSION=0.1.71
 ARG CARGO_DIESEL_CLI_VERSION=2.2.6
 RUN curl -sSO https://static.rust-lang.org/rustup/dist/$(uname -m)-unknown-linux-gnu/rustup-init && whoami && \
 	chmod +x rustup-init && \
@@ -315,7 +275,6 @@ RUN curl -sSO https://static.rust-lang.org/rustup/dist/$(uname -m)-unknown-linux
    cargo install cargo-deny --locked --version ${CARGO_DENY_VERSION} && \
    cargo install cargo-hack          --version ${CARGO_HACK_VERSION} && \
    cargo install cargo-nextest       --version ${CARGO_NEXTEST_VERSION} && \
-    cargo install cargo-chef --locked --version ${CARGO_CHEF_VERSION} && \
    cargo install diesel_cli          --version ${CARGO_DIESEL_CLI_VERSION} \
                                      --features postgres-bundled --no-default-features && \
    rm -rf /home/nonroot/.cargo/registry && \
--- a/build_tools/patches/pgcopydbv017.patch
+++ b/build_tools/patches/pgcopydbv017.patch
@@ -1,57 +0,0 @@
-diff --git a/src/bin/pgcopydb/copydb.c b/src/bin/pgcopydb/copydb.c
-index d730b03..69a9be9 100644
--- a/src/bin/pgcopydb/copydb.c
-+++ b/src/bin/pgcopydb/copydb.c
-@@ -44,6 +44,7 @@ GUC dstSettings[] = {
- 	{ "synchronous_commit", "'off'" },
- 	{ "statement_timeout", "0" },
- 	{ "lock_timeout", "0" },
-+	{ "idle_in_transaction_session_timeout", "0" },
- 	{ NULL, NULL },
- };
- 
-diff --git a/src/bin/pgcopydb/pgsql.c b/src/bin/pgcopydb/pgsql.c
-index 94f2f46..e051ba8 100644
--- a/src/bin/pgcopydb/pgsql.c
-+++ b/src/bin/pgcopydb/pgsql.c
-@@ -2319,6 +2319,11 @@ pgsql_execute_log_error(PGSQL *pgsql,
- 
- 	LinesBuffer lbuf = { 0 };
- 
-+	if (message != NULL){
-+		// make sure message is writable by splitLines
-+		message = strdup(message);
-+	}
-+
- 	if (!splitLines(&lbuf, message))
- 	{
- 		/* errors have already been logged */
-@@ -2332,6 +2337,7 @@ pgsql_execute_log_error(PGSQL *pgsql,
- 				  PQbackendPID(pgsql->connection),
- 				  lbuf.lines[lineNumber]);
- 	}
-+        free(message); // free copy of message we created above
- 
- 	if (pgsql->logSQL)
- 	{
-@@ -3174,11 +3180,18 @@ pgcopy_log_error(PGSQL *pgsql, PGresult *res, const char *context)
- 		/* errors have already been logged */
- 		return;
- 	}
-
- 	if (res != NULL)
- 	{
- 		char *sqlstate = PQresultErrorField(res, PG_DIAG_SQLSTATE);
-		strlcpy(pgsql->sqlstate, sqlstate, sizeof(pgsql->sqlstate));
-+		if (sqlstate == NULL)
-+		{
-+			// PQresultErrorField returned NULL!
-+			pgsql->sqlstate[0] = '\0';  // Set to an empty string to avoid segfault
-+		}
-+		else
-+		{
-+			strlcpy(pgsql->sqlstate, sqlstate, sizeof(pgsql->sqlstate));
-+		}
- 	}
- 
- 	char *endpoint =
--- a/compute/compute-node.Dockerfile
+++ b/compute/compute-node.Dockerfile
@@ -83,48 +83,21 @@ ARG TAG=pinned
 ARG BUILD_TAG
 ARG DEBIAN_VERSION=bookworm
 ARG DEBIAN_FLAVOR=${DEBIAN_VERSION}-slim
-
-# Here are the INDEX DIGESTS for the images we use.
-# You can get them following next steps for now:
-# 1. Get an authentication token from DockerHub:
-#    TOKEN=$(curl -s "https://auth.docker.io/token?service=registry.docker.io&scope=repository:library/debian:pull" | jq -r .token)
-# 2. Using that token, query index for the given tag:
-#    curl -s -H "Authorization: Bearer $TOKEN" \
-#       -H "Accept: application/vnd.docker.distribution.manifest.list.v2+json" \
-#       "https://registry.hub.docker.com/v2/library/debian/manifests/bullseye-slim" \
-#       -I | grep -i docker-content-digest
-# 3. As a next step, TODO(fedordikarev): create script and schedule workflow to run these checks
-#    and updates on regular bases and in automated way.
-ARG BOOKWORM_SLIM_SHA=sha256:40b107342c492725bc7aacbe93a49945445191ae364184a6d24fedb28172f6f7
-ARG BULLSEYE_SLIM_SHA=sha256:e831d9a884d63734fe3dd9c491ed9a5a3d4c6a6d32c5b14f2067357c49b0b7e1
-
-# Here we use ${var/search/replace} syntax, to check
-# if base image is one of the images, we pin image index for.
-# If var will match one the known images, we will replace it with the known sha.
-# If no match, than value will be unaffected, and will process with no-pinned image.
-ARG BASE_IMAGE_SHA=debian:${DEBIAN_FLAVOR}
-ARG BASE_IMAGE_SHA=${BASE_IMAGE_SHA/debian:bookworm-slim/debian@$BOOKWORM_SLIM_SHA}
-ARG BASE_IMAGE_SHA=${BASE_IMAGE_SHA/debian:bullseye-slim/debian@$BULLSEYE_SLIM_SHA}
-
-# By default, build all PostgreSQL extensions. For quick local testing when you don't
-# care about the extensions, pass EXTENSIONS=none or EXTENSIONS=minimal
-ARG EXTENSIONS=all
+ARG ALPINE_CURL_VERSION=8.11.1

 #########################################################################################
 #
 # Layer "build-deps"
 #
 #########################################################################################
-FROM $BASE_IMAGE_SHA AS build-deps
+FROM debian:$DEBIAN_FLAVOR AS build-deps
 ARG DEBIAN_VERSION

 # Use strict mode for bash to catch errors early
 SHELL ["/bin/bash", "-euo", "pipefail", "-c"]

-# By default, /bin/sh used in debian images will treat '\n' as eol,
-# but as we use bash as SHELL, and built-in echo in bash requires '-e' flag for that.
 RUN echo 'Acquire::Retries "5";' > /etc/apt/apt.conf.d/80-retries && \
-    echo -e "retry_connrefused = on\ntimeout=15\ntries=5\nretry-on-host-error=on\n" > /root/.wgetrc && \
+    echo -e "retry_connrefused = on\ntimeout=15\ntries=5\n" > /root/.wgetrc \
    echo -e "--retry-connrefused\n--connect-timeout 15\n--retry 5\n--max-time 300\n" > /root/.curlrc

 RUN case $DEBIAN_VERSION in \
@@ -148,7 +121,7 @@ RUN case $DEBIAN_VERSION in \
    apt install --no-install-recommends --no-install-suggests -y \
    ninja-build git autoconf automake libtool build-essential bison flex libreadline-dev \
    zlib1g-dev libxml2-dev libcurl4-openssl-dev libossp-uuid-dev wget ca-certificates pkg-config libssl-dev \
-    libicu-dev libxslt1-dev liblz4-dev libzstd-dev zstd curl unzip \
+    libicu-dev libxslt1-dev liblz4-dev libzstd-dev zstd \
    $VERSION_INSTALLS \
    && apt clean && rm -rf /var/lib/apt/lists/*

@@ -160,11 +133,11 @@ RUN case $DEBIAN_VERSION in \
 #########################################################################################
 FROM build-deps AS pg-build
 ARG PG_VERSION
-COPY vendor/postgres-${PG_VERSION:?} postgres
+COPY vendor/postgres-${PG_VERSION} postgres
 RUN cd postgres && \
    export CONFIGURE_CMD="./configure CFLAGS='-O2 -g3' --enable-debug --with-openssl --with-uuid=ossp \
    --with-icu --with-libxml --with-libxslt --with-lz4" && \
-    if [ "${PG_VERSION:?}" != "v14" ]; then \
+    if [ "${PG_VERSION}" != "v14" ]; then \
        # zstd is available only from PG15
        export CONFIGURE_CMD="${CONFIGURE_CMD} --with-zstd"; \
    fi && \
@@ -258,7 +231,7 @@ RUN case "${DEBIAN_VERSION}" in \

 # Postgis 3.5.0 supports v17
 WORKDIR /ext-src
-RUN case "${PG_VERSION:?}" in \
+RUN case "${PG_VERSION}" in \
    "v17") \
        export POSTGIS_VERSION=3.5.0 \
        export POSTGIS_CHECKSUM=ca698a22cc2b2b3467ac4e063b43a28413f3004ddd505bdccdd74c56a647f510 \
@@ -333,7 +306,7 @@ FROM build-deps AS pgrouting-src
 ARG DEBIAN_VERSION
 ARG PG_VERSION
 WORKDIR /ext-src
-RUN case "${PG_VERSION:?}" in \
+RUN case "${PG_VERSION}" in \
    "v17") \
        export PGROUTING_VERSION=3.6.2 \
        export PGROUTING_CHECKSUM=f4a1ed79d6f714e52548eca3bb8e5593c6745f1bde92eb5fb858efd8984dffa2 \
@@ -379,7 +352,7 @@ COPY compute/patches/plv8-3.1.10.patch .
 #
 # Use new version only for v17
 # because since v3.2, plv8 doesn't include plcoffee and plls extensions
-RUN case "${PG_VERSION:?}" in \
+RUN case "${PG_VERSION}" in \
    "v17") \
        export PLV8_TAG=v3.2.3 \
    ;; \
@@ -393,7 +366,7 @@ RUN case "${PG_VERSION:?}" in \
    git clone --recurse-submodules --depth 1 --branch ${PLV8_TAG} https://github.com/plv8/plv8.git plv8-src && \
    tar -czf plv8.tar.gz --exclude .git plv8-src && \
    cd plv8-src && \
-    if [[ "${PG_VERSION:?}" < "v17" ]]; then patch -p1 < /ext-src/plv8-3.1.10.patch; fi
+    if [[ "${PG_VERSION}" < "v17" ]]; then patch -p1 < /ext-src/plv8-3.1.10.patch; fi

 FROM pg-build AS plv8-build
 ARG PG_VERSION
@@ -413,7 +386,7 @@ RUN \
    find /usr/local/pgsql/ -name "plv8-*.so" | xargs strip && \
    # don't break computes with installed old version of plv8
    cd /usr/local/pgsql/lib/ && \
-    case "${PG_VERSION:?}" in \
+    case "${PG_VERSION}" in \
    "v17") \
        ln -s plv8-3.2.3.so plv8-3.1.8.so && \
        ln -s plv8-3.2.3.so plv8-3.1.5.so && \
@@ -750,7 +723,7 @@ FROM build-deps AS timescaledb-src
 ARG PG_VERSION

 WORKDIR /ext-src
-RUN case "${PG_VERSION:?}" in \
+RUN case "${PG_VERSION}" in \
      "v14" | "v15") \
        export TIMESCALEDB_VERSION=2.10.1 \
        export TIMESCALEDB_CHECKSUM=6fca72a6ed0f6d32d2b3523951ede73dc5f9b0077b38450a029a5f411fdb8c73 \
@@ -788,7 +761,7 @@ ARG PG_VERSION

 # version-specific, has separate releases for each version
 WORKDIR /ext-src
-RUN case "${PG_VERSION:?}" in \
+RUN case "${PG_VERSION}" in \
      "v14") \
        export PG_HINT_PLAN_VERSION=14_1_4_1 \
        export PG_HINT_PLAN_CHECKSUM=c3501becf70ead27f70626bce80ea401ceac6a77e2083ee5f3ff1f1444ec1ad1 \
@@ -864,7 +837,7 @@ ARG PG_VERSION
 # https://github.com/rdkit/rdkit/releases/tag/Release_2024_09_1

 WORKDIR /ext-src
-RUN case "${PG_VERSION:?}" in \
+RUN case "${PG_VERSION}" in \
    "v17") \
        export RDKIT_VERSION=Release_2024_09_1 \
        export RDKIT_CHECKSUM=034c00d6e9de323506834da03400761ed8c3721095114369d06805409747a60f \
@@ -991,7 +964,7 @@ ARG PG_VERSION
 #
 # last release v0.40.0 - Jul 22, 2024
 WORKDIR /ext-src
-RUN case "${PG_VERSION:?}" in \
+RUN case "${PG_VERSION}" in \
    "v17") \
        export SEMVER_VERSION=0.40.0 \
        export SEMVER_CHECKSUM=3e50bcc29a0e2e481e7b6d2bc937cadc5f5869f55d983b5a1aafeb49f5425cfc \
@@ -1027,7 +1000,7 @@ ARG PG_VERSION
 # This is our extension, support stopped in favor of pgvector
 # TODO: deprecate it
 WORKDIR /ext-src
-RUN case "${PG_VERSION:?}" in \
+RUN case "${PG_VERSION}" in \
      "v14" | "v15") \
        export PG_EMBEDDING_VERSION=0.3.5 \
        export PG_EMBEDDING_CHECKSUM=0e95b27b8b6196e2cf0a0c9ec143fe2219b82e54c5bb4ee064e76398cbe69ae9 \
@@ -1060,7 +1033,7 @@ ARG PG_VERSION
 # This is an experimental extension, never got to real production.
 # !Do not remove! It can be present in shared_preload_libraries and compute will fail to start if library is not found.
 WORKDIR /ext-src
-RUN case "${PG_VERSION:?}" in "v17") \
+RUN case "${PG_VERSION}" in "v17") \
    echo "postgresql_anonymizer does not yet support PG17" && exit 0;; \
    esac && \
    wget  https://github.com/neondatabase/postgresql_anonymizer/archive/refs/tags/neon_1.1.1.tar.gz -O pg_anon.tar.gz && \
@@ -1095,7 +1068,6 @@ ENV PATH="/home/nonroot/.cargo/bin:$PATH"
 USER nonroot
 WORKDIR /home/nonroot

-# See comment on the top of the file regading `echo` and `\n`
 RUN echo -e "--retry-connrefused\n--connect-timeout 15\n--retry 5\n--max-time 300\n" > /home/nonroot/.curlrc

 RUN curl -sSO https://static.rust-lang.org/rustup/dist/$(uname -m)-unknown-linux-gnu/rustup-init && \
@@ -1112,7 +1084,7 @@ RUN curl -sSO https://static.rust-lang.org/rustup/dist/$(uname -m)-unknown-linux
 FROM pg-build-nonroot-with-cargo AS rust-extensions-build
 ARG PG_VERSION

-RUN case "${PG_VERSION:?}" in \
+RUN case "${PG_VERSION}" in \
        'v17') \
            echo 'v17 is not supported yet by pgrx. Quit' && exit 0;; \
    esac && \
@@ -1291,7 +1263,7 @@ FROM build-deps AS pgx_ulid-src
 ARG PG_VERSION

 WORKDIR /ext-src
-RUN case "${PG_VERSION:?}" in \
+RUN case "${PG_VERSION}" in \
    "v14" | "v15" | "v16") \
        ;; \
    *) \
@@ -1323,7 +1295,7 @@ FROM build-deps AS pgx_ulid-pgrx12-src
 ARG PG_VERSION

 WORKDIR /ext-src
-RUN case "${PG_VERSION:?}" in \
+RUN case "${PG_VERSION}" in \
    "v17") \
        ;; \
    *) \
@@ -1451,8 +1423,8 @@ RUN make -j $(getconf _NPROCESSORS_ONLN) && \
 FROM build-deps AS pg_mooncake-src
 ARG PG_VERSION
 WORKDIR /ext-src
-RUN wget https://github.com/Mooncake-Labs/pg_mooncake/releases/download/v0.1.2/pg_mooncake-0.1.2.tar.gz -O pg_mooncake.tar.gz && \
-    echo "4550473784fcdd2e1e18062bc01eb9c286abd27cdf5e11a4399be6c0a426ba90 pg_mooncake.tar.gz" | sha256sum --check && \
+RUN wget https://github.com/Mooncake-Labs/pg_mooncake/releases/download/v0.1.1/pg_mooncake-0.1.1.tar.gz -O pg_mooncake.tar.gz && \
+    echo "a2d16eff7948dde64f072609ca5d2962d6b4d07cb89d45952add473529c55f55 pg_mooncake.tar.gz" | sha256sum --check && \
    mkdir pg_mooncake-src && cd pg_mooncake-src && tar xzf ../pg_mooncake.tar.gz --strip-components=1 -C . && \
    echo "make -f pg_mooncake-src/Makefile.build installcheck TEST_DIR=./test SQL_DIR=./sql SRC_DIR=./src" > neon-test.sh && \
    chmod a+x neon-test.sh
@@ -1509,35 +1481,12 @@ RUN make -j $(getconf _NPROCESSORS_ONLN) \

 #########################################################################################
 #
-# Layer "extensions-none"
-#
-#########################################################################################
-FROM build-deps AS extensions-none
-
-RUN mkdir /usr/local/pgsql
-
-#########################################################################################
-#
-# Layer "extensions-minimal"
-#
-# This subset of extensions includes the extensions that we have in
-# shared_preload_libraries by default.
-#
-#########################################################################################
-FROM build-deps AS extensions-minimal
-
-COPY --from=pgrag-build /usr/local/pgsql/ /usr/local/pgsql/
-COPY --from=timescaledb-build /usr/local/pgsql/ /usr/local/pgsql/
-COPY --from=pg_cron-build /usr/local/pgsql/ /usr/local/pgsql/
-COPY --from=pg_partman-build /usr/local/pgsql/ /usr/local/pgsql/
-
-#########################################################################################
-#
-# Layer "extensions-all"
+# Layer "all-extensions"
 # Bundle together all the extensions
 #
 #########################################################################################
-FROM build-deps AS extensions-all
+FROM build-deps AS all-extensions
+ARG PG_VERSION

 # Public extensions
 COPY --from=postgis-build /usr/local/pgsql/ /usr/local/pgsql/
@@ -1579,13 +1528,7 @@ COPY --from=pg_partman-build /usr/local/pgsql/ /usr/local/pgsql/
 COPY --from=pg_mooncake-build /usr/local/pgsql/ /usr/local/pgsql/
 COPY --from=pg_repack-build /usr/local/pgsql/ /usr/local/pgsql/

-#########################################################################################
-#
-# Layer "neon-pg-ext-build"
-# Includes Postgres and all the extensions chosen by EXTENSIONS arg.
-#
-#########################################################################################
-FROM extensions-${EXTENSIONS} AS neon-pg-ext-build
+COPY --from=neon-ext-build /usr/local/pgsql/ /usr/local/pgsql/

 #########################################################################################
 #
@@ -1599,15 +1542,7 @@ ENV BUILD_TAG=$BUILD_TAG
 USER nonroot
 # Copy entire project to get Cargo.* files with proper dependencies for the whole project
 COPY --chown=nonroot . .
-RUN --mount=type=cache,uid=1000,target=/home/nonroot/.cargo/registry \
-    --mount=type=cache,uid=1000,target=/home/nonroot/.cargo/git \
-    --mount=type=cache,uid=1000,target=/home/nonroot/target \
-    mold -run cargo build --locked --profile release-line-debug-size-lto --bin compute_ctl --bin fast_import --bin local_proxy && \
-    mkdir target-bin && \
-    cp target/release-line-debug-size-lto/compute_ctl \
-       target/release-line-debug-size-lto/fast_import \
-       target/release-line-debug-size-lto/local_proxy \
-       target-bin
+RUN mold -run cargo build --locked --profile release-line-debug-size-lto --bin compute_ctl --bin fast_import --bin local_proxy

 #########################################################################################
 #
@@ -1615,7 +1550,7 @@ RUN --mount=type=cache,uid=1000,target=/home/nonroot/.cargo/registry \
 #
 #########################################################################################

-FROM $BASE_IMAGE_SHA AS pgbouncer
+FROM debian:$DEBIAN_FLAVOR AS pgbouncer
 RUN set -e \
    && echo 'Acquire::Retries "5";' > /etc/apt/apt.conf.d/80-retries \
    && apt update \
@@ -1636,7 +1571,7 @@ RUN set -e \
    && git clone --recurse-submodules --depth 1 --branch ${PGBOUNCER_TAG} https://github.com/pgbouncer/pgbouncer.git pgbouncer \
    && cd pgbouncer \
    && ./autogen.sh \
-    && ./configure --prefix=/usr/local/pgbouncer --without-openssl \
+    && LDFLAGS=-static ./configure --prefix=/usr/local/pgbouncer --without-openssl \
    && make -j $(nproc) dist_man_MANS= \
    && make install dist_man_MANS=

@@ -1645,12 +1580,12 @@ RUN set -e \
 # Layer "exporters"
 #
 #########################################################################################
-FROM build-deps AS exporters
+FROM alpine/curl:${ALPINE_CURL_VERSION} AS exporters
 ARG TARGETARCH
 # Keep sql_exporter version same as in build-tools.Dockerfile and
 # test_runner/regress/test_compute_metrics.py
-# See comment on the top of the file regading `echo`, `-e` and `\n`
-RUN if [ "$TARGETARCH" = "amd64" ]; then\
+RUN echo -e "--retry-connrefused\n--connect-timeout 15\n--retry 5\n--max-time 300\n" > /root/.curlrc; \
+    if [ "$TARGETARCH" = "amd64" ]; then\
        postgres_exporter_sha256='027e75dda7af621237ff8f5ac66b78a40b0093595f06768612b92b1374bd3105';\
        pgbouncer_exporter_sha256='c9f7cf8dcff44f0472057e9bf52613d93f3ffbc381ad7547a959daa63c5e84ac';\
        sql_exporter_sha256='38e439732bbf6e28ca4a94d7bc3686d3fa1abdb0050773d5617a9efdb9e64d08';\
@@ -1669,37 +1604,13 @@ RUN if [ "$TARGETARCH" = "amd64" ]; then\
    && echo "${pgbouncer_exporter_sha256} pgbouncer_exporter" | sha256sum -c -\
    && echo "${sql_exporter_sha256} sql_exporter" | sha256sum -c -

-#########################################################################################
-#
-# Layer "awscli"
-#
-#########################################################################################
-FROM build-deps AS awscli
-ARG TARGETARCH
-RUN set -ex; \
-    if [ "${TARGETARCH}" = "amd64" ]; then \
-        TARGETARCH_ALT="x86_64"; \
-        CHECKSUM="c9a9df3770a3ff9259cb469b6179e02829687a464e0824d5c32d378820b53a00"; \
-    elif [ "${TARGETARCH}" = "arm64" ]; then \
-        TARGETARCH_ALT="aarch64"; \
-        CHECKSUM="8181730be7891582b38b028112e81b4899ca817e8c616aad807c9e9d1289223a"; \
-    else \
-        echo "Unsupported architecture: ${TARGETARCH}"; exit 1; \
-    fi; \
-    curl --retry 5 -L "https://awscli.amazonaws.com/awscli-exe-linux-${TARGETARCH_ALT}-2.17.5.zip" -o /tmp/awscliv2.zip; \
-    echo "${CHECKSUM}  /tmp/awscliv2.zip" | sha256sum -c -; \
-    unzip /tmp/awscliv2.zip -d /tmp/awscliv2; \
-    /tmp/awscliv2/aws/install; \
-    rm -rf /tmp/awscliv2.zip /tmp/awscliv2
-
 #########################################################################################
 #
 # Clean up postgres folder before inclusion
 #
 #########################################################################################
 FROM neon-ext-build AS postgres-cleanup-layer
-
-COPY --from=neon-pg-ext-build /usr/local/pgsql /usr/local/pgsql
+COPY --from=all-extensions /usr/local/pgsql /usr/local/pgsql

 # Remove binaries from /bin/ that we won't use (or would manually copy & install otherwise)
 RUN cd /usr/local/pgsql/bin && rm -f ecpg raster2pgsql shp2pgsql pgtopo_export pgtopo_import pgsql2shp
@@ -1724,7 +1635,7 @@ USER nonroot

 COPY --chown=nonroot compute compute

-RUN make PG_VERSION="${PG_VERSION:?}" -C compute
+RUN make PG_VERSION="${PG_VERSION}" -C compute

 #########################################################################################
 #
@@ -1750,15 +1661,15 @@ COPY --from=pg_graphql-src /ext-src/ /ext-src/
 COPY --from=hypopg-src /ext-src/ /ext-src/
 COPY --from=pg_hashids-src /ext-src/ /ext-src/
 COPY --from=rum-src /ext-src/ /ext-src/
-COPY --from=pgtap-src /ext-src/ /ext-src/
+#COPY --from=pgtap-src /ext-src/ /ext-src/
 COPY --from=ip4r-src /ext-src/ /ext-src/
 COPY --from=prefix-src /ext-src/ /ext-src/
 COPY --from=hll-src /ext-src/ /ext-src/
 COPY --from=plpgsql_check-src /ext-src/ /ext-src/
 #COPY --from=timescaledb-src /ext-src/ /ext-src/
 COPY --from=pg_hint_plan-src /ext-src/ /ext-src/
-COPY compute/patches/pg_hint_plan_${PG_VERSION:?}.patch /ext-src
-RUN cd /ext-src/pg_hint_plan-src && patch -p1 < /ext-src/pg_hint_plan_${PG_VERSION:?}.patch
+COPY compute/patches/pg_hint_plan_${PG_VERSION}.patch /ext-src
+RUN cd /ext-src/pg_hint_plan-src && patch -p1 < /ext-src/pg_hint_plan_${PG_VERSION}.patch
 COPY --from=pg_cron-src /ext-src/ /ext-src/
 #COPY --from=pgx_ulid-src /ext-src/ /ext-src/
 #COPY --from=pgx_ulid-pgrx12-src /ext-src/ /ext-src/
@@ -1787,11 +1698,46 @@ ENV PGDATABASE=postgres
 # Put it all together into the final image
 #
 #########################################################################################
-FROM $BASE_IMAGE_SHA
+FROM debian:$DEBIAN_FLAVOR
 ARG DEBIAN_VERSION
+# Add user postgres
+RUN mkdir /var/db && useradd -m -d /var/db/postgres postgres && \
+    echo "postgres:test_console_pass" | chpasswd && \
+    mkdir /var/db/postgres/compute && mkdir /var/db/postgres/specs && \
+    mkdir /var/db/postgres/pgbouncer && \
+    chown -R postgres:postgres /var/db/postgres && \
+    chmod 0750 /var/db/postgres/compute && \
+    chmod 0750 /var/db/postgres/pgbouncer && \
+    echo '/usr/local/lib' >> /etc/ld.so.conf && /sbin/ldconfig && \
+    # create folder for file cache
+    mkdir -p -m 777 /neon/cache

-# Use strict mode for bash to catch errors early
-SHELL ["/bin/bash", "-euo", "pipefail", "-c"]
+COPY --from=postgres-cleanup-layer --chown=postgres /usr/local/pgsql /usr/local
+COPY --from=compute-tools --chown=postgres /home/nonroot/target/release-line-debug-size-lto/compute_ctl /usr/local/bin/compute_ctl
+COPY --from=compute-tools --chown=postgres /home/nonroot/target/release-line-debug-size-lto/fast_import /usr/local/bin/fast_import
+
+# pgbouncer and its config
+COPY --from=pgbouncer         /usr/local/pgbouncer/bin/pgbouncer /usr/local/bin/pgbouncer
+COPY --chmod=0666 --chown=postgres compute/etc/pgbouncer.ini /etc/pgbouncer.ini
+
+# local_proxy and its config
+COPY --from=compute-tools --chown=postgres /home/nonroot/target/release-line-debug-size-lto/local_proxy /usr/local/bin/local_proxy
+RUN mkdir -p /etc/local_proxy && chown postgres:postgres /etc/local_proxy
+
+# Metrics exporter binaries and configuration files
+COPY --from=exporters ./postgres_exporter /bin/postgres_exporter
+COPY --from=exporters ./pgbouncer_exporter /bin/pgbouncer_exporter
+COPY --from=exporters ./sql_exporter /bin/sql_exporter
+
+COPY --chown=postgres compute/etc/postgres_exporter.yml /etc/postgres_exporter.yml
+
+COPY --from=sql_exporter_preprocessor --chmod=0644 /home/nonroot/compute/etc/sql_exporter.yml               /etc/sql_exporter.yml
+COPY --from=sql_exporter_preprocessor --chmod=0644 /home/nonroot/compute/etc/neon_collector.yml             /etc/neon_collector.yml
+COPY --from=sql_exporter_preprocessor --chmod=0644 /home/nonroot/compute/etc/sql_exporter_autoscaling.yml   /etc/sql_exporter_autoscaling.yml
+COPY --from=sql_exporter_preprocessor --chmod=0644 /home/nonroot/compute/etc/neon_collector_autoscaling.yml /etc/neon_collector_autoscaling.yml
+
+# Create remote extension download directory
+RUN mkdir /usr/local/download_extensions && chown -R postgres:postgres /usr/local/download_extensions

 # Install:
 # libreadline8 for psql
@@ -1802,9 +1748,10 @@ SHELL ["/bin/bash", "-euo", "pipefail", "-c"]
 # libzstd1 for zstd
 # libboost* for rdkit
 # ca-certificates for communicating with s3 by compute_ctl
-# libevent for pgbouncer
+
 RUN echo 'Acquire::Retries "5";' > /etc/apt/apt.conf.d/80-retries && \
    echo -e "retry_connrefused = on\ntimeout=15\ntries=5\n" > /root/.wgetrc
+
 RUN apt update && \
    case $DEBIAN_VERSION in \
      # Version-specific installs for Bullseye (PG14-PG16):
@@ -1839,57 +1786,33 @@ RUN apt update && \
        libxslt1.1 \
        libzstd1 \
        libcurl4 \
-        libevent-2.1-7 \
        locales \
        procps \
        ca-certificates \
+        curl \
+        unzip \
        $VERSION_INSTALLS && \
    apt clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* && \
    localedef -i en_US -c -f UTF-8 -A /usr/share/locale/locale.alias en_US.UTF-8

-# Add user postgres
-RUN mkdir /var/db && useradd -m -d /var/db/postgres postgres && \
-    echo "postgres:test_console_pass" | chpasswd && \
-    mkdir /var/db/postgres/compute && mkdir /var/db/postgres/specs && \
-    mkdir /var/db/postgres/pgbouncer && \
-    chown -R postgres:postgres /var/db/postgres && \
-    chmod 0750 /var/db/postgres/compute && \
-    chmod 0750 /var/db/postgres/pgbouncer && \
-    # create folder for file cache
-    mkdir -p -m 777 /neon/cache && \
-    # Create remote extension download directory
-    mkdir /usr/local/download_extensions && \
-    chown -R postgres:postgres /usr/local/download_extensions
-
-# aws cli is used by fast_import
-COPY --from=awscli /usr/local/aws-cli /usr/local/aws-cli
-
-# pgbouncer and its config
-COPY --from=pgbouncer         /usr/local/pgbouncer/bin/pgbouncer /usr/local/bin/pgbouncer
-COPY --chmod=0666 --chown=postgres compute/etc/pgbouncer.ini /etc/pgbouncer.ini
-
-COPY --from=postgres-cleanup-layer --chown=postgres /usr/local/pgsql /usr/local
-COPY --from=compute-tools --chown=postgres /home/nonroot/target-bin/compute_ctl /usr/local/bin/compute_ctl
-COPY --from=compute-tools --chown=postgres /home/nonroot/target-bin/fast_import /usr/local/bin/fast_import
-
-# local_proxy and its config
-COPY --from=compute-tools --chown=postgres /home/nonroot/target-bin/local_proxy /usr/local/bin/local_proxy
-RUN mkdir -p /etc/local_proxy && chown postgres:postgres /etc/local_proxy
-
-# Metrics exporter binaries and configuration files
-COPY --from=exporters ./postgres_exporter /bin/postgres_exporter
-COPY --from=exporters ./pgbouncer_exporter /bin/pgbouncer_exporter
-COPY --from=exporters ./sql_exporter /bin/sql_exporter
-
-COPY --chown=postgres compute/etc/postgres_exporter.yml /etc/postgres_exporter.yml
-
-COPY --from=sql_exporter_preprocessor --chmod=0644 /home/nonroot/compute/etc/sql_exporter.yml               /etc/sql_exporter.yml
-COPY --from=sql_exporter_preprocessor --chmod=0644 /home/nonroot/compute/etc/neon_collector.yml             /etc/neon_collector.yml
-COPY --from=sql_exporter_preprocessor --chmod=0644 /home/nonroot/compute/etc/sql_exporter_autoscaling.yml   /etc/sql_exporter_autoscaling.yml
-COPY --from=sql_exporter_preprocessor --chmod=0644 /home/nonroot/compute/etc/neon_collector_autoscaling.yml /etc/neon_collector_autoscaling.yml
-
-# Make the libraries we built available
-RUN echo '/usr/local/lib' >> /etc/ld.so.conf && /sbin/ldconfig
+# aws cli is used by fast_import (curl and unzip above are at this time only used for this installation step)
+ARG TARGETARCH
+RUN set -ex; \
+    if [ "${TARGETARCH}" = "amd64" ]; then \
+        TARGETARCH_ALT="x86_64"; \
+        CHECKSUM="c9a9df3770a3ff9259cb469b6179e02829687a464e0824d5c32d378820b53a00"; \
+    elif [ "${TARGETARCH}" = "arm64" ]; then \
+        TARGETARCH_ALT="aarch64"; \
+        CHECKSUM="8181730be7891582b38b028112e81b4899ca817e8c616aad807c9e9d1289223a"; \
+    else \
+        echo "Unsupported architecture: ${TARGETARCH}"; exit 1; \
+    fi; \
+    curl --retry 5 -L "https://awscli.amazonaws.com/awscli-exe-linux-${TARGETARCH_ALT}-2.17.5.zip" -o /tmp/awscliv2.zip; \
+    echo "${CHECKSUM}  /tmp/awscliv2.zip" | sha256sum -c -; \
+    unzip /tmp/awscliv2.zip -d /tmp/awscliv2; \
+    /tmp/awscliv2/aws/install; \
+    rm -rf /tmp/awscliv2.zip /tmp/awscliv2; \
+    true

 ENV LANG=en_US.utf8
 USER postgres
--- a/compute/patches/pg_hint_plan_v16.patch
+++ b/compute/patches/pg_hint_plan_v16.patch
@@ -6,16 +6,16 @@ index da723b8..5328114 100644
 ----
 -- No.A-1-1-3
 CREATE EXTENSION pg_hint_plan;
-+LOG:  Sending request to compute_ctl: http://localhost:3081/extension_server/pg_hint_plan
+LOG:  Sending request to compute_ctl: http://localhost:3080/extension_server/pg_hint_plan
 -- No.A-1-2-3
 DROP EXTENSION pg_hint_plan;
 -- No.A-1-1-4
 CREATE SCHEMA other_schema;
 CREATE EXTENSION pg_hint_plan SCHEMA other_schema;
-+LOG:  Sending request to compute_ctl: http://localhost:3081/extension_server/pg_hint_plan
+LOG:  Sending request to compute_ctl: http://localhost:3080/extension_server/pg_hint_plan
 ERROR:  extension "pg_hint_plan" must be installed in schema "hint_plan"
 CREATE EXTENSION pg_hint_plan;
-+LOG:  Sending request to compute_ctl: http://localhost:3081/extension_server/pg_hint_plan
+LOG:  Sending request to compute_ctl: http://localhost:3080/extension_server/pg_hint_plan
 DROP SCHEMA other_schema;
 ----
 ---- No. A-5-1 comment pattern
@@ -35,7 +35,7 @@ index d372459..6282afe 100644
 SET client_min_messages TO LOG;
 SET pg_hint_plan.enable_hint TO on;
 CREATE EXTENSION file_fdw;
-+LOG:  Sending request to compute_ctl: http://localhost:3081/extension_server/file_fdw
+LOG:  Sending request to compute_ctl: http://localhost:3080/extension_server/file_fdw
 CREATE SERVER file_server FOREIGN DATA WRAPPER file_fdw;
 CREATE USER MAPPING FOR PUBLIC SERVER file_server;
 CREATE FOREIGN TABLE ft1 (id int, val int) SERVER file_server OPTIONS (format 'csv', filename :'filename');
--- a/compute/patches/pg_hint_plan_v17.patch
+++ b/compute/patches/pg_hint_plan_v17.patch
@@ -6,16 +6,16 @@ index e7d68a1..65a056c 100644
 ----
 -- No.A-1-1-3
 CREATE EXTENSION pg_hint_plan;
-+LOG:  Sending request to compute_ctl: http://localhost:3081/extension_server/pg_hint_plan
+LOG:  Sending request to compute_ctl: http://localhost:3080/extension_server/pg_hint_plan
 -- No.A-1-2-3
 DROP EXTENSION pg_hint_plan;
 -- No.A-1-1-4
 CREATE SCHEMA other_schema;
 CREATE EXTENSION pg_hint_plan SCHEMA other_schema;
-+LOG:  Sending request to compute_ctl: http://localhost:3081/extension_server/pg_hint_plan
+LOG:  Sending request to compute_ctl: http://localhost:3080/extension_server/pg_hint_plan
 ERROR:  extension "pg_hint_plan" must be installed in schema "hint_plan"
 CREATE EXTENSION pg_hint_plan;
-+LOG:  Sending request to compute_ctl: http://localhost:3081/extension_server/pg_hint_plan
+LOG:  Sending request to compute_ctl: http://localhost:3080/extension_server/pg_hint_plan
 DROP SCHEMA other_schema;
 ----
 ---- No. A-5-1 comment pattern
@@ -168,7 +168,7 @@ index 017fa4b..98d989b 100644
 SET client_min_messages TO LOG;
 SET pg_hint_plan.enable_hint TO on;
 CREATE EXTENSION file_fdw;
-+LOG:  Sending request to compute_ctl: http://localhost:3081/extension_server/file_fdw
+LOG:  Sending request to compute_ctl: http://localhost:3080/extension_server/file_fdw
 CREATE SERVER file_server FOREIGN DATA WRAPPER file_fdw;
 CREATE USER MAPPING FOR PUBLIC SERVER file_server;
 CREATE FOREIGN TABLE ft1 (id int, val int) SERVER file_server OPTIONS (format 'csv', filename :'filename');
--- a/compute/vm-image-spec-bookworm.yaml
+++ b/compute/vm-image-spec-bookworm.yaml
@@ -47,9 +47,7 @@ files:
      # Allow postgres user (which is what compute_ctl runs as) to run /neonvm/bin/resize-swap
      # and /neonvm/bin/set-disk-quota as root without requiring entering a password (NOPASSWD),
      # regardless of hostname (ALL)
-      #
-      # Also allow it to shut down the VM. The fast_import job does that when it's finished.
-      postgres ALL=(root) NOPASSWD: /neonvm/bin/resize-swap, /neonvm/bin/set-disk-quota, /neonvm/bin/poweroff
+      postgres ALL=(root) NOPASSWD: /neonvm/bin/resize-swap, /neonvm/bin/set-disk-quota
  - filename: cgconfig.conf
    content: |
      # Configuration for cgroups in VM compute nodes
@@ -74,8 +72,8 @@ build: |
  # At time of migration to bookworm (2024-10-09), debian has a version of libcgroup/cgroup-tools 2.0.2,
  # and it _probably_ can be used as-is. However, we'll build it ourselves to minimise the changeset
  # for debian version migration.
-  ARG BOOKWORM_SLIM_SHA=sha256:40b107342c492725bc7aacbe93a49945445191ae364184a6d24fedb28172f6f7
-  FROM debian@$BOOKWORM_SLIM_SHA as libcgroup-builder
+  #
+  FROM debian:bookworm-slim as libcgroup-builder
  ENV LIBCGROUP_VERSION=v2.0.3

  RUN set -exu \
--- a/compute/vm-image-spec-bullseye.yaml
+++ b/compute/vm-image-spec-bullseye.yaml
@@ -68,8 +68,7 @@ build: |
  # At time of writing (2023-03-14), debian bullseye has a version of cgroup-tools (technically
  # libcgroup) that doesn't support cgroup v2 (version 0.41-11). Unfortunately, the vm-monitor
  # requires cgroup v2, so we'll build cgroup-tools ourselves.
-  ARG BULLSEYE_SLIM_SHA=sha256:e831d9a884d63734fe3dd9c491ed9a5a3d4c6a6d32c5b14f2067357c49b0b7e1
-  FROM debian@$BULLSEYE_SLIM_SHA as libcgroup-builder
+  FROM debian:bullseye-slim as libcgroup-builder
  ENV LIBCGROUP_VERSION=v2.0.3

  RUN set -exu \
--- a/compute_tools/Cargo.toml
+++ b/compute_tools/Cargo.toml
@@ -24,7 +24,6 @@ fail.workspace = true
 flate2.workspace = true
 futures.workspace = true
 http.workspace = true
-jsonwebtoken.workspace = true
 metrics.workspace = true
 nix.workspace = true
 notify.workspace = true
--- a/compute_tools/src/bin/compute_ctl.rs
+++ b/compute_tools/src/bin/compute_ctl.rs
@@ -41,21 +41,19 @@ use std::process::exit;
 use std::str::FromStr;
 use std::sync::atomic::Ordering;
 use std::sync::{mpsc, Arc, Condvar, Mutex, RwLock};
-use std::time::SystemTime;
 use std::{thread, time::Duration};

 use anyhow::{Context, Result};
 use chrono::Utc;
 use clap::Parser;
 use compute_tools::disk_quota::set_disk_quota;
-use compute_tools::http::server::Server;
 use compute_tools::lsn_lease::launch_lsn_lease_bg_task_for_static;
 use signal_hook::consts::{SIGQUIT, SIGTERM};
 use signal_hook::{consts::SIGINT, iterator::Signals};
 use tracing::{error, info, warn};
 use url::Url;

-use compute_api::responses::{ComputeCtlConfig, ComputeStatus};
+use compute_api::responses::ComputeStatus;
 use compute_api::spec::ComputeSpec;

 use compute_tools::compute::{
@@ -63,6 +61,7 @@ use compute_tools::compute::{
 };
 use compute_tools::configurator::launch_configurator;
 use compute_tools::extension_server::get_pg_version_string;
+use compute_tools::http::launch_http_server;
 use compute_tools::logger::*;
 use compute_tools::monitor::launch_monitor;
 use compute_tools::params::*;
@@ -86,19 +85,6 @@ fn parse_remote_ext_config(arg: &str) -> Result<String> {
    }
 }

-/// Generate a compute ID if one is not supplied. This exists to keep forward
-/// compatibility tests working, but will be removed in a future iteration.
-fn generate_compute_id() -> String {
-    let now = SystemTime::now();
-
-    format!(
-        "compute-{}",
-        now.duration_since(SystemTime::UNIX_EPOCH)
-            .unwrap()
-            .as_secs()
-    )
-}
-
 #[derive(Parser)]
 #[command(rename_all = "kebab-case")]
 struct Cli {
@@ -108,20 +94,8 @@ struct Cli {
    #[arg(short = 'r', long, value_parser = parse_remote_ext_config)]
    pub remote_ext_config: Option<String>,

-    /// The port to bind the external listening HTTP server to. Clients running
-    /// outside the compute will talk to the compute through this port. Keep
-    /// the previous name for this argument around for a smoother release
-    /// with the control plane.
-    ///
-    /// TODO: Remove the alias after the control plane release which teaches the
-    /// control plane about the renamed argument.
-    #[arg(long, alias = "http-port", default_value_t = 3080)]
-    pub external_http_port: u16,
-
-    /// The port to bind the internal listening HTTP server to. Clients like
-    /// the neon extension (for installing remote extensions) and local_proxy.
-    #[arg(long)]
-    pub internal_http_port: Option<u16>,
+    #[arg(long, default_value_t = 3080)]
+    pub http_port: u16,

    #[arg(short = 'D', long, value_name = "DATADIR")]
    pub pgdata: String,
@@ -156,26 +130,17 @@ struct Cli {
    #[arg(short = 'S', long, group = "spec-path")]
    pub spec_path: Option<OsString>,

-    #[arg(short = 'i', long, group = "compute-id", default_value = generate_compute_id())]
-    pub compute_id: String,
+    #[arg(short = 'i', long, group = "compute-id", conflicts_with_all = ["spec", "spec-path"])]
+    pub compute_id: Option<String>,

-    #[arg(short = 'p', long, conflicts_with_all = ["spec", "spec-path"], value_name = "CONTROL_PLANE_API_BASE_URL")]
+    #[arg(short = 'p', long, conflicts_with_all = ["spec", "spec-path"], requires = "compute-id", value_name = "CONTROL_PLANE_API_BASE_URL")]
    pub control_plane_uri: Option<String>,
 }

 fn main() -> Result<()> {
    let cli = Cli::parse();

-    // For historical reasons, the main thread that processes the spec and launches postgres
-    // is synchronous, but we always have this tokio runtime available and we "enter" it so
-    // that you can use tokio::spawn() and tokio::runtime::Handle::current().block_on(...)
-    // from all parts of compute_ctl.
-    let runtime = tokio::runtime::Builder::new_multi_thread()
-        .enable_all()
-        .build()?;
-    let _rt_guard = runtime.enter();
-
-    let build_tag = runtime.block_on(init())?;
+    let build_tag = init()?;

    let scenario = failpoint_support::init();

@@ -207,8 +172,8 @@ fn main() -> Result<()> {
    deinit_and_exit(wait_pg_result);
 }

-async fn init() -> Result<String> {
-    init_tracing_and_logging(DEFAULT_LOG_LEVEL).await?;
+fn init() -> Result<String> {
+    init_tracing_and_logging(DEFAULT_LOG_LEVEL)?;

    let mut signals = Signals::new([SIGINT, SIGTERM, SIGQUIT])?;
    thread::spawn(move || {
@@ -281,7 +246,6 @@ fn try_spec_from_cli(cli: &Cli) -> Result<CliSpecParams> {
        info!("got spec from cli argument {}", spec_json);
        return Ok(CliSpecParams {
            spec: Some(serde_json::from_str(spec_json)?),
-            compute_ctl_config: ComputeCtlConfig::default(),
            live_config_allowed: false,
        });
    }
@@ -291,19 +255,26 @@ fn try_spec_from_cli(cli: &Cli) -> Result<CliSpecParams> {
        let file = File::open(Path::new(spec_path))?;
        return Ok(CliSpecParams {
            spec: Some(serde_json::from_reader(file)?),
-            compute_ctl_config: ComputeCtlConfig::default(),
            live_config_allowed: true,
        });
    }

+    if cli.compute_id.is_none() {
+        panic!(
+            "compute spec should be provided by one of the following ways: \
+                --spec OR --spec-path OR --control-plane-uri and --compute-id"
+        );
+    };
    if cli.control_plane_uri.is_none() {
-        panic!("must specify --control-plane-uri");
+        panic!("must specify both --control-plane-uri and --compute-id or none");
    };

-    match get_spec_from_control_plane(cli.control_plane_uri.as_ref().unwrap(), &cli.compute_id) {
-        Ok(resp) => Ok(CliSpecParams {
-            spec: resp.0,
-            compute_ctl_config: resp.1,
+    match get_spec_from_control_plane(
+        cli.control_plane_uri.as_ref().unwrap(),
+        cli.compute_id.as_ref().unwrap(),
+    ) {
+        Ok(spec) => Ok(CliSpecParams {
+            spec,
            live_config_allowed: true,
        }),
        Err(e) => {
@@ -320,8 +291,6 @@ fn try_spec_from_cli(cli: &Cli) -> Result<CliSpecParams> {
 struct CliSpecParams {
    /// If a spec was provided via CLI or file, the [`ComputeSpec`]
    spec: Option<ComputeSpec>,
-    #[allow(dead_code)]
-    compute_ctl_config: ComputeCtlConfig,
    live_config_allowed: bool,
 }

@@ -331,7 +300,6 @@ fn wait_spec(
    CliSpecParams {
        spec,
        live_config_allowed,
-        compute_ctl_config: _,
    }: CliSpecParams,
 ) -> Result<Arc<ComputeNode>> {
    let mut new_state = ComputeState::new();
@@ -351,15 +319,13 @@ fn wait_spec(
    let tokio_conn_conf = tokio_postgres::config::Config::from_str(connstr.as_str())
        .context("cannot build tokio postgres config from connstr")?;
    let compute_node = ComputeNode {
-        compute_id: cli.compute_id.clone(),
        connstr,
        conn_conf,
        tokio_conn_conf,
        pgdata: cli.pgdata.clone(),
        pgbin: cli.pgbin.clone(),
        pgversion: get_pg_version_string(&cli.pgbin),
-        external_http_port: cli.external_http_port,
-        internal_http_port: cli.internal_http_port.unwrap_or(cli.external_http_port + 1),
+        http_port: cli.http_port,
        live_config_allowed,
        state: Mutex::new(new_state),
        state_changed: Condvar::new(),
@@ -377,13 +343,10 @@ fn wait_spec(
        compute.prewarm_postgres()?;
    }

-    // Launch the external HTTP server first, so that we can serve control plane
-    // requests while configuration is still in progress.
-    Server::External(cli.external_http_port).launch(&compute);
-
-    // The internal HTTP server could be launched later, but there isn't much
-    // sense in waiting.
-    Server::Internal(cli.internal_http_port.unwrap_or(cli.external_http_port + 1)).launch(&compute);
+    // Launch http service first, so that we can serve control-plane requests
+    // while configuration is still in progress.
+    let _http_handle =
+        launch_http_server(cli.http_port, &compute).expect("cannot launch http endpoint thread");

    if !spec_set {
        // No spec provided, hang waiting for it.
@@ -521,6 +484,21 @@ fn start_postgres(
            use std::env;
            use tokio_util::sync::CancellationToken;

+            // Note: it seems like you can make a runtime in an inner scope and
+            // if you start a task in it it won't be dropped. However, make it
+            // in the outermost scope just to be safe.
+            let rt = if env::var_os("AUTOSCALING").is_some() {
+                Some(
+                    tokio::runtime::Builder::new_multi_thread()
+                        .worker_threads(4)
+                        .enable_all()
+                        .build()
+                        .expect("failed to create tokio runtime for monitor")
+                )
+            } else {
+                None
+            };
+
            // This token is used internally by the monitor to clean up all threads
            let token = CancellationToken::new();

@@ -531,19 +509,16 @@ fn start_postgres(
                Some(cli.filecache_connstr.clone())
            };

-            let vm_monitor = if env::var_os("AUTOSCALING").is_some() {
-                let vm_monitor = tokio::spawn(vm_monitor::start(
+            let vm_monitor = rt.as_ref().map(|rt| {
+                rt.spawn(vm_monitor::start(
                    Box::leak(Box::new(vm_monitor::Args {
                        cgroup: Some(cli.cgroup.clone()),
                        pgconnstr,
                        addr: cli.vm_monitor_addr.clone(),
                    })),
                    token.clone(),
-                ));
-                Some(vm_monitor)
-            } else {
-                None
-            };
+                ))
+            });
        }
    }

@@ -553,6 +528,8 @@ fn start_postgres(
            delay_exit,
            compute,
            #[cfg(target_os = "linux")]
+            rt,
+            #[cfg(target_os = "linux")]
            token,
            #[cfg(target_os = "linux")]
            vm_monitor,
@@ -560,13 +537,15 @@ fn start_postgres(
    ))
 }

-type PostgresHandle = (std::process::Child, tokio::task::JoinHandle<Result<()>>);
+type PostgresHandle = (std::process::Child, std::thread::JoinHandle<()>);

 struct StartPostgresResult {
    delay_exit: bool,
    // passed through from WaitSpecResult
    compute: Arc<ComputeNode>,

+    #[cfg(target_os = "linux")]
+    rt: Option<tokio::runtime::Runtime>,
    #[cfg(target_os = "linux")]
    token: tokio_util::sync::CancellationToken,
    #[cfg(target_os = "linux")]
@@ -585,10 +564,10 @@ fn wait_postgres(pg: Option<PostgresHandle>) -> Result<WaitPostgresResult> {
            .expect("failed to start waiting on Postgres process");
        PG_PID.store(0, Ordering::SeqCst);

-        // Process has exited. Wait for the log collecting task to finish.
-        let _ = tokio::runtime::Handle::current()
-            .block_on(logs_handle)
-            .map_err(|e| tracing::error!("log task panicked: {:?}", e));
+        // Process has exited, so we can join the logs thread.
+        let _ = logs_handle
+            .join()
+            .map_err(|e| tracing::error!("log thread panicked: {:?}", e));

        info!("Postgres exited with code {}, shutting down", ecode);
        exit_code = ecode.code()
@@ -609,6 +588,8 @@ fn cleanup_after_postgres_exit(
        vm_monitor,
        #[cfg(target_os = "linux")]
        token,
+        #[cfg(target_os = "linux")]
+        rt,
    }: StartPostgresResult,
 ) -> Result<bool> {
    // Terminate the vm_monitor so it releases the file watcher on
@@ -621,6 +602,10 @@ fn cleanup_after_postgres_exit(
                token.cancel();
                // Kills the actual task running the monitor
                handle.abort();
+
+                // If handle is some, rt must have been used to produce it, and
+                // hence is also some
+                rt.unwrap().shutdown_timeout(Duration::from_secs(2));
            }
        }
    }
--- a/compute_tools/src/bin/fast_import.rs
+++ b/compute_tools/src/bin/fast_import.rs
@@ -60,16 +60,6 @@ struct Args {
    pg_lib_dir: Utf8PathBuf,
    #[clap(long)]
    pg_port: Option<u16>, // port to run postgres on, 5432 is default
-
-    /// Number of CPUs in the system. This is used to configure # of
-    /// parallel worker processes, for index creation.
-    #[clap(long, env = "NEON_IMPORTER_NUM_CPUS")]
-    num_cpus: Option<usize>,
-
-    /// Amount of RAM in the system. This is used to configure shared_buffers
-    /// and maintenance_work_mem.
-    #[clap(long, env = "NEON_IMPORTER_MEMORY_MB")]
-    memory_mb: Option<usize>,
 }

 #[serde_with::serde_as]
@@ -212,16 +202,7 @@ pub(crate) async fn main() -> anyhow::Result<()> {
    .await
    .context("initdb")?;

-    // If the caller didn't specify CPU / RAM to use for sizing, default to
-    // number of CPUs in the system, and pretty arbitrarily, 256 MB of RAM.
-    let nproc = args.num_cpus.unwrap_or_else(num_cpus::get);
-    let memory_mb = args.memory_mb.unwrap_or(256);
-
-    // Somewhat arbitrarily, use 10 % of memory for shared buffer cache, 70% for
-    // maintenance_work_mem (i.e. for sorting during index creation), and leave the rest
-    // available for misc other stuff that PostgreSQL uses memory for.
-    let shared_buffers_mb = ((memory_mb as f32) * 0.10) as usize;
-    let maintenance_work_mem_mb = ((memory_mb as f32) * 0.70) as usize;
+    let nproc = num_cpus::get();

    //
    // Launch postgres process
@@ -231,15 +212,12 @@ pub(crate) async fn main() -> anyhow::Result<()> {
        .arg(&pgdata_dir)
        .args(["-p", &format!("{pg_port}")])
        .args(["-c", "wal_level=minimal"])
-        .args(["-c", &format!("shared_buffers={shared_buffers_mb}MB")])
+        .args(["-c", "shared_buffers=10GB"])
        .args(["-c", "max_wal_senders=0"])
        .args(["-c", "fsync=off"])
        .args(["-c", "full_page_writes=off"])
        .args(["-c", "synchronous_commit=off"])
-        .args([
-            "-c",
-            &format!("maintenance_work_mem={maintenance_work_mem_mb}MB"),
-        ])
+        .args(["-c", "maintenance_work_mem=8388608"])
        .args(["-c", &format!("max_parallel_maintenance_workers={nproc}")])
        .args(["-c", &format!("max_parallel_workers={nproc}")])
        .args(["-c", &format!("max_parallel_workers_per_gather={nproc}")])
@@ -253,14 +231,6 @@ pub(crate) async fn main() -> anyhow::Result<()> {
        ])
        .env_clear()
        .env("LD_LIBRARY_PATH", &pg_lib_dir)
-        .env(
-            "ASAN_OPTIONS",
-            std::env::var("ASAN_OPTIONS").unwrap_or_default(),
-        )
-        .env(
-            "UBSAN_OPTIONS",
-            std::env::var("UBSAN_OPTIONS").unwrap_or_default(),
-        )
        .stdout(std::process::Stdio::piped())
        .stderr(std::process::Stdio::piped())
        .spawn()
--- a/compute_tools/src/catalog.rs
+++ b/compute_tools/src/catalog.rs
@@ -140,34 +140,5 @@ pub async fn get_database_schema(
            warn!("pg_dump stderr: {}", line)
        }
    });
-
-    #[allow(dead_code)]
-    struct SchemaStream<S> {
-        // We keep a reference to the child process to ensure it stays alive
-        // while the stream is being consumed. When SchemaStream is dropped,
-        // cmd will be dropped, which triggers kill_on_drop and terminates pg_dump
-        cmd: tokio::process::Child,
-        stream: S,
-    }
-
-    impl<S> Stream for SchemaStream<S>
-    where
-        S: Stream<Item = Result<bytes::Bytes, std::io::Error>> + Unpin,
-    {
-        type Item = Result<bytes::Bytes, std::io::Error>;
-
-        fn poll_next(
-            mut self: std::pin::Pin<&mut Self>,
-            cx: &mut std::task::Context<'_>,
-        ) -> std::task::Poll<Option<Self::Item>> {
-            Stream::poll_next(std::pin::Pin::new(&mut self.stream), cx)
-        }
-    }
-
-    let schema_stream = SchemaStream {
-        cmd,
-        stream: initial_stream.chain(stdout_reader.map(|res| res.map(|b| b.freeze()))),
-    };
-
-    Ok(schema_stream)
+    Ok(initial_stream.chain(stdout_reader.map(|res| res.map(|b| b.freeze()))))
 }
--- a/compute_tools/src/compute.rs
+++ b/compute_tools/src/compute.rs
@@ -9,6 +9,7 @@ use std::str::FromStr;
 use std::sync::atomic::AtomicU32;
 use std::sync::atomic::Ordering;
 use std::sync::{Arc, Condvar, Mutex, RwLock};
+use std::thread;
 use std::time::Duration;
 use std::time::Instant;

@@ -58,8 +59,6 @@ pub static PG_PID: AtomicU32 = AtomicU32::new(0);

 /// Compute node info shared across several `compute_ctl` threads.
 pub struct ComputeNode {
-    /// The ID of the compute
-    pub compute_id: String,
    // Url type maintains proper escaping
    pub connstr: url::Url,
    // We connect to Postgres from many different places, so build configs once
@@ -82,10 +81,8 @@ pub struct ComputeNode {
    /// - we push spec and it does configuration
    /// - but then it is restarted without any spec again
    pub live_config_allowed: bool,
-    /// The port that the compute's external HTTP server listens on
-    pub external_http_port: u16,
-    /// The port that the compute's internal HTTP server listens on
-    pub internal_http_port: u16,
+    /// The port that the compute's HTTP server listens on
+    pub http_port: u16,
    /// Volatile part of the `ComputeNode`, which should be used under `Mutex`.
    /// To allow HTTP API server to serving status requests, while configuration
    /// is in progress, lock should be held only for short periods of time to do
@@ -549,7 +546,11 @@ impl ComputeNode {
    pub fn check_safekeepers_synced(&self, compute_state: &ComputeState) -> Result<Option<Lsn>> {
        let start_time = Utc::now();

-        let rt = tokio::runtime::Handle::current();
+        // Run actual work with new tokio runtime
+        let rt = tokio::runtime::Builder::new_current_thread()
+            .enable_all()
+            .build()
+            .expect("failed to create rt");
        let result = rt.block_on(self.check_safekeepers_synced_async(compute_state));

        // Record runtime
@@ -596,9 +597,9 @@ impl ComputeNode {
        SYNC_SAFEKEEPERS_PID.store(0, Ordering::SeqCst);

        // Process has exited, so we can join the logs thread.
-        let _ = tokio::runtime::Handle::current()
-            .block_on(logs_handle)
-            .map_err(|e| tracing::error!("log task panicked: {:?}", e));
+        let _ = logs_handle
+            .join()
+            .map_err(|e| tracing::error!("log thread panicked: {:?}", e));

        if !sync_output.status.success() {
            anyhow::bail!(
@@ -633,7 +634,7 @@ impl ComputeNode {
        config::write_postgres_conf(
            &pgdata_path.join("postgresql.conf"),
            &pspec.spec,
-            self.internal_http_port,
+            self.http_port,
        )?;

        // Syncing safekeepers is only safe with primary nodes: if a primary
@@ -783,7 +784,7 @@ impl ComputeNode {
    pub fn start_postgres(
        &self,
        storage_auth_token: Option<String>,
-    ) -> Result<(std::process::Child, tokio::task::JoinHandle<Result<()>>)> {
+    ) -> Result<(std::process::Child, std::thread::JoinHandle<()>)> {
        let pgdata_path = Path::new(&self.pgdata);

        // Run postgres as a child process.
@@ -799,7 +800,7 @@ impl ComputeNode {
            .expect("cannot start postgres process");
        PG_PID.store(pg.id(), Ordering::SeqCst);

-        // Start a task to collect logs from stderr.
+        // Start a thread to collect logs from stderr.
        let stderr = pg.stderr.take().expect("stderr should be captured");
        let logs_handle = handle_postgres_logs(stderr);

@@ -808,28 +809,20 @@ impl ComputeNode {
        Ok((pg, logs_handle))
    }

-    /// Do post configuration of the already started Postgres. This function spawns a background task to
+    /// Do post configuration of the already started Postgres. This function spawns a background thread to
    /// configure the database after applying the compute spec. Currently, it upgrades the neon extension
    /// version. In the future, it may upgrade all 3rd-party extensions.
    #[instrument(skip_all)]
    pub fn post_apply_config(&self) -> Result<()> {
-        let conf = self.get_tokio_conn_conf(Some("compute_ctl:post_apply_config"));
-        tokio::spawn(async move {
-            let res = async {
-                let (mut client, connection) = conf.connect(NoTls).await?;
-                tokio::spawn(async move {
-                    if let Err(e) = connection.await {
-                        eprintln!("connection error: {}", e);
-                    }
-                });
-
+        let conf = self.get_conn_conf(Some("compute_ctl:post_apply_config"));
+        thread::spawn(move || {
+            let func = || {
+                let mut client = conf.connect(NoTls)?;
                handle_neon_extension_upgrade(&mut client)
-                    .await
                    .context("handle_neon_extension_upgrade")?;
                Ok::<_, anyhow::Error>(())
-            }
-            .await;
-            if let Err(err) = res {
+            };
+            if let Err(err) = func() {
                error!("error while post_apply_config: {err:#}");
            }
        });
@@ -926,10 +919,13 @@ impl ComputeNode {
        conf: Arc<tokio_postgres::Config>,
        concurrency: usize,
    ) -> Result<()> {
+        let rt = tokio::runtime::Builder::new_multi_thread()
+            .enable_all()
+            .build()?;
+
        info!("Applying config with max {} concurrency", concurrency);
        debug!("Config: {:?}", spec);

-        let rt = tokio::runtime::Handle::current();
        rt.block_on(async {
            // Proceed with post-startup configuration. Note, that order of operations is important.
            let client = Self::get_maintenance_client(&conf).await?;
@@ -1323,18 +1319,14 @@ impl ComputeNode {
        }

        // Run migrations separately to not hold up cold starts
-        tokio::spawn(async move {
-            let mut conf = conf.as_ref().clone();
+        thread::spawn(move || {
+            let conf = conf.as_ref().clone();
+            let mut conf = postgres::config::Config::from(conf);
            conf.application_name("compute_ctl:migrations");

-            match conf.connect(NoTls).await {
-                Ok((mut client, connection)) => {
-                    tokio::spawn(async move {
-                        if let Err(e) = connection.await {
-                            eprintln!("connection error: {}", e);
-                        }
-                    });
-                    if let Err(e) = handle_migrations(&mut client).await {
+            match conf.connect(NoTls) {
+                Ok(mut client) => {
+                    if let Err(e) = handle_migrations(&mut client) {
                        error!("Failed to run migrations: {}", e);
                    }
                }
@@ -1371,11 +1363,16 @@ impl ComputeNode {
        if let Some(ref pgbouncer_settings) = spec.pgbouncer_settings {
            info!("tuning pgbouncer");

-            // Spawn a background task to do the tuning,
+            let rt = tokio::runtime::Builder::new_current_thread()
+                .enable_all()
+                .build()
+                .expect("failed to create rt");
+
+            // Spawn a thread to do the tuning,
            // so that we don't block the main thread that starts Postgres.
            let pgbouncer_settings = pgbouncer_settings.clone();
-            tokio::spawn(async move {
-                let res = tune_pgbouncer(pgbouncer_settings).await;
+            let _handle = thread::spawn(move || {
+                let res = rt.block_on(tune_pgbouncer(pgbouncer_settings));
                if let Err(err) = res {
                    error!("error while tuning pgbouncer: {err:?}");
                }
@@ -1385,42 +1382,41 @@ impl ComputeNode {
        if let Some(ref local_proxy) = spec.local_proxy_config {
            info!("configuring local_proxy");

-            // Spawn a background task to do the configuration,
+            // Spawn a thread to do the configuration,
            // so that we don't block the main thread that starts Postgres.
            let local_proxy = local_proxy.clone();
-            tokio::spawn(async move {
+            let _handle = Some(thread::spawn(move || {
                if let Err(err) = local_proxy::configure(&local_proxy) {
                    error!("error while configuring local_proxy: {err:?}");
                }
-            });
+            }));
        }

        // Write new config
        let pgdata_path = Path::new(&self.pgdata);
        let postgresql_conf_path = pgdata_path.join("postgresql.conf");
-        config::write_postgres_conf(&postgresql_conf_path, &spec, self.internal_http_port)?;
+        config::write_postgres_conf(&postgresql_conf_path, &spec, self.http_port)?;

-        if !spec.skip_pg_catalog_updates {
-            let max_concurrent_connections = spec.reconfigure_concurrency;
-            // Temporarily reset max_cluster_size in config
-            // to avoid the possibility of hitting the limit, while we are reconfiguring:
-            // creating new extensions, roles, etc.
-            config::with_compute_ctl_tmp_override(pgdata_path, "neon.max_cluster_size=-1", || {
-                self.pg_reload_conf()?;
+        let max_concurrent_connections = spec.reconfigure_concurrency;

-                if spec.mode == ComputeMode::Primary {
-                    let mut conf = tokio_postgres::Config::from_str(self.connstr.as_str()).unwrap();
-                    conf.application_name("apply_config");
-                    let conf = Arc::new(conf);
+        // Temporarily reset max_cluster_size in config
+        // to avoid the possibility of hitting the limit, while we are reconfiguring:
+        // creating new extensions, roles, etc.
+        config::with_compute_ctl_tmp_override(pgdata_path, "neon.max_cluster_size=-1", || {
+            self.pg_reload_conf()?;

-                    let spec = Arc::new(spec.clone());
+            if spec.mode == ComputeMode::Primary {
+                let mut conf = tokio_postgres::Config::from_str(self.connstr.as_str()).unwrap();
+                conf.application_name("apply_config");
+                let conf = Arc::new(conf);

-                    self.apply_spec_sql(spec, conf, max_concurrent_connections)?;
-                }
+                let spec = Arc::new(spec.clone());

-                Ok(())
-            })?;
-        }
+                self.apply_spec_sql(spec, conf, max_concurrent_connections)?;
+            }
+
+            Ok(())
+        })?;

        self.pg_reload_conf()?;

@@ -1435,9 +1431,7 @@ impl ComputeNode {
    }

    #[instrument(skip_all)]
-    pub fn start_compute(
-        &self,
-    ) -> Result<(std::process::Child, tokio::task::JoinHandle<Result<()>>)> {
+    pub fn start_compute(&self) -> Result<(std::process::Child, std::thread::JoinHandle<()>)> {
        let compute_state = self.state.lock().unwrap().clone();
        let pspec = compute_state.pspec.as_ref().expect("spec must be set");
        info!(
@@ -1452,11 +1446,16 @@ impl ComputeNode {
        if let Some(pgbouncer_settings) = &pspec.spec.pgbouncer_settings {
            info!("tuning pgbouncer");

-            // Spawn a background task to do the tuning,
+            let rt = tokio::runtime::Builder::new_current_thread()
+                .enable_all()
+                .build()
+                .expect("failed to create rt");
+
+            // Spawn a thread to do the tuning,
            // so that we don't block the main thread that starts Postgres.
            let pgbouncer_settings = pgbouncer_settings.clone();
-            let _handle = tokio::spawn(async move {
-                let res = tune_pgbouncer(pgbouncer_settings).await;
+            let _handle = thread::spawn(move || {
+                let res = rt.block_on(tune_pgbouncer(pgbouncer_settings));
                if let Err(err) = res {
                    error!("error while tuning pgbouncer: {err:?}");
                }
@@ -1466,10 +1465,10 @@ impl ComputeNode {
        if let Some(local_proxy) = &pspec.spec.local_proxy_config {
            info!("configuring local_proxy");

-            // Spawn a background task to do the configuration,
+            // Spawn a thread to do the configuration,
            // so that we don't block the main thread that starts Postgres.
            let local_proxy = local_proxy.clone();
-            let _handle = tokio::spawn(async move {
+            let _handle = thread::spawn(move || {
                if let Err(err) = local_proxy::configure(&local_proxy) {
                    error!("error while configuring local_proxy: {err:?}");
                }
@@ -1488,8 +1487,7 @@ impl ComputeNode {
            extension_server::create_control_files(remote_extensions, &self.pgbin);

            let library_load_start_time = Utc::now();
-            let rt = tokio::runtime::Handle::current();
-            let remote_ext_metrics = rt.block_on(self.prepare_preload_libraries(&pspec.spec))?;
+            let remote_ext_metrics = self.prepare_preload_libraries(&pspec.spec)?;

            let library_load_time = Utc::now()
                .signed_duration_since(library_load_start_time)
@@ -1544,7 +1542,7 @@ impl ComputeNode {
            self.post_apply_config()?;

            let conf = self.get_conn_conf(None);
-            tokio::task::spawn_blocking(|| {
+            thread::spawn(move || {
                let res = get_installed_extensions(conf);
                match res {
                    Ok(extensions) => {
@@ -1893,6 +1891,7 @@ LIMIT 100",
        Ok(ext_version)
    }

+    #[tokio::main]
    pub async fn prepare_preload_libraries(
        &self,
        spec: &ComputeSpec,
--- a/compute_tools/src/configurator.rs
+++ b/compute_tools/src/configurator.rs
@@ -51,12 +51,9 @@ fn configurator_main_loop(compute: &Arc<ComputeNode>) {
 pub fn launch_configurator(compute: &Arc<ComputeNode>) -> thread::JoinHandle<()> {
    let compute = Arc::clone(compute);

-    let runtime = tokio::runtime::Handle::current();
-
    thread::Builder::new()
        .name("compute-configurator".into())
        .spawn(move || {
-            let _rt_guard = runtime.enter();
            configurator_main_loop(&compute);
            info!("configurator thread is exited");
        })
--- a/compute_tools/src/http/mod.rs
+++ b/compute_tools/src/http/mod.rs
@@ -4,9 +4,11 @@ use http::{header::CONTENT_TYPE, StatusCode};
 use serde::Serialize;
 use tracing::error;

+pub use server::launch_http_server;
+
 mod extract;
 mod routes;
-pub mod server;
+mod server;

 /// Convenience response builder for JSON responses
 struct JsonResponse;
--- a/compute_tools/src/http/routes/failpoints.rs
+++ b/compute_tools/src/http/routes/failpoints.rs
@@ -1,21 +1,7 @@
 use axum::response::{IntoResponse, Response};
 use http::StatusCode;
-use serde::{Deserialize, Serialize};
 use tracing::info;
-use utils::failpoint_support::apply_failpoint;
-
-pub type ConfigureFailpointsRequest = Vec<FailpointConfig>;
-
-/// Information for configuring a single fail point
-#[derive(Debug, Serialize, Deserialize)]
-pub struct FailpointConfig {
-    /// Name of the fail point
-    pub name: String,
-    /// List of actions to take, using the format described in `fail::cfg`
-    ///
-    /// We also support `actions = "exit"` to cause the fail point to immediately exit.
-    pub actions: String,
-}
+use utils::failpoint_support::{apply_failpoint, ConfigureFailpointsRequest};

 use crate::http::{extract::Json, JsonResponse};

--- a/compute_tools/src/http/server.rs
+++ b/compute_tools/src/http/server.rs
@@ -1,7 +1,7 @@
 use std::{
-    fmt::Display,
    net::{IpAddr, Ipv6Addr, SocketAddr},
    sync::Arc,
+    thread,
    time::Duration,
 };

@@ -26,65 +26,46 @@ use super::routes::{
 };
 use crate::compute::ComputeNode;

+async fn handle_404() -> Response {
+    StatusCode::NOT_FOUND.into_response()
+}
+
 const X_REQUEST_ID: &str = "x-request-id";

-/// `compute_ctl` has two servers: internal and external. The internal server
-/// binds to the loopback interface and handles communication from clients on
-/// the compute. The external server is what receives communication from the
-/// control plane, the metrics scraper, etc. We make the distinction because
-/// certain routes in `compute_ctl` only need to be exposed to local processes
-/// like Postgres via the neon extension and local_proxy.
-#[derive(Clone, Copy, Debug)]
-pub enum Server {
-    Internal(u16),
-    External(u16),
-}
+/// This middleware function allows compute_ctl to generate its own request ID
+/// if one isn't supplied. The control plane will always send one as a UUID. The
+/// neon Postgres extension on the other hand does not send one.
+async fn maybe_add_request_id_header(mut request: Request, next: Next) -> Response {
+    let headers = request.headers_mut();

-impl Display for Server {
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        match self {
-            Server::Internal(_) => f.write_str("internal"),
-            Server::External(_) => f.write_str("external"),
-        }
+    if headers.get(X_REQUEST_ID).is_none() {
+        headers.append(X_REQUEST_ID, Uuid::new_v4().to_string().parse().unwrap());
    }
+
+    next.run(request).await
 }

-impl From<Server> for Router<Arc<ComputeNode>> {
-    fn from(server: Server) -> Self {
-        let mut router = Router::<Arc<ComputeNode>>::new();
-
-        router = match server {
-            Server::Internal(_) => {
-                router = router
-                    .route(
-                        "/extension_server/{*filename}",
-                        post(extension_server::download_extension),
-                    )
-                    .route("/extensions", post(extensions::install_extension))
-                    .route("/grants", post(grants::add_grant));
-
-                // Add in any testing support
-                if cfg!(feature = "testing") {
-                    use super::routes::failpoints;
-
-                    router = router.route("/failpoints", post(failpoints::configure_failpoints));
-                }
-
-                router
-            }
-            Server::External(_) => router
-                .route("/check_writability", post(check_writability::is_writable))
-                .route("/configure", post(configure::configure))
-                .route("/database_schema", get(database_schema::get_schema_dump))
-                .route("/dbs_and_roles", get(dbs_and_roles::get_catalog_objects))
-                .route("/insights", get(insights::get_insights))
-                .route("/metrics", get(metrics::get_metrics))
-                .route("/metrics.json", get(metrics_json::get_metrics))
-                .route("/status", get(status::get_status))
-                .route("/terminate", post(terminate::terminate)),
-        };
-
-        router.fallback(Server::handle_404).method_not_allowed_fallback(Server::handle_405).layer(
+/// Run the HTTP server and wait on it forever.
+#[tokio::main]
+async fn serve(port: u16, compute: Arc<ComputeNode>) {
+    let mut app = Router::new()
+        .route("/check_writability", post(check_writability::is_writable))
+        .route("/configure", post(configure::configure))
+        .route("/database_schema", get(database_schema::get_schema_dump))
+        .route("/dbs_and_roles", get(dbs_and_roles::get_catalog_objects))
+        .route(
+            "/extension_server/{*filename}",
+            post(extension_server::download_extension),
+        )
+        .route("/extensions", post(extensions::install_extension))
+        .route("/grants", post(grants::add_grant))
+        .route("/insights", get(insights::get_insights))
+        .route("/metrics", get(metrics::get_metrics))
+        .route("/metrics.json", get(metrics_json::get_metrics))
+        .route("/status", get(status::get_status))
+        .route("/terminate", post(terminate::terminate))
+        .fallback(handle_404)
+        .layer(
            ServiceBuilder::new()
                // Add this middleware since we assume the request ID exists
                .layer(middleware::from_fn(maybe_add_request_id_header))
@@ -124,88 +105,45 @@ impl From<Server> for Router<Arc<ComputeNode>> {
                )
                .layer(PropagateRequestIdLayer::x_request_id()),
        )
-    }
-}
+        .with_state(compute);

-impl Server {
-    async fn handle_404() -> impl IntoResponse {
-        StatusCode::NOT_FOUND
+    // Add in any testing support
+    if cfg!(feature = "testing") {
+        use super::routes::failpoints;
+
+        app = app.route("/failpoints", post(failpoints::configure_failpoints))
    }

-    async fn handle_405() -> impl IntoResponse {
-        StatusCode::METHOD_NOT_ALLOWED
-    }
-
-    async fn listener(&self) -> Result<TcpListener> {
-        let addr = SocketAddr::new(self.ip(), self.port());
-        let listener = TcpListener::bind(&addr).await?;
-
-        Ok(listener)
-    }
-
-    fn ip(&self) -> IpAddr {
-        match self {
-            // TODO: Change this to Ipv6Addr::LOCALHOST when the GitHub runners
-            // allow binding to localhost
-            Server::Internal(_) => IpAddr::from(Ipv6Addr::UNSPECIFIED),
-            Server::External(_) => IpAddr::from(Ipv6Addr::UNSPECIFIED),
-        }
-    }
-
-    fn port(self) -> u16 {
-        match self {
-            Server::Internal(port) => port,
-            Server::External(port) => port,
-        }
-    }
-
-    async fn serve(self, compute: Arc<ComputeNode>) {
-        let listener = self.listener().await.unwrap_or_else(|e| {
-            // If we can't bind, the compute cannot operate correctly
-            panic!(
-                "failed to bind the compute_ctl {} HTTP server to {}: {}",
-                self,
-                SocketAddr::new(self.ip(), self.port()),
-                e
-            );
-        });
-
-        if tracing::enabled!(tracing::Level::INFO) {
-            let local_addr = match listener.local_addr() {
-                Ok(local_addr) => local_addr,
-                Err(_) => SocketAddr::new(self.ip(), self.port()),
-            };
-
-            info!(
-                "compute_ctl {} HTTP server listening at {}",
-                self, local_addr
+    // This usually binds to both IPv4 and IPv6 on Linux, see
+    // https://github.com/rust-lang/rust/pull/34440 for more information
+    let addr = SocketAddr::new(IpAddr::from(Ipv6Addr::UNSPECIFIED), port);
+    let listener = match TcpListener::bind(&addr).await {
+        Ok(listener) => listener,
+        Err(e) => {
+            error!(
+                "failed to bind the compute_ctl HTTP server to port {}: {}",
+                port, e
            );
+            return;
        }
+    };

-        let router = Router::from(self).with_state(compute);
-
-        if let Err(e) = axum::serve(listener, router).await {
-            error!("compute_ctl {} HTTP server error: {}", self, e);
-        }
+    if let Ok(local_addr) = listener.local_addr() {
+        info!("compute_ctl HTTP server listening on {}", local_addr);
+    } else {
+        info!("compute_ctl HTTP server listening on port {}", port);
    }

-    pub fn launch(self, compute: &Arc<ComputeNode>) {
-        let state = Arc::clone(compute);
-
-        info!("Launching the {} server", self);
-
-        tokio::spawn(self.serve(state));
+    if let Err(e) = axum::serve(listener, app).await {
+        error!("compute_ctl HTTP server error: {}", e);
    }
 }

-/// This middleware function allows compute_ctl to generate its own request ID
-/// if one isn't supplied. The control plane will always send one as a UUID. The
-/// neon Postgres extension on the other hand does not send one.
-async fn maybe_add_request_id_header(mut request: Request, next: Next) -> Response {
-    let headers = request.headers_mut();
-    if headers.get(X_REQUEST_ID).is_none() {
-        headers.append(X_REQUEST_ID, Uuid::new_v4().to_string().parse().unwrap());
-    }
+/// Launch a separate HTTP server thread and return its `JoinHandle`.
+pub fn launch_http_server(port: u16, state: &Arc<ComputeNode>) -> Result<thread::JoinHandle<()>> {
+    let state = Arc::clone(state);

-    next.run(request).await
+    Ok(thread::Builder::new()
+        .name("http-server".into())
+        .spawn(move || serve(port, state))?)
 }
--- a/compute_tools/src/logger.rs
+++ b/compute_tools/src/logger.rs
@@ -11,7 +11,7 @@ use tracing_subscriber::prelude::*;
 /// set `OTEL_EXPORTER_OTLP_ENDPOINT=http://jaeger:4318`. See
 /// `tracing-utils` package description.
 ///
-pub async fn init_tracing_and_logging(default_log_level: &str) -> anyhow::Result<()> {
+pub fn init_tracing_and_logging(default_log_level: &str) -> anyhow::Result<()> {
    // Initialize Logging
    let env_filter = tracing_subscriber::EnvFilter::try_from_default_env()
        .unwrap_or_else(|_| tracing_subscriber::EnvFilter::new(default_log_level));
@@ -22,7 +22,7 @@ pub async fn init_tracing_and_logging(default_log_level: &str) -> anyhow::Result
        .with_writer(std::io::stderr);

    // Initialize OpenTelemetry
-    let otlp_layer = tracing_utils::init_tracing("compute_ctl").await;
+    let otlp_layer = tracing_utils::init_tracing_without_runtime("compute_ctl");

    // Put it all together
    tracing_subscriber::registry()
--- a/compute_tools/src/migration.rs
+++ b/compute_tools/src/migration.rs
@@ -1,6 +1,6 @@
 use anyhow::{Context, Result};
 use fail::fail_point;
-use tokio_postgres::{Client, Transaction};
+use postgres::{Client, Transaction};
 use tracing::{error, info};

 use crate::metrics::DB_MIGRATION_FAILED;
@@ -21,11 +21,10 @@ impl<'m> MigrationRunner<'m> {
    }

    /// Get the current value neon_migration.migration_id
-    async fn get_migration_id(&mut self) -> Result<i64> {
+    fn get_migration_id(&mut self) -> Result<i64> {
        let row = self
            .client
-            .query_one("SELECT id FROM neon_migration.migration_id", &[])
-            .await?;
+            .query_one("SELECT id FROM neon_migration.migration_id", &[])?;

        Ok(row.get::<&str, i64>("id"))
    }
@@ -35,7 +34,7 @@ impl<'m> MigrationRunner<'m> {
    /// This function has a fail point called compute-migration, which can be
    /// used if you would like to fail the application of a series of migrations
    /// at some point.
-    async fn update_migration_id(txn: &mut Transaction<'_>, migration_id: i64) -> Result<()> {
+    fn update_migration_id(txn: &mut Transaction, migration_id: i64) -> Result<()> {
        // We use this fail point in order to check that failing in the
        // middle of applying a series of migrations fails in an expected
        // manner
@@ -60,38 +59,31 @@ impl<'m> MigrationRunner<'m> {
            "UPDATE neon_migration.migration_id SET id = $1",
            &[&migration_id],
        )
-        .await
        .with_context(|| format!("update neon_migration.migration_id to {migration_id}"))?;

        Ok(())
    }

    /// Prepare the migrations the target database for handling migrations
-    async fn prepare_database(&mut self) -> Result<()> {
+    fn prepare_database(&mut self) -> Result<()> {
        self.client
-            .simple_query("CREATE SCHEMA IF NOT EXISTS neon_migration")
-            .await?;
-        self.client.simple_query("CREATE TABLE IF NOT EXISTS neon_migration.migration_id (key INT NOT NULL PRIMARY KEY, id bigint NOT NULL DEFAULT 0)").await?;
+            .simple_query("CREATE SCHEMA IF NOT EXISTS neon_migration")?;
+        self.client.simple_query("CREATE TABLE IF NOT EXISTS neon_migration.migration_id (key INT NOT NULL PRIMARY KEY, id bigint NOT NULL DEFAULT 0)")?;
+        self.client.simple_query(
+            "INSERT INTO neon_migration.migration_id VALUES (0, 0) ON CONFLICT DO NOTHING",
+        )?;
        self.client
-            .simple_query(
-                "INSERT INTO neon_migration.migration_id VALUES (0, 0) ON CONFLICT DO NOTHING",
-            )
-            .await?;
+            .simple_query("ALTER SCHEMA neon_migration OWNER TO cloud_admin")?;
        self.client
-            .simple_query("ALTER SCHEMA neon_migration OWNER TO cloud_admin")
-            .await?;
-        self.client
-            .simple_query("REVOKE ALL ON SCHEMA neon_migration FROM PUBLIC")
-            .await?;
+            .simple_query("REVOKE ALL ON SCHEMA neon_migration FROM PUBLIC")?;

        Ok(())
    }

    /// Run an individual migration in a separate transaction block.
-    async fn run_migration(client: &mut Client, migration_id: i64, migration: &str) -> Result<()> {
+    fn run_migration(client: &mut Client, migration_id: i64, migration: &str) -> Result<()> {
        let mut txn = client
            .transaction()
-            .await
            .with_context(|| format!("begin transaction for migration {migration_id}"))?;

        if migration.starts_with("-- SKIP") {
@@ -100,38 +92,35 @@ impl<'m> MigrationRunner<'m> {
            // Even though we are skipping the migration, updating the
            // migration ID should help keep logic easy to understand when
            // trying to understand the state of a cluster.
-            Self::update_migration_id(&mut txn, migration_id).await?;
+            Self::update_migration_id(&mut txn, migration_id)?;
        } else {
            info!("Running migration id={}:\n{}\n", migration_id, migration);

            txn.simple_query(migration)
-                .await
                .with_context(|| format!("apply migration {migration_id}"))?;

-            Self::update_migration_id(&mut txn, migration_id).await?;
+            Self::update_migration_id(&mut txn, migration_id)?;
        }

        txn.commit()
-            .await
            .with_context(|| format!("commit transaction for migration {migration_id}"))?;

        Ok(())
    }

    /// Run the configured set of migrations
-    pub async fn run_migrations(mut self) -> Result<()> {
+    pub fn run_migrations(mut self) -> Result<()> {
        self.prepare_database()
-            .await
            .context("prepare database to handle migrations")?;

-        let mut current_migration = self.get_migration_id().await? as usize;
+        let mut current_migration = self.get_migration_id()? as usize;
        while current_migration < self.migrations.len() {
            // The index lags the migration ID by 1, so the current migration
            // ID is also the next index
            let migration_id = (current_migration + 1) as i64;
            let migration = self.migrations[current_migration];

-            match Self::run_migration(self.client, migration_id, migration).await {
+            match Self::run_migration(self.client, migration_id, migration) {
                Ok(_) => {
                    info!("Finished migration id={}", migration_id);
                }
--- a/compute_tools/src/pg_helpers.rs
+++ b/compute_tools/src/pg_helpers.rs
@@ -7,6 +7,7 @@ use std::os::unix::fs::PermissionsExt;
 use std::path::Path;
 use std::process::Child;
 use std::str::FromStr;
+use std::thread::JoinHandle;
 use std::time::{Duration, Instant};

 use anyhow::{bail, Result};
@@ -15,7 +16,6 @@ use ini::Ini;
 use notify::{RecursiveMode, Watcher};
 use postgres::config::Config;
 use tokio::io::AsyncBufReadExt;
-use tokio::task::JoinHandle;
 use tokio::time::timeout;
 use tokio_postgres;
 use tokio_postgres::NoTls;
@@ -477,13 +477,23 @@ pub async fn tune_pgbouncer(pgbouncer_config: HashMap<String, String>) -> Result
    Ok(())
 }

-/// Spawn a task that will read Postgres logs from `stderr`, join multiline logs
+/// Spawn a thread that will read Postgres logs from `stderr`, join multiline logs
 /// and send them to the logger. In the future we may also want to add context to
 /// these logs.
-pub fn handle_postgres_logs(stderr: std::process::ChildStderr) -> JoinHandle<Result<()>> {
-    tokio::spawn(async move {
-        let stderr = tokio::process::ChildStderr::from_std(stderr)?;
-        handle_postgres_logs_async(stderr).await
+pub fn handle_postgres_logs(stderr: std::process::ChildStderr) -> JoinHandle<()> {
+    std::thread::spawn(move || {
+        let runtime = tokio::runtime::Builder::new_current_thread()
+            .enable_all()
+            .build()
+            .expect("failed to build tokio runtime");
+
+        let res = runtime.block_on(async move {
+            let stderr = tokio::process::ChildStderr::from_std(stderr)?;
+            handle_postgres_logs_async(stderr).await
+        });
+        if let Err(e) = res {
+            tracing::error!("error while processing postgres logs: {}", e);
+        }
    })
 }

--- a/compute_tools/src/spec.rs
+++ b/compute_tools/src/spec.rs
@@ -1,8 +1,8 @@
 use anyhow::{anyhow, bail, Result};
+use postgres::Client;
 use reqwest::StatusCode;
 use std::fs::File;
 use std::path::Path;
-use tokio_postgres::Client;
 use tracing::{error, info, instrument, warn};

 use crate::config;
@@ -11,9 +11,7 @@ use crate::migration::MigrationRunner;
 use crate::params::PG_HBA_ALL_MD5;
 use crate::pg_helpers::*;

-use compute_api::responses::{
-    ComputeCtlConfig, ControlPlaneComputeStatus, ControlPlaneSpecResponse,
-};
+use compute_api::responses::{ControlPlaneComputeStatus, ControlPlaneSpecResponse};
 use compute_api::spec::ComputeSpec;

 // Do control plane request and return response if any. In case of error it
@@ -75,13 +73,14 @@ fn do_control_plane_request(
 pub fn get_spec_from_control_plane(
    base_uri: &str,
    compute_id: &str,
-) -> Result<(Option<ComputeSpec>, ComputeCtlConfig)> {
+) -> Result<Option<ComputeSpec>> {
    let cp_uri = format!("{base_uri}/compute/api/v2/computes/{compute_id}/spec");
    let jwt: String = match std::env::var("NEON_CONTROL_PLANE_TOKEN") {
        Ok(v) => v,
        Err(_) => "".to_string(),
    };
    let mut attempt = 1;
+    let mut spec: Result<Option<ComputeSpec>> = Ok(None);

    info!("getting spec from control plane: {}", cp_uri);

@@ -91,7 +90,7 @@ pub fn get_spec_from_control_plane(
    // - no spec for compute yet (Empty state) -> return Ok(None)
    // - got spec -> return Ok(Some(spec))
    while attempt < 4 {
-        let result = match do_control_plane_request(&cp_uri, &jwt) {
+        spec = match do_control_plane_request(&cp_uri, &jwt) {
            Ok(spec_resp) => {
                CPLANE_REQUESTS_TOTAL
                    .with_label_values(&[
@@ -100,10 +99,10 @@ pub fn get_spec_from_control_plane(
                    ])
                    .inc();
                match spec_resp.status {
-                    ControlPlaneComputeStatus::Empty => Ok((None, spec_resp.compute_ctl_config)),
+                    ControlPlaneComputeStatus::Empty => Ok(None),
                    ControlPlaneComputeStatus::Attached => {
                        if let Some(spec) = spec_resp.spec {
-                            Ok((Some(spec), spec_resp.compute_ctl_config))
+                            Ok(Some(spec))
                        } else {
                            bail!("compute is attached, but spec is empty")
                        }
@@ -122,10 +121,10 @@ pub fn get_spec_from_control_plane(
            }
        };

-        if let Err(e) = &result {
+        if let Err(e) = &spec {
            error!("attempt {} to get spec failed with: {}", attempt, e);
        } else {
-            return result;
+            return spec;
        }

        attempt += 1;
@@ -133,9 +132,7 @@ pub fn get_spec_from_control_plane(
    }

    // All attempts failed, return error.
-    Err(anyhow::anyhow!(
-        "Exhausted all attempts to retrieve the spec from the control plane"
-    ))
+    spec
 }

 /// Check `pg_hba.conf` and update if needed to allow external connections.
@@ -169,17 +166,17 @@ pub fn add_standby_signal(pgdata_path: &Path) -> Result<()> {
 }

 #[instrument(skip_all)]
-pub async fn handle_neon_extension_upgrade(client: &mut Client) -> Result<()> {
+pub fn handle_neon_extension_upgrade(client: &mut Client) -> Result<()> {
    info!("handle neon extension upgrade");
    let query = "ALTER EXTENSION neon UPDATE";
    info!("update neon extension version with query: {}", query);
-    client.simple_query(query).await?;
+    client.simple_query(query)?;

    Ok(())
 }

 #[instrument(skip_all)]
-pub async fn handle_migrations(client: &mut Client) -> Result<()> {
+pub fn handle_migrations(client: &mut Client) -> Result<()> {
    info!("handle migrations");

    // !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
@@ -209,9 +206,7 @@ pub async fn handle_migrations(client: &mut Client) -> Result<()> {
        ),
    ];

-    MigrationRunner::new(client, &migrations)
-        .run_migrations()
-        .await?;
+    MigrationRunner::new(client, &migrations).run_migrations()?;

    Ok(())
 }
@@ -219,7 +214,7 @@ pub async fn handle_migrations(client: &mut Client) -> Result<()> {
 /// Connect to the database as superuser and pre-create anon extension
 /// if it is present in shared_preload_libraries
 #[instrument(skip_all)]
-pub async fn handle_extension_anon(
+pub fn handle_extension_anon(
    spec: &ComputeSpec,
    db_owner: &str,
    db_client: &mut Client,
@@ -232,7 +227,7 @@ pub async fn handle_extension_anon(
            if !grants_only {
                // check if extension is already initialized using anon.is_initialized()
                let query = "SELECT anon.is_initialized()";
-                match db_client.query(query, &[]).await {
+                match db_client.query(query, &[]) {
                    Ok(rows) => {
                        if !rows.is_empty() {
                            let is_initialized: bool = rows[0].get(0);
@@ -254,7 +249,7 @@ pub async fn handle_extension_anon(
                // Users cannot create it themselves, because superuser is required.
                let mut query = "CREATE EXTENSION IF NOT EXISTS anon CASCADE";
                info!("creating anon extension with query: {}", query);
-                match db_client.query(query, &[]).await {
+                match db_client.query(query, &[]) {
                    Ok(_) => {}
                    Err(e) => {
                        error!("anon extension creation failed with error: {}", e);
@@ -264,7 +259,7 @@ pub async fn handle_extension_anon(

                // check that extension is installed
                query = "SELECT extname FROM pg_extension WHERE extname = 'anon'";
-                let rows = db_client.query(query, &[]).await?;
+                let rows = db_client.query(query, &[])?;
                if rows.is_empty() {
                    error!("anon extension is not installed");
                    return Ok(());
@@ -273,7 +268,7 @@ pub async fn handle_extension_anon(
                // Initialize anon extension
                // This also requires superuser privileges, so users cannot do it themselves.
                query = "SELECT anon.init()";
-                match db_client.query(query, &[]).await {
+                match db_client.query(query, &[]) {
                    Ok(_) => {}
                    Err(e) => {
                        error!("anon.init() failed with error: {}", e);
@@ -284,7 +279,7 @@ pub async fn handle_extension_anon(

            // check that extension is installed, if not bail early
            let query = "SELECT extname FROM pg_extension WHERE extname = 'anon'";
-            match db_client.query(query, &[]).await {
+            match db_client.query(query, &[]) {
                Ok(rows) => {
                    if rows.is_empty() {
                        error!("anon extension is not installed");
@@ -299,12 +294,12 @@ pub async fn handle_extension_anon(

            let query = format!("GRANT ALL ON SCHEMA anon TO {}", db_owner);
            info!("granting anon extension permissions with query: {}", query);
-            db_client.simple_query(&query).await?;
+            db_client.simple_query(&query)?;

            // Grant permissions to db_owner to use anon extension functions
            let query = format!("GRANT ALL ON ALL FUNCTIONS IN SCHEMA anon TO {}", db_owner);
            info!("granting anon extension permissions with query: {}", query);
-            db_client.simple_query(&query).await?;
+            db_client.simple_query(&query)?;

            // This is needed, because some functions are defined as SECURITY DEFINER.
            // In Postgres SECURITY DEFINER functions are executed with the privileges
@@ -319,16 +314,16 @@ pub async fn handle_extension_anon(
                where nsp.nspname = 'anon';", db_owner);

            info!("change anon extension functions owner to db owner");
-            db_client.simple_query(&query).await?;
+            db_client.simple_query(&query)?;

            //  affects views as well
            let query = format!("GRANT ALL ON ALL TABLES IN SCHEMA anon TO {}", db_owner);
            info!("granting anon extension permissions with query: {}", query);
-            db_client.simple_query(&query).await?;
+            db_client.simple_query(&query)?;

            let query = format!("GRANT ALL ON ALL SEQUENCES IN SCHEMA anon TO {}", db_owner);
            info!("granting anon extension permissions with query: {}", query);
-            db_client.simple_query(&query).await?;
+            db_client.simple_query(&query)?;
        }
    }

--- a/control_plane/Cargo.toml
+++ b/control_plane/Cargo.toml
@@ -33,7 +33,6 @@ postgres_backend.workspace = true
 safekeeper_api.workspace = true
 postgres_connection.workspace = true
 storage_broker.workspace = true
-http-utils.workspace = true
 utils.workspace = true
 whoami.workspace = true

--- a/control_plane/src/background_process.rs
+++ b/control_plane/src/background_process.rs
@@ -261,13 +261,7 @@ fn fill_rust_env_vars(cmd: &mut Command) -> &mut Command {
    let mut filled_cmd = cmd.env_clear().env("RUST_BACKTRACE", backtrace_setting);

    // Pass through these environment variables to the command
-    for var in [
-        "LLVM_PROFILE_FILE",
-        "FAILPOINTS",
-        "RUST_LOG",
-        "ASAN_OPTIONS",
-        "UBSAN_OPTIONS",
-    ] {
+    for var in ["LLVM_PROFILE_FILE", "FAILPOINTS", "RUST_LOG"] {
        if let Some(val) = std::env::var_os(var) {
            filled_cmd = filled_cmd.env(var, val);
        }
--- a/control_plane/src/bin/neon_local.rs
+++ b/control_plane/src/bin/neon_local.rs
@@ -552,10 +552,8 @@ struct EndpointCreateCmdArgs {
    lsn: Option<Lsn>,
    #[clap(long)]
    pg_port: Option<u16>,
-    #[clap(long, alias = "http-port")]
-    external_http_port: Option<u16>,
    #[clap(long)]
-    internal_http_port: Option<u16>,
+    http_port: Option<u16>,
    #[clap(long = "pageserver-id")]
    endpoint_pageserver_id: Option<NodeId>,

@@ -1355,8 +1353,7 @@ async fn handle_endpoint(subcmd: &EndpointCmd, env: &local_env::LocalEnv) -> Res
                tenant_id,
                timeline_id,
                args.pg_port,
-                args.external_http_port,
-                args.internal_http_port,
+                args.http_port,
                args.pg_version,
                mode,
                !args.update_catalog,
--- a/control_plane/src/endpoint.rs
+++ b/control_plane/src/endpoint.rs
@@ -37,8 +37,6 @@
 //! ```
 //!
 use std::collections::BTreeMap;
-use std::net::IpAddr;
-use std::net::Ipv4Addr;
 use std::net::SocketAddr;
 use std::net::TcpStream;
 use std::path::PathBuf;
@@ -48,8 +46,6 @@ use std::sync::Arc;
 use std::time::Duration;

 use anyhow::{anyhow, bail, Context, Result};
-use compute_api::requests::ConfigurationRequest;
-use compute_api::responses::ComputeCtlConfig;
 use compute_api::spec::Database;
 use compute_api::spec::PgIdent;
 use compute_api::spec::RemoteExtSpec;
@@ -77,8 +73,7 @@ pub struct EndpointConf {
    timeline_id: TimelineId,
    mode: ComputeMode,
    pg_port: u16,
-    external_http_port: u16,
-    internal_http_port: u16,
+    http_port: u16,
    pg_version: u32,
    skip_pg_catalog_updates: bool,
    drop_subscriptions_before_start: bool,
@@ -133,7 +128,7 @@ impl ComputeControlPlane {
        1 + self
            .endpoints
            .values()
-            .map(|ep| std::cmp::max(ep.pg_address.port(), ep.external_http_address.port()))
+            .map(|ep| std::cmp::max(ep.pg_address.port(), ep.http_address.port()))
            .max()
            .unwrap_or(self.base_port)
    }
@@ -145,27 +140,18 @@ impl ComputeControlPlane {
        tenant_id: TenantId,
        timeline_id: TimelineId,
        pg_port: Option<u16>,
-        external_http_port: Option<u16>,
-        internal_http_port: Option<u16>,
+        http_port: Option<u16>,
        pg_version: u32,
        mode: ComputeMode,
        skip_pg_catalog_updates: bool,
        drop_subscriptions_before_start: bool,
    ) -> Result<Arc<Endpoint>> {
        let pg_port = pg_port.unwrap_or_else(|| self.get_port());
-        let external_http_port = external_http_port.unwrap_or_else(|| self.get_port() + 1);
-        let internal_http_port = internal_http_port.unwrap_or_else(|| external_http_port + 1);
+        let http_port = http_port.unwrap_or_else(|| self.get_port() + 1);
        let ep = Arc::new(Endpoint {
            endpoint_id: endpoint_id.to_owned(),
-            pg_address: SocketAddr::new(IpAddr::from(Ipv4Addr::LOCALHOST), pg_port),
-            external_http_address: SocketAddr::new(
-                IpAddr::from(Ipv4Addr::UNSPECIFIED),
-                external_http_port,
-            ),
-            internal_http_address: SocketAddr::new(
-                IpAddr::from(Ipv4Addr::LOCALHOST),
-                internal_http_port,
-            ),
+            pg_address: SocketAddr::new("127.0.0.1".parse().unwrap(), pg_port),
+            http_address: SocketAddr::new("127.0.0.1".parse().unwrap(), http_port),
            env: self.env.clone(),
            timeline_id,
            mode,
@@ -190,8 +176,7 @@ impl ComputeControlPlane {
                tenant_id,
                timeline_id,
                mode,
-                external_http_port,
-                internal_http_port,
+                http_port,
                pg_port,
                pg_version,
                skip_pg_catalog_updates,
@@ -245,10 +230,9 @@ pub struct Endpoint {
    pub timeline_id: TimelineId,
    pub mode: ComputeMode,

-    // port and address of the Postgres server and `compute_ctl`'s HTTP APIs
+    // port and address of the Postgres server and `compute_ctl`'s HTTP API
    pub pg_address: SocketAddr,
-    pub external_http_address: SocketAddr,
-    pub internal_http_address: SocketAddr,
+    pub http_address: SocketAddr,

    // postgres major version in the format: 14, 15, etc.
    pg_version: u32,
@@ -303,15 +287,8 @@ impl Endpoint {
            serde_json::from_slice(&std::fs::read(entry.path().join("endpoint.json"))?)?;

        Ok(Endpoint {
-            pg_address: SocketAddr::new(IpAddr::from(Ipv4Addr::LOCALHOST), conf.pg_port),
-            external_http_address: SocketAddr::new(
-                IpAddr::from(Ipv4Addr::UNSPECIFIED),
-                conf.external_http_port,
-            ),
-            internal_http_address: SocketAddr::new(
-                IpAddr::from(Ipv4Addr::LOCALHOST),
-                conf.internal_http_port,
-            ),
+            pg_address: SocketAddr::new("127.0.0.1".parse().unwrap(), conf.pg_port),
+            http_address: SocketAddr::new("127.0.0.1".parse().unwrap(), conf.http_port),
            endpoint_id,
            env: env.clone(),
            timeline_id: conf.timeline_id,
@@ -673,51 +650,24 @@ impl Endpoint {
            println!("Also at '{}'", conn_str);
        }
        let mut cmd = Command::new(self.env.neon_distrib_dir.join("compute_ctl"));
-        //cmd.args([
-        //    "--external-http-port",
-        //    &self.external_http_address.port().to_string(),
-        //])
-        //.args([
-        //    "--internal-http-port",
-        //    &self.internal_http_address.port().to_string(),
-        //])
-        cmd.args([
-            "--http-port",
-            &self.external_http_address.port().to_string(),
-        ])
-        .args(["--pgdata", self.pgdata().to_str().unwrap()])
-        .args(["--connstr", &conn_str])
-        .args([
-            "--spec-path",
-            self.endpoint_path().join("spec.json").to_str().unwrap(),
-        ])
-        .args([
-            "--pgbin",
-            self.env
-                .pg_bin_dir(self.pg_version)?
-                .join("postgres")
-                .to_str()
-                .unwrap(),
-        ])
-        // TODO: It would be nice if we generated compute IDs with the same
-        // algorithm as the real control plane.
-        //
-        // TODO: Add this back when
-        // https://github.com/neondatabase/neon/pull/10747 is merged.
-        //
-        //.args([
-        //    "--compute-id",
-        //    &format!(
-        //        "compute-{}",
-        //        SystemTime::now()
-        //            .duration_since(UNIX_EPOCH)
-        //            .unwrap()
-        //            .as_secs()
-        //    ),
-        //])
-        .stdin(std::process::Stdio::null())
-        .stderr(logfile.try_clone()?)
-        .stdout(logfile);
+        cmd.args(["--http-port", &self.http_address.port().to_string()])
+            .args(["--pgdata", self.pgdata().to_str().unwrap()])
+            .args(["--connstr", &conn_str])
+            .args([
+                "--spec-path",
+                self.endpoint_path().join("spec.json").to_str().unwrap(),
+            ])
+            .args([
+                "--pgbin",
+                self.env
+                    .pg_bin_dir(self.pg_version)?
+                    .join("postgres")
+                    .to_str()
+                    .unwrap(),
+            ])
+            .stdin(std::process::Stdio::null())
+            .stderr(logfile.try_clone()?)
+            .stdout(logfile);

        if let Some(remote_ext_config) = remote_ext_config {
            cmd.args(["--remote-ext-config", remote_ext_config]);
@@ -804,8 +754,8 @@ impl Endpoint {
                reqwest::Method::GET,
                format!(
                    "http://{}:{}/status",
-                    self.external_http_address.ip(),
-                    self.external_http_address.port()
+                    self.http_address.ip(),
+                    self.http_address.port()
                ),
            )
            .send()
@@ -878,17 +828,14 @@ impl Endpoint {
        let response = client
            .post(format!(
                "http://{}:{}/configure",
-                self.external_http_address.ip(),
-                self.external_http_address.port()
+                self.http_address.ip(),
+                self.http_address.port()
            ))
            .header(CONTENT_TYPE.as_str(), "application/json")
-            .body(
-                serde_json::to_string(&ConfigurationRequest {
-                    spec,
-                    compute_ctl_config: ComputeCtlConfig::default(),
-                })
-                .unwrap(),
-            )
+            .body(format!(
+                "{{\"spec\":{}}}",
+                serde_json::to_string_pretty(&spec)?
+            ))
            .send()
            .await?;

--- a/control_plane/src/pageserver.rs
+++ b/control_plane/src/pageserver.rs
@@ -357,16 +357,6 @@ impl PageServerNode {
                .map(serde_json::from_str)
                .transpose()
                .context("Failed to parse 'compaction_algorithm' json")?,
-            compaction_l0_first: settings
-                .remove("compaction_l0_first")
-                .map(|x| x.parse::<bool>())
-                .transpose()
-                .context("Failed to parse 'compaction_l0_first' as a bool")?,
-            compaction_l0_semaphore: settings
-                .remove("compaction_l0_semaphore")
-                .map(|x| x.parse::<bool>())
-                .transpose()
-                .context("Failed to parse 'compaction_l0_semaphore' as a bool")?,
            l0_flush_delay_threshold: settings
                .remove("l0_flush_delay_threshold")
                .map(|x| x.parse::<usize>())
--- a/control_plane/src/safekeeper.rs
+++ b/control_plane/src/safekeeper.rs
@@ -17,10 +17,8 @@ use camino::Utf8PathBuf;
 use postgres_connection::PgConnectionConfig;
 use reqwest::{IntoUrl, Method};
 use thiserror::Error;
-
-use http_utils::error::HttpErrorBody;
 use utils::auth::{Claims, Scope};
-use utils::id::NodeId;
+use utils::{http::error::HttpErrorBody, id::NodeId};

 use crate::{
    background_process,
--- a/control_plane/src/storage_controller.rs
+++ b/control_plane/src/storage_controller.rs
@@ -221,17 +221,7 @@ impl StorageController {
            "-p",
            &format!("{}", postgres_port),
        ];
-        let pg_lib_dir = self.get_pg_lib_dir().await.unwrap();
-        let envs = [
-            ("LD_LIBRARY_PATH".to_owned(), pg_lib_dir.to_string()),
-            ("DYLD_LIBRARY_PATH".to_owned(), pg_lib_dir.to_string()),
-        ];
-        let exitcode = Command::new(bin_path)
-            .args(args)
-            .envs(envs)
-            .spawn()?
-            .wait()
-            .await?;
+        let exitcode = Command::new(bin_path).args(args).spawn()?.wait().await?;

        Ok(exitcode.success())
    }
@@ -252,11 +242,6 @@ impl StorageController {

        let pg_bin_dir = self.get_pg_bin_dir().await?;
        let createdb_path = pg_bin_dir.join("createdb");
-        let pg_lib_dir = self.get_pg_lib_dir().await.unwrap();
-        let envs = [
-            ("LD_LIBRARY_PATH".to_owned(), pg_lib_dir.to_string()),
-            ("DYLD_LIBRARY_PATH".to_owned(), pg_lib_dir.to_string()),
-        ];
        let output = Command::new(&createdb_path)
            .args([
                "-h",
@@ -269,7 +254,6 @@ impl StorageController {
                &username(),
                DB_NAME,
            ])
-            .envs(envs)
            .output()
            .await
            .expect("Failed to spawn createdb");
@@ -838,10 +822,7 @@ impl StorageController {
        self.dispatch(
            Method::PUT,
            format!("control/v1/tenant/{tenant_shard_id}/migrate"),
-            Some(TenantShardMigrateRequest {
-                node_id,
-                migration_config: None,
-            }),
+            Some(TenantShardMigrateRequest { node_id }),
        )
        .await
    }
--- a/control_plane/storcon_cli/src/main.rs
+++ b/control_plane/storcon_cli/src/main.rs
@@ -609,10 +609,7 @@ async fn main() -> anyhow::Result<()> {
            tenant_shard_id,
            node,
        } => {
-            let req = TenantShardMigrateRequest {
-                node_id: node,
-                migration_config: None,
-            };
+            let req = TenantShardMigrateRequest { node_id: node };

            storcon_client
                .dispatch::<TenantShardMigrateRequest, TenantShardMigrateResponse>(
@@ -626,10 +623,7 @@ async fn main() -> anyhow::Result<()> {
            tenant_shard_id,
            node,
        } => {
-            let req = TenantShardMigrateRequest {
-                node_id: node,
-                migration_config: None,
-            };
+            let req = TenantShardMigrateRequest { node_id: node };

            storcon_client
                .dispatch::<TenantShardMigrateRequest, TenantShardMigrateResponse>(
@@ -1088,10 +1082,7 @@ async fn main() -> anyhow::Result<()> {
                            .dispatch::<TenantShardMigrateRequest, TenantShardMigrateResponse>(
                                Method::PUT,
                                format!("control/v1/tenant/{}/migrate", mv.tenant_shard_id),
-                                Some(TenantShardMigrateRequest {
-                                    node_id: mv.to,
-                                    migration_config: None,
-                                }),
+                                Some(TenantShardMigrateRequest { node_id: mv.to }),
                            )
                            .await
                            .map_err(|e| (mv.tenant_shard_id, mv.from, mv.to, e))
--- a/deny.toml
+++ b/deny.toml
@@ -32,7 +32,6 @@ reason = "the marvin attack only affects private key decryption, not public key
 # https://embarkstudios.github.io/cargo-deny/checks/licenses/cfg.html
 [licenses]
 allow = [
-    "0BSD",
    "Apache-2.0",
    "BSD-2-Clause",
    "BSD-3-Clause",
--- a/docker-compose/docker_compose_test.sh
+++ b/docker-compose/docker_compose_test.sh
@@ -52,7 +52,6 @@ for pg_version in ${TEST_VERSION_ONLY-14 15 16 17}; do

    if [ $pg_version -ge 16 ]; then
        docker cp ext-src $TEST_CONTAINER_NAME:/
-        docker exec $TEST_CONTAINER_NAME bash -c "apt update && apt install -y libtap-parser-sourcehandler-pgtap-perl"
        # This is required for the pg_hint_plan test, to prevent flaky log message causing the test to fail
        # It cannot be moved to Dockerfile now because the database directory is created after the start of the container
        echo Adding dummy config
@@ -71,7 +70,7 @@ for pg_version in ${TEST_VERSION_ONLY-14 15 16 17}; do
        cat ../compute/patches/contrib_pg${pg_version}.patch | docker exec -i $TEST_CONTAINER_NAME bash -c "(cd /postgres && patch -p1)"
        # We are running tests now
        rm -f testout.txt testout_contrib.txt
-        docker exec -e USE_PGXS=1 -e SKIP=timescaledb-src,rdkit-src,postgis-src,pgx_ulid-src,pg_tiktoken-src,pg_jsonschema-src,kq_imcx-src,wal2json_2_5-src \
+        docker exec -e USE_PGXS=1 -e SKIP=timescaledb-src,rdkit-src,postgis-src,pgx_ulid-src,pgtap-src,pg_tiktoken-src,pg_jsonschema-src,kq_imcx-src,wal2json_2_5-src \
        $TEST_CONTAINER_NAME /run-tests.sh /ext-src | tee testout.txt && EXT_SUCCESS=1 || EXT_SUCCESS=0
        docker exec -e SKIP=start-scripts,postgres_fdw,ltree_plpython,jsonb_plpython,jsonb_plperl,hstore_plpython,hstore_plperl,dblink,bool_plperl \
        $TEST_CONTAINER_NAME /run-tests.sh /postgres/contrib | tee testout_contrib.txt && CONTRIB_SUCCESS=1 || CONTRIB_SUCCESS=0
--- a/docker-compose/ext-src/pgjwt-src/neon-test.sh
+++ b/docker-compose/ext-src/pgjwt-src/neon-test.sh
@@ -1,4 +0,0 @@
-#!/bin/bash
-set -ex
-cd "$(dirname "${0}")"
-pg_prove test.sql
--- a/docker-compose/ext-src/pgjwt-src/test-upgrade.patch
+++ b/docker-compose/ext-src/pgjwt-src/test-upgrade.patch
@@ -1,15 +0,0 @@
-diff --git a/test.sql b/test.sql
-index d7a0ca8..f15bc76 100644
--- a/test.sql
-+++ b/test.sql
-@@ -9,9 +9,7 @@
- \set ON_ERROR_STOP true
- \set QUIET 1
- 
-CREATE EXTENSION pgcrypto;
-CREATE EXTENSION pgtap;
-CREATE EXTENSION pgjwt;
-+CREATE EXTENSION IF NOT EXISTS pgtap;
- 
- BEGIN;
- SELECT plan(23);
--- a/docker-compose/ext-src/pgjwt-src/test-upgrade.sh
+++ b/docker-compose/ext-src/pgjwt-src/test-upgrade.sh
@@ -1,5 +0,0 @@
-#!/bin/sh
-set -ex
-cd "$(dirname ${0})"
-patch -p1 <test-upgrade.patch
-pg_prove -d contrib_regression test.sql
--- a/docker-compose/ext-src/pgtap-src/test-upgrade.patch
+++ b/docker-compose/ext-src/pgtap-src/test-upgrade.patch
@@ -1,15 +0,0 @@
-diff --git a/test/schedule/create.sql b/test/schedule/create.sql
-index ba355ed..7e250f5 100644
--- a/test/schedule/create.sql
-+++ b/test/schedule/create.sql
-@@ -1,3 +1,2 @@
- \unset ECHO
- \i test/psql.sql
-CREATE EXTENSION pgtap;
-diff --git a/test/schedule/main.sch b/test/schedule/main.sch
-index a8a5fbc..0463fc4 100644
--- a/test/schedule/main.sch
-+++ b/test/schedule/main.sch
-@@ -1,2 +1 @@
-test: build
- test: create
--- a/docker-compose/ext-src/pgtap-src/test-upgrade.sh
+++ b/docker-compose/ext-src/pgtap-src/test-upgrade.sh
@@ -1,6 +0,0 @@
-#!/bin/sh
-set -ex
-cd "$(dirname ${0})"
-patch -p1 <test-upgrade.patch
-PG_REGRESS=$(dirname "$(pg_config --pgxs)")/../test/regress/pg_regress
-${PG_REGRESS} --inputdir=./ --bindir='/usr/local/pgsql/bin'    --inputdir=test --max-connections=86 --schedule test/schedule/main.sch   --schedule test/build/run.sch --dbname contrib_regression --use-existing
--- a/docker-compose/test_extensions_upgrade.sh
+++ b/docker-compose/test_extensions_upgrade.sh
@@ -24,7 +24,7 @@ function wait_for_ready {
 }
 function create_extensions() {
  for ext in ${1}; do
-    docker compose exec neon-test-extensions psql -X -v ON_ERROR_STOP=1 -d contrib_regression -c "CREATE EXTENSION IF NOT EXISTS ${ext} CASCADE"
+    docker compose exec neon-test-extensions psql -X -v ON_ERROR_STOP=1 -d contrib_regression -c "CREATE EXTENSION IF NOT EXISTS ${ext}"
  done
 }
 EXTENSIONS='[
@@ -40,9 +40,7 @@ EXTENSIONS='[
 {"extname": "pg_uuidv7", "extdir": "pg_uuidv7-src"},
 {"extname": "roaringbitmap", "extdir": "pg_roaringbitmap-src"},
 {"extname": "semver", "extdir": "pg_semver-src"},
-{"extname": "pg_ivm", "extdir": "pg_ivm-src"},
-{"extname": "pgjwt", "extdir": "pgjwt-src"},
-{"extname": "pgtap", "extdir": "pgtap-src"}
+{"extname": "pg_ivm", "extdir": "pg_ivm-src"}
 ]'
 EXTNAMES=$(echo ${EXTENSIONS} | jq -r '.[].extname' | paste -sd ' ' -)
 TAG=${NEWTAG} docker compose --profile test-extensions up --quiet-pull --build -d
--- a/docs/rfcs/035-safekeeper-dynamic-membership-change.md
+++ b/docs/rfcs/035-safekeeper-dynamic-membership-change.md
@@ -285,10 +285,10 @@ To summarize, list of cplane changes:

 ### storage_controller implementation

-If desired, we may continue using current 'load everything on startup and keep
-in memory' approach: single timeline shouldn't take more than 100 bytes (it's 16
-byte tenant_id, 16 byte timeline_id, int generation, vec of ~3 safekeeper ids
-plus some flags), so 10^6 of timelines shouldn't take more than 100MB.
+Current 'load everything on startup and keep in memory' easy design is fine.
+Single timeline shouldn't take more than 100 bytes (it's 16 byte tenant_id, 16
+byte timeline_id, int generation, vec of ~3 safekeeper ids plus some flags), so
+10^6 of timelines shouldn't take more than 100MB.

 Similar to pageserver attachment Intents storage_controller would have in-memory
 `MigrationRequest` (or its absense) for each timeline and pool of tasks trying
@@ -296,7 +296,7 @@ to make these request reality; this ensures one instance of storage_controller
 won't do several migrations on the same timeline concurrently. In the first
 version it is simpler to have more manual control and no retries, i.e. migration
 failure removes the request. Later we can build retries and automatic
-scheduling/migration around. `MigrationRequest` is
+scheduling/migration. `MigrationRequest` is
 ```
 enum MigrationRequest {
    To(Vec<NodeId>),
@@ -313,9 +313,9 @@ similarly, in the first version it is ok to trigger it manually).
 #### Schema

 `safekeepers` table mirroring current `nodes` should be added, except that for
-`scheduling_policy`: it is enough to have at least in the beginning only 3
-fields: 1) `active` 2) `paused` (initially means only not assign new tlis there
-3) `decomissioned` (node is removed).
+`scheduling_policy` field (seems like `status` is a better name for it): it is enough
+to have at least in the beginning only 3 fields: 1) `active` 2) `offline` 3)
+`decomissioned`.

 `timelines` table:
 ```
@@ -324,24 +324,18 @@ table! {
    timelines (tenant_id, timeline_id) {
        timeline_id -> Varchar,
        tenant_id -> Varchar,
-        start_lsn -> pg_lsn,
        generation -> Int4,
        sk_set -> Array<Int4>, // list of safekeeper ids
-        new_sk_set -> Nullable<Array<Int8>>, // list of safekeeper ids, null if not joint conf
+        new_sk_set -> Nullable<Array<Int4>>, // list of safekeeper ids, null if not joint conf
        cplane_notified_generation -> Int4,
-        deleted_at -> Nullable<Timestamptz>,
    }
 }
 ```

-`start_lsn` is needed to create timeline on safekeepers properly, see below. We
-might also want to add ancestor_timeline_id to preserve the hierarchy, but for
-this RFC it is not needed.
-
 #### API

 Node management is similar to pageserver:
-1) POST `/control/v1/safekeepers` inserts safekeeper.
+1) POST `/control/v1/safekeepers` upserts safekeeper.
 2) GET `/control/v1/safekeepers` lists safekeepers.
 3) GET `/control/v1/safekeepers/:node_id` gets safekeeper.
 4) PUT `/control/v1/safekepers/:node_id/status` changes status to e.g.
@@ -351,15 +345,25 @@ Node management is similar to pageserver:
 Safekeeper deploy scripts should register safekeeper at storage_contorller as
 they currently do with cplane, under the same id.

-Timeline creation/deletion will work through already existing POST and DELETE
-`tenant/:tenant_id/timeline`. Cplane is expected to retry both until they
-succeed. See next section on the implementation details.
+Timeline creation/deletion: already existing POST `tenant/:tenant_id/timeline`
+would 1) choose initial set of safekeepers; 2) write to the db initial
+`Configuration` with `INSERT ON CONFLICT DO NOTHING` returning existing row in
+case of conflict; 3) create timeline on the majority of safekeepers (already
+created is ok).

-We don't want to block timeline creation/deletion when one safekeeper is down.
-Currently this is crutched by compute implicitly creating timeline on any
-safekeeper it is connected to. This creates ugly timeline state on safekeeper
-when timeline is created, but start LSN is not defined yet. Next section
-describes dealing with this.
+We don't want to block timeline creation when one safekeeper is down. Currently
+this is solved by compute implicitly creating timeline on any safekeeper it is
+connected to. This creates ugly timeline state on safekeeper when timeline is
+created, but start LSN is not defined yet. It would be nice to remove this; to
+do that, controller can in the background retry to create timeline on
+safekeeper(s) which missed that during initial creation call. It can do that
+through `pull_timeline` from majority so it doesn't need to remember
+`parent_lsn` in its db.
+
+Timeline deletion removes the row from the db and forwards deletion to the
+current configuration members. Without additional actions deletions might leak,
+see below on this; initially let's ignore these, reporting to cplane success if
+at least one safekeeper deleted the timeline (this will remove s3 data).

 Tenant deletion repeats timeline deletion for all timelines.

@@ -391,6 +395,26 @@ Similar call should be added for the tenant.
 It would be great to have some way of subscribing to the results (apart from
 looking at logs/metrics).

+Migration is executed as described above. One subtlety is that (local) deletion on
+source safekeeper might fail, which is not a problem if we are going to
+decomission the node but leaves garbage otherwise. I'd propose in the first version
+1) Don't attempt deletion at all if node status is `offline`.
+2) If it failed, just issue warning.
+And add PUT `/control/v1/safekeepers/:node_id/scrub` endpoint which would find and
+remove garbage timelines for manual use. It will 1) list all timelines on the
+safekeeper 2) compare each one against configuration storage: if timeline
+doesn't exist at all (had been deleted), it can be deleted. Otherwise, it can
+be deleted under generation number if node is not member of current generation.
+
+Automating this is untrivial; we'd need to register all potential missing
+deletions <tenant_id, timeline_id, generation, node_id> in the same transaction
+which switches configurations. Similarly when timeline is fully deleted to
+prevent cplane operation from blocking when some safekeeper is not available
+deletion should be also registered.
+
+One more task pool should infinitely retry notifying control plane about changed
+safekeeper sets.
+
 3) GET `/control/v1/tenant/:tenant_id/timeline/:timeline_id/` should return
   current in memory state of the timeline and pending `MigrationRequest`,
   if any.
@@ -399,153 +423,12 @@ looking at logs/metrics).
   migration by switching configuration from the joint to the one with (previous) `sk_set` under CAS
   (incrementing generation as always).

-#### API implementation and reconciliation
-
-For timeline creation/deletion we want to preserve the basic assumption that
-unreachable minority (1 sk of 3) doesn't block their completion, but eventually
-we want to finish creation/deletion on nodes which missed it (unless they are
-removed). Similarly for migration; it may and should finish even though excluded
-members missed their exclusion. And of course e.g. such pending exclusion on
-node C after migration ABC -> ABD must not prevent next migration ABD -> ABE. As
-another example, if some node missed timeline creation it clearly must not block
-migration from it. Hence it is natural to have per safekeeper background
-reconciler which retries these ops until they succeed. There are 3 possible
-operation types, and the type is defined by timeline state (membership
-configuration and whether it is deleted) and safekeeper id: we may need to
-create timeline on sk (node added), locally delete it (node excluded, somewhat
-similar to detach) or globally delete it (timeline is deleted).
-
-Next, on storage controller restart in principle these pending operations can be
-figured out by comparing safekeepers state against storcon state. But it seems
-better to me to materialize them in the database; it is not expensive, avoids
-these startup scans which themselves can fail etc and makes it very easy to see
-outstanding work directly at the source of truth -- the db. So we can add table
-`safekeeper_timeline_pending_ops`
-```
-table! {
-    // timeline_id, sk_id is primary key
-    safekeeper_timeline_pending_ops (sk_id, tenant_id, timeline_id) {
-        sk_id -> int8,
-        tenant_id -> Varchar,
-        timeline_id -> Varchar,
-        generation -> Int4,
-        op_type -> Varchar,
-    }
-}
-```
-
-`op_type` can be `include` (seed from peers and ensure generation is up to
-date), `exclude` (remove locally) and `delete`. Field is actually not strictly
-needed as it can be computed from current configuration, but gives more explicit
-observability.
-
-`generation` is necessary there because after op is done reconciler must remove
-it and not remove another row with higher gen which in theory might appear.
-
-Any insert of row should overwrite (remove) all rows with the same sk and
-timeline id but lower `generation` as next op makes previous obsolete. Insertion
-of `op_type` `delete` overwrites all rows.
-
-About `exclude`: rather than adding explicit safekeeper http endpoint, it is
-reasonable to reuse membership switch endpoint: if safekeeper is not member
-of the configuration it locally removes the timeline on the switch. In this case
-404 should also be considered an 'ok' answer by the caller.
-
-So, main loop of per sk reconcile reads `safekeeper_timeline_pending_ops`
-joined with timeline configuration to get current conf (with generation `n`)
-for the safekeeper and does the jobs, infinitely retrying failures:
-1) If node is member (`include`):
-  - Check if timeline exists on it, if not, call pull_timeline on it from 
-     other members
-  - Call switch configuration to the current
-2) If node is not member (`exclude`):
-  - Call switch configuration to the current, 404 is ok.
-3) If timeline is deleted (`delete`), call delete.
-
-In cases 1 and 2 remove `safekeeper_timeline_pending_ops` for the sk and 
-timeline with generation <= `n` if `op_type` is not `delete`.
-In case 3 also remove `safekeeper_timeline_pending_ops` 
-entry + remove `timelines` entry if there is nothing left  in `safekeeper_timeline_pending_ops` for the timeline.
-
-Let's consider in details how APIs can be implemented from this angle.
-
-Timeline creation. It is assumed that cplane retries it until success, so all
-actions must be idempotent. Now, a tricky point here is timeline start LSN. For
-the initial (tenant creation) call cplane doesn't know it. However, setting
-start_lsn on safekeepers during creation is a good thing -- it provides a
-guarantee that walproposer can always find a common point in WAL histories of
-safekeeper and its own, and so absense of it would be a clear sign of
-corruption. The following sequence works:
-1) Create timeline (or observe that it exists) on pageserver,
-   figuring out last_record_lsn in response.
-2) Choose safekeepers and insert (ON CONFLICT DO NOTHING) timeline row into the
-   db. Note that last_record_lsn returned on the previous step is movable as it
-   changes once ingestion starts, insert must not overwrite it (as well as other
-   fields like membership conf). On the contrary, start_lsn used in the next
-   step must be set to the value in the db. cplane_notified_generation can be set
-   to 1 (initial generation) in insert to avoid notifying cplane about initial 
-   conf as cplane will receive it in timeline creation request anyway.
-3) Issue timeline creation calls to at least majority of safekeepers. Using
-   majority here is not necessary but handy because it guarantees that any live
-   majority will have at least one sk with created timeline and so
-   reconciliation task can use pull_timeline shared with migration instead of
-   create timeline special init case. OFC if timeline is already exists call is
-   ignored.
-4) For minority of safekeepers which could have missed creation insert
-   entries to `safekeeper_timeline_pending_ops`. We won't miss this insertion 
-   because response to cplane is sent only after it has happened, and cplane 
-   retries the call until 200 response.
-
-   There is a small question how request handler (timeline creation in this
-   case) would interact with per sk reconciler. As always I prefer to do the
-   simplest possible thing and here it seems to be just waking it up so it
-   re-reads the db for work to do. Passing work in memory is faster, but
-   that shouldn't matter, and path to scan db for work will exist anyway, 
-   simpler to reuse it.
-
-For pg version / wal segment size: while we may persist them in `timelines`
-table, it is not necessary as initial creation at step 3 can take them from
-pageserver or cplane creation call and later pull_timeline will carry them
-around.
-
-Timeline migration.
-1) CAS to the db to create joint conf, and in the same transaction create
-   `safekeeper_timeline_pending_ops` `include` entries to initialize new members
-   as well as deliver this conf to current ones; poke per sk reconcilers to work
-   on it. Also any conf change should also poke cplane notifier task(s).
-2) Once it becomes possible per alg description above, get out of joint conf
-   with another CAS. Task should get wakeups from per sk reconcilers because 
-   conf switch is required for advancement; however retries should be sleep
-   based as well as LSN advancement might be needed, though in happy path 
-   it isn't. To see whether further transition is possible on wakup migration
-   executor polls safekeepers per the algorithm. CAS creating new conf with only
-   new members should again insert entries to `safekeeper_timeline_pending_ops`
-   to switch them there, as well as `exclude` rows to remove timeline from 
-   old members.
-
-Timeline deletion: just set `deleted_at` on the timeline row and insert
-`safekeeper_timeline_pending_ops` entries in the same xact, the rest is done by
-per sk reconcilers.
-
-When node is removed (set to `decomissioned`), `safekeeper_timeline_pending_ops`
-for it must be cleared in the same transaction.
-
-One more task pool should infinitely retry notifying control plane about changed
-safekeeper sets (trying making `cplane_notified_generation` equal `generation`).
-
 #### Dealing with multiple instances of storage_controller

 Operations described above executed concurrently might create some errors but do
 not prevent progress, so while we normally don't want to run multiple instances
 of storage_controller it is fine to have it temporarily, e.g. during redeploy.

-To harden against some controller instance creating some work in
-`safekeeper_timeline_pending_ops` and then disappearing without anyone pickup up
-the job per sk reconcilers apart from explicit wakups should scan for work
-periodically. It is possible to remove that though if all db updates are
-protected with leadership token/term -- then such scans are needed only after
-leadership is acquired.
-
 Any interactions with db update in-memory controller state, e.g. if migration
 request failed because different one is in progress, controller remembers that
 and tries to finish it.
@@ -662,7 +545,7 @@ Aurora does this but similarly I don't think this is needed.

 We should use Compute <-> safekeeper protocol change to include other (long
 yearned) modifications:
- send data in network order without putting whole structs to be arch independent
+- send data in network order to make arm work.
 - remove term_start_lsn from AppendRequest
 - add horizon to TermHistory
 - add to ProposerGreeting number of connection from this wp to sk
--- a/libs/compute_api/Cargo.toml
+++ b/libs/compute_api/Cargo.toml
@@ -7,7 +7,6 @@ license.workspace = true
 [dependencies]
 anyhow.workspace = true
 chrono.workspace = true
-jsonwebtoken.workspace = true
 serde.workspace = true
 serde_json.workspace = true
 regex.workspace = true
--- a/libs/compute_api/src/requests.rs
+++ b/libs/compute_api/src/requests.rs
@@ -1,20 +1,18 @@
 //! Structs representing the JSON formats used in the compute_ctl's HTTP API.
 use crate::{
    privilege::Privilege,
-    responses::ComputeCtlConfig,
    spec::{ComputeSpec, ExtVersion, PgIdent},
 };
-use serde::{Deserialize, Serialize};
+use serde::Deserialize;

 /// Request of the /configure API
 ///
 /// We now pass only `spec` in the configuration request, but later we can
 /// extend it and something like `restart: bool` or something else. So put
 /// `spec` into a struct initially to be more flexible in the future.
-#[derive(Debug, Deserialize, Serialize)]
+#[derive(Deserialize, Debug)]
 pub struct ConfigurationRequest {
    pub spec: ComputeSpec,
-    pub compute_ctl_config: ComputeCtlConfig,
 }

 #[derive(Deserialize, Debug)]
--- a/libs/compute_api/src/responses.rs
+++ b/libs/compute_api/src/responses.rs
@@ -3,7 +3,6 @@
 use std::fmt::Display;

 use chrono::{DateTime, Utc};
-use jsonwebtoken::jwk::JwkSet;
 use serde::{Deserialize, Serialize, Serializer};

 use crate::{
@@ -136,27 +135,13 @@ pub struct CatalogObjects {
    pub databases: Vec<Database>,
 }

-#[derive(Debug, Deserialize, Serialize)]
-pub struct ComputeCtlConfig {
-    pub jwks: JwkSet,
-}
-
-impl Default for ComputeCtlConfig {
-    fn default() -> Self {
-        Self {
-            jwks: JwkSet {
-                keys: Vec::default(),
-            },
-        }
-    }
-}
-
 /// Response of the `/computes/{compute_id}/spec` control-plane API.
+/// This is not actually a compute API response, so consider moving
+/// to a different place.
 #[derive(Deserialize, Debug)]
 pub struct ControlPlaneSpecResponse {
    pub spec: Option<ComputeSpec>,
    pub status: ControlPlaneComputeStatus,
-    pub compute_ctl_config: ComputeCtlConfig,
 }

 #[derive(Deserialize, Clone, Copy, Debug, PartialEq, Eq)]
--- a/libs/compute_api/src/spec.rs
+++ b/libs/compute_api/src/spec.rs
@@ -204,16 +204,14 @@ impl RemoteExtSpec {

        // Check if extension is present in public or custom.
        // If not, then it is not allowed to be used by this compute.
-        if !self
-            .public_extensions
-            .as_ref()
-            .is_some_and(|exts| exts.iter().any(|e| e == real_ext_name))
-            && !self
-                .custom_extensions
-                .as_ref()
-                .is_some_and(|exts| exts.iter().any(|e| e == real_ext_name))
-        {
-            return Err(anyhow::anyhow!("extension {} is not found", real_ext_name));
+        if let Some(public_extensions) = &self.public_extensions {
+            if !public_extensions.contains(&real_ext_name.to_string()) {
+                if let Some(custom_extensions) = &self.custom_extensions {
+                    if !custom_extensions.contains(&real_ext_name.to_string()) {
+                        return Err(anyhow::anyhow!("extension {} is not found", real_ext_name));
+                    }
+                }
+            }
        }

        match self.extension_data.get(real_ext_name) {
@@ -342,102 +340,6 @@ mod tests {
    use super::*;
    use std::fs::File;

-    #[test]
-    fn allow_installing_remote_extensions() {
-        let rspec: RemoteExtSpec = serde_json::from_value(serde_json::json!({
-            "public_extensions": null,
-            "custom_extensions": null,
-            "library_index": {},
-            "extension_data": {},
-        }))
-        .unwrap();
-
-        rspec
-            .get_ext("ext", false, "latest", "v17")
-            .expect_err("Extension should not be found");
-
-        let rspec: RemoteExtSpec = serde_json::from_value(serde_json::json!({
-            "public_extensions": [],
-            "custom_extensions": null,
-            "library_index": {},
-            "extension_data": {},
-        }))
-        .unwrap();
-
-        rspec
-            .get_ext("ext", false, "latest", "v17")
-            .expect_err("Extension should not be found");
-
-        let rspec: RemoteExtSpec = serde_json::from_value(serde_json::json!({
-            "public_extensions": [],
-            "custom_extensions": [],
-            "library_index": {
-                "ext": "ext"
-            },
-            "extension_data": {
-                "ext": {
-                    "control_data": {
-                        "ext.control": ""
-                    },
-                    "archive_path": ""
-                }
-            },
-        }))
-        .unwrap();
-
-        rspec
-            .get_ext("ext", false, "latest", "v17")
-            .expect_err("Extension should not be found");
-
-        let rspec: RemoteExtSpec = serde_json::from_value(serde_json::json!({
-            "public_extensions": [],
-            "custom_extensions": ["ext"],
-            "library_index": {
-                "ext": "ext"
-            },
-            "extension_data": {
-                "ext": {
-                    "control_data": {
-                        "ext.control": ""
-                    },
-                    "archive_path": ""
-                }
-            },
-        }))
-        .unwrap();
-
-        rspec
-            .get_ext("ext", false, "latest", "v17")
-            .expect("Extension should be found");
-
-        let rspec: RemoteExtSpec = serde_json::from_value(serde_json::json!({
-            "public_extensions": ["ext"],
-            "custom_extensions": [],
-            "library_index": {
-                "extlib": "ext",
-            },
-            "extension_data": {
-                "ext": {
-                    "control_data": {
-                        "ext.control": ""
-                    },
-                    "archive_path": ""
-                }
-            },
-        }))
-        .unwrap();
-
-        rspec
-            .get_ext("ext", false, "latest", "v17")
-            .expect("Extension should be found");
-
-        // test library index for the case when library name
-        // doesn't match the extension name
-        rspec
-            .get_ext("extlib", true, "latest", "v17")
-            .expect("Library should be found");
-    }
-
    #[test]
    fn parse_spec_file() {
        let file = File::open("tests/cluster_spec.json").unwrap();
--- a/libs/http-utils/Cargo.toml
+++ b/libs/http-utils/Cargo.toml
@@ -1,37 +0,0 @@
-[package]
-name = "http-utils"
-version = "0.1.0"
-edition.workspace = true
-license.workspace = true
-
-[dependencies]
-anyhow.workspace = true
-backtrace.workspace = true
-bytes.workspace = true
-inferno.workspace = true
-fail.workspace = true
-flate2.workspace = true
-hyper0.workspace = true
-itertools.workspace = true
-jemalloc_pprof.workspace = true
-once_cell.workspace = true
-pprof.workspace = true
-regex.workspace = true
-routerify.workspace = true
-serde.workspace = true
-serde_json.workspace = true
-serde_path_to_error.workspace = true
-thiserror.workspace = true
-tracing.workspace = true
-tokio.workspace = true
-tokio-util.workspace = true
-url.workspace = true
-uuid.workspace = true
-
-# to use tokio channels as streams, this is faster to compile than async_stream
-# why is it only here? no other crate should use it, streams are rarely needed.
-tokio-stream = { version = "0.1.14" }
-
-metrics.workspace = true
-utils.workspace = true
-workspace_hack = { version = "0.1", path = "../../workspace_hack" }
--- a/libs/http-utils/src/failpoints.rs
+++ b/libs/http-utils/src/failpoints.rs
@@ -1,50 +0,0 @@
-use crate::error::ApiError;
-use crate::json::{json_request, json_response};
-
-use hyper::{Body, Request, Response, StatusCode};
-use serde::{Deserialize, Serialize};
-use tokio_util::sync::CancellationToken;
-
-use utils::failpoint_support::apply_failpoint;
-
-pub type ConfigureFailpointsRequest = Vec<FailpointConfig>;
-
-/// Information for configuring a single fail point
-#[derive(Debug, Serialize, Deserialize)]
-pub struct FailpointConfig {
-    /// Name of the fail point
-    pub name: String,
-    /// List of actions to take, using the format described in `fail::cfg`
-    ///
-    /// We also support `actions = "exit"` to cause the fail point to immediately exit.
-    pub actions: String,
-}
-
-/// Configure failpoints through http.
-pub async fn failpoints_handler(
-    mut request: Request<Body>,
-    _cancel: CancellationToken,
-) -> Result<Response<Body>, ApiError> {
-    if !fail::has_failpoints() {
-        return Err(ApiError::BadRequest(anyhow::anyhow!(
-            "Cannot manage failpoints because neon was compiled without failpoints support"
-        )));
-    }
-
-    let failpoints: ConfigureFailpointsRequest = json_request(&mut request).await?;
-    for fp in failpoints {
-        tracing::info!("cfg failpoint: {} {}", fp.name, fp.actions);
-
-        // We recognize one extra "action" that's not natively recognized
-        // by the failpoints crate: exit, to immediately kill the process
-        let cfg_result = apply_failpoint(&fp.name, &fp.actions);
-
-        if let Err(err_msg) = cfg_result {
-            return Err(ApiError::BadRequest(anyhow::anyhow!(
-                "Failed to configure failpoints: {err_msg}"
-            )));
-        }
-    }
-
-    json_response(StatusCode::OK, ())
-}
--- a/libs/pageserver_api/src/config.rs
+++ b/libs/pageserver_api/src/config.rs
@@ -121,7 +121,6 @@ pub struct ConfigToml {
    pub wal_receiver_protocol: PostgresClientProtocol,
    pub page_service_pipelining: PageServicePipeliningConfig,
    pub get_vectored_concurrent_io: GetVectoredConcurrentIo,
-    pub enable_read_path_debugging: Option<bool>,
 }

 #[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
@@ -263,11 +262,6 @@ pub struct TenantConfigToml {
    /// size exceeds `compaction_upper_limit * checkpoint_distance`.
    pub compaction_upper_limit: usize,
    pub compaction_algorithm: crate::models::CompactionAlgorithmSettings,
-    /// If true, compact down L0 across all tenant timelines before doing regular compaction.
-    pub compaction_l0_first: bool,
-    /// If true, use a separate semaphore (i.e. concurrency limit) for the L0 compaction pass. Only
-    /// has an effect if `compaction_l0_first` is `true`.
-    pub compaction_l0_semaphore: bool,
    /// Level0 delta layer threshold at which to delay layer flushes for compaction backpressure,
    /// such that they take 2x as long, and start waiting for layer flushes during ephemeral layer
    /// rolls. This helps compaction keep up with WAL ingestion, and avoids read amplification
@@ -496,7 +490,7 @@ impl Default for ConfigToml {
                NonZeroUsize::new(DEFAULT_MAX_VECTORED_READ_BYTES).unwrap(),
            )),
            image_compression: (DEFAULT_IMAGE_COMPRESSION),
-            timeline_offloading: true,
+            timeline_offloading: false,
            ephemeral_bytes_per_memory_kb: (DEFAULT_EPHEMERAL_BYTES_PER_MEMORY_KB),
            l0_flush: None,
            virtual_file_io_mode: None,
@@ -516,11 +510,6 @@ impl Default for ConfigToml {
            } else {
                GetVectoredConcurrentIo::SidecarTask
            },
-            enable_read_path_debugging: if cfg!(test) || cfg!(feature = "testing") {
-                Some(true)
-            } else {
-                None
-            },
        }
    }
 }
@@ -548,8 +537,6 @@ pub mod tenant_conf_defaults {
    // most of our pageservers. Compaction ~50 layers requires about 2GB memory (could be reduced later by optimizing L0 hole
    // calculation to avoid loading all keys into the memory). So with this config, we can get a maximum peak compaction usage of 18GB.
    pub const DEFAULT_COMPACTION_UPPER_LIMIT: usize = 50;
-    pub const DEFAULT_COMPACTION_L0_FIRST: bool = false;
-    pub const DEFAULT_COMPACTION_L0_SEMAPHORE: bool = true;

    pub const DEFAULT_COMPACTION_ALGORITHM: crate::models::CompactionAlgorithm =
        crate::models::CompactionAlgorithm::Legacy;
@@ -599,8 +586,6 @@ impl Default for TenantConfigToml {
            compaction_algorithm: crate::models::CompactionAlgorithmSettings {
                kind: DEFAULT_COMPACTION_ALGORITHM,
            },
-            compaction_l0_first: DEFAULT_COMPACTION_L0_FIRST,
-            compaction_l0_semaphore: DEFAULT_COMPACTION_L0_SEMAPHORE,
            l0_flush_delay_threshold: None,
            l0_flush_stall_threshold: None,
            l0_flush_wait_upload: DEFAULT_L0_FLUSH_WAIT_UPLOAD,
@@ -631,7 +616,7 @@ impl Default for TenantConfigToml {
            image_creation_preempt_threshold: DEFAULT_IMAGE_CREATION_PREEMPT_THRESHOLD,
            lsn_lease_length: LsnLease::DEFAULT_LENGTH,
            lsn_lease_length_for_ts: LsnLease::DEFAULT_LENGTH_FOR_TS,
-            timeline_offloading: true,
+            timeline_offloading: false,
            wal_receiver_protocol_override: None,
            rel_size_v2_enabled: None,
            gc_compaction_enabled: DEFAULT_GC_COMPACTION_ENABLED,
--- a/libs/pageserver_api/src/controller_api.rs
+++ b/libs/pageserver_api/src/controller_api.rs
@@ -182,18 +182,6 @@ pub struct TenantDescribeResponseShard {
 #[derive(Serialize, Deserialize, Debug)]
 pub struct TenantShardMigrateRequest {
    pub node_id: NodeId,
-    #[serde(default)]
-    pub migration_config: Option<MigrationConfig>,
-}
-
-#[derive(Serialize, Deserialize, Debug)]
-pub struct MigrationConfig {
-    #[serde(default)]
-    #[serde(with = "humantime_serde")]
-    pub secondary_warmup_timeout: Option<Duration>,
-    #[serde(default)]
-    #[serde(with = "humantime_serde")]
-    pub secondary_download_request_timeout: Option<Duration>,
 }

 #[derive(Serialize, Clone, Debug)]
--- a/libs/pageserver_api/src/models.rs
+++ b/libs/pageserver_api/src/models.rs
@@ -464,10 +464,6 @@ pub struct TenantConfigPatch {
    #[serde(skip_serializing_if = "FieldPatch::is_noop")]
    pub compaction_algorithm: FieldPatch<CompactionAlgorithmSettings>,
    #[serde(skip_serializing_if = "FieldPatch::is_noop")]
-    pub compaction_l0_first: FieldPatch<bool>,
-    #[serde(skip_serializing_if = "FieldPatch::is_noop")]
-    pub compaction_l0_semaphore: FieldPatch<bool>,
-    #[serde(skip_serializing_if = "FieldPatch::is_noop")]
    pub l0_flush_delay_threshold: FieldPatch<usize>,
    #[serde(skip_serializing_if = "FieldPatch::is_noop")]
    pub l0_flush_stall_threshold: FieldPatch<usize>,
@@ -533,8 +529,6 @@ pub struct TenantConfig {
    pub compaction_upper_limit: Option<usize>,
    // defer parsing compaction_algorithm, like eviction_policy
    pub compaction_algorithm: Option<CompactionAlgorithmSettings>,
-    pub compaction_l0_first: Option<bool>,
-    pub compaction_l0_semaphore: Option<bool>,
    pub l0_flush_delay_threshold: Option<usize>,
    pub l0_flush_stall_threshold: Option<usize>,
    pub l0_flush_wait_upload: Option<bool>,
@@ -573,8 +567,6 @@ impl TenantConfig {
            mut compaction_threshold,
            mut compaction_upper_limit,
            mut compaction_algorithm,
-            mut compaction_l0_first,
-            mut compaction_l0_semaphore,
            mut l0_flush_delay_threshold,
            mut l0_flush_stall_threshold,
            mut l0_flush_wait_upload,
@@ -614,10 +606,6 @@ impl TenantConfig {
            .compaction_upper_limit
            .apply(&mut compaction_upper_limit);
        patch.compaction_algorithm.apply(&mut compaction_algorithm);
-        patch.compaction_l0_first.apply(&mut compaction_l0_first);
-        patch
-            .compaction_l0_semaphore
-            .apply(&mut compaction_l0_semaphore);
        patch
            .l0_flush_delay_threshold
            .apply(&mut l0_flush_delay_threshold);
@@ -681,8 +669,6 @@ impl TenantConfig {
            compaction_threshold,
            compaction_upper_limit,
            compaction_algorithm,
-            compaction_l0_first,
-            compaction_l0_semaphore,
            l0_flush_delay_threshold,
            l0_flush_stall_threshold,
            l0_flush_wait_upload,
@@ -1136,24 +1122,7 @@ pub struct TimelineInfo {
    pub ancestor_lsn: Option<Lsn>,
    pub last_record_lsn: Lsn,
    pub prev_record_lsn: Option<Lsn>,
-
-    /// Legacy field for compat with control plane.  Synonym of `min_readable_lsn`.
-    /// TODO: remove once control plane no longer reads it.
    pub latest_gc_cutoff_lsn: Lsn,
-
-    /// The LSN up to which GC has advanced: older data may still exist but it is not available for clients.
-    /// This LSN is not suitable for deciding where to create branches etc: use [`TimelineInfo::min_readable_lsn`] instead,
-    /// as it is easier to reason about.
-    pub applied_gc_cutoff_lsn: Lsn,
-
-    /// The upper bound of data which is either already GC'ed, or elegible to be GC'ed at any time based on PITR interval.
-    /// This LSN represents the "end of history" for this timeline, and callers should use it to figure out the oldest
-    /// LSN at which it is legal to create a branch or ephemeral endpoint.
-    ///
-    /// Note that holders of valid LSN leases may be able to create branches and read pages earlier
-    /// than this LSN, but new leases may not be taken out earlier than this LSN.
-    pub min_readable_lsn: Lsn,
-
    pub disk_consistent_lsn: Lsn,

    /// The LSN that we have succesfully uploaded to remote storage
--- a/libs/postgres_ffi/wal_craft/src/lib.rs
+++ b/libs/postgres_ffi/wal_craft/src/lib.rs
@@ -76,15 +76,7 @@ impl Conf {
        let mut cmd = Command::new(path);
        cmd.env_clear()
            .env("LD_LIBRARY_PATH", self.pg_lib_dir()?)
-            .env("DYLD_LIBRARY_PATH", self.pg_lib_dir()?)
-            .env(
-                "ASAN_OPTIONS",
-                std::env::var("ASAN_OPTIONS").unwrap_or_default(),
-            )
-            .env(
-                "UBSAN_OPTIONS",
-                std::env::var("UBSAN_OPTIONS").unwrap_or_default(),
-            );
+            .env("DYLD_LIBRARY_PATH", self.pg_lib_dir()?);
        Ok(cmd)
    }

--- a/libs/postgres_initdb/src/lib.rs
+++ b/libs/postgres_initdb/src/lib.rs
@@ -64,14 +64,6 @@ pub async fn do_run_initdb(args: RunInitdbArgs<'_>) -> Result<(), Error> {
        .env_clear()
        .env("LD_LIBRARY_PATH", library_search_path)
        .env("DYLD_LIBRARY_PATH", library_search_path)
-        .env(
-            "ASAN_OPTIONS",
-            std::env::var("ASAN_OPTIONS").unwrap_or_default(),
-        )
-        .env(
-            "UBSAN_OPTIONS",
-            std::env::var("UBSAN_OPTIONS").unwrap_or_default(),
-        )
        .stdin(std::process::Stdio::null())
        // stdout invocation produces the same output every time, we don't need it
        .stdout(std::process::Stdio::null())
--- a/libs/remote_storage/src/config.rs
+++ b/libs/remote_storage/src/config.rs
@@ -7,7 +7,7 @@ use serde::{Deserialize, Serialize};

 use crate::{
    DEFAULT_MAX_KEYS_PER_LIST_RESPONSE, DEFAULT_REMOTE_STORAGE_AZURE_CONCURRENCY_LIMIT,
-    DEFAULT_REMOTE_STORAGE_LOCALFS_CONCURRENCY_LIMIT, DEFAULT_REMOTE_STORAGE_S3_CONCURRENCY_LIMIT,
+    DEFAULT_REMOTE_STORAGE_S3_CONCURRENCY_LIMIT,
 };

 /// External backup storage configuration, enough for creating a client for that storage.
@@ -45,11 +45,11 @@ impl RemoteStorageKind {

 impl RemoteStorageConfig {
    /// Helper to fetch the configured concurrency limit.
-    pub fn concurrency_limit(&self) -> usize {
+    pub fn concurrency_limit(&self) -> Option<usize> {
        match &self.storage {
-            RemoteStorageKind::LocalFs { .. } => DEFAULT_REMOTE_STORAGE_LOCALFS_CONCURRENCY_LIMIT,
-            RemoteStorageKind::AwsS3(c) => c.concurrency_limit.into(),
-            RemoteStorageKind::AzureContainer(c) => c.concurrency_limit.into(),
+            RemoteStorageKind::LocalFs { .. } => None,
+            RemoteStorageKind::AwsS3(c) => Some(c.concurrency_limit.into()),
+            RemoteStorageKind::AzureContainer(c) => Some(c.concurrency_limit.into()),
        }
    }
 }
--- a/libs/remote_storage/src/lib.rs
+++ b/libs/remote_storage/src/lib.rs
@@ -65,12 +65,6 @@ pub const DEFAULT_REMOTE_STORAGE_S3_CONCURRENCY_LIMIT: usize = 100;
 /// Here, a limit of max 20k concurrent connections was noted.
 /// <https://learn.microsoft.com/en-us/answers/questions/1301863/is-there-any-limitation-to-concurrent-connections>
 pub const DEFAULT_REMOTE_STORAGE_AZURE_CONCURRENCY_LIMIT: usize = 100;
-/// Set this limit analogously to the S3 limit.
-///
-/// The local filesystem backend doesn't enforce a concurrency limit itself, but this also bounds
-/// the upload queue concurrency. Some tests create thousands of uploads, which slows down the
-/// quadratic scheduling of the upload queue, and there is no point spawning so many Tokio tasks.
-pub const DEFAULT_REMOTE_STORAGE_LOCALFS_CONCURRENCY_LIMIT: usize = 100;
 /// No limits on the client side, which currenltly means 1000 for AWS S3.
 /// <https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListObjectsV2.html#API_ListObjectsV2_RequestSyntax>
 pub const DEFAULT_MAX_KEYS_PER_LIST_RESPONSE: Option<i32> = None;
--- a/libs/utils/Cargo.toml
+++ b/libs/utils/Cargo.toml
@@ -21,17 +21,23 @@ bytes.workspace = true
 camino.workspace = true
 chrono.workspace = true
 diatomic-waker.workspace = true
+flate2.workspace = true
 git-version.workspace = true
 hex = { workspace = true, features = ["serde"] }
 humantime.workspace = true
+hyper0 = { workspace = true, features = ["full"] }
 inferno.workspace = true
+itertools.workspace = true
 fail.workspace = true
 futures = { workspace = true }
+jemalloc_pprof.workspace = true
 jsonwebtoken.workspace = true
 nix.workspace = true
 once_cell.workspace = true
 pin-project-lite.workspace = true
+pprof.workspace = true
 regex.workspace = true
+routerify.workspace = true
 serde.workspace = true
 serde_with.workspace = true
 serde_json.workspace = true
@@ -48,6 +54,8 @@ rand.workspace = true
 scopeguard.workspace = true
 strum.workspace = true
 strum_macros.workspace = true
+url.workspace = true
+uuid.workspace = true
 walkdir.workspace = true

 pq_proto.workspace = true
@@ -56,6 +64,12 @@ metrics.workspace = true

 const_format.workspace = true

+# to use tokio channels as streams, this is faster to compile than async_stream
+# why is it only here? no other crate should use it, streams are rarely needed.
+tokio-stream = { version = "0.1.14" }
+
+serde_path_to_error.workspace = true
+
 [dev-dependencies]
 byteorder.workspace = true
 bytes.workspace = true
--- a/libs/utils/scripts/restore_from_wal.sh
+++ b/libs/utils/scripts/restore_from_wal.sh
@@ -39,7 +39,7 @@ function initdb_with_args {
            ;;
    esac

-    eval env -i LD_LIBRARY_PATH="$PG_BIN"/../lib ASAN_OPTIONS="${ASAN_OPTIONS-}" UBSAN_OPTIONS="${UBSAN_OPTIONS-}" "${cmd[*]}"
+    eval env -i LD_LIBRARY_PATH="$PG_BIN"/../lib "${cmd[*]}"
 }

 rm -fr "$DATA_DIR"
--- a/libs/utils/src/auth.rs
+++ b/libs/utils/src/auth.rs
@@ -10,7 +10,7 @@ use jsonwebtoken::{
 };
 use serde::{Deserialize, Serialize};

-use crate::id::TenantId;
+use crate::{http::error::ApiError, id::TenantId};

 /// Algorithm to use. We require EdDSA.
 const STORAGE_TOKEN_ALGORITHM: Algorithm = Algorithm::EdDSA;
@@ -90,6 +90,15 @@ impl Display for AuthError {
    }
 }

+impl From<AuthError> for ApiError {
+    fn from(_value: AuthError) -> Self {
+        // Don't pass on the value of the AuthError as a precautionary measure.
+        // Being intentionally vague in public error communication hurts debugability
+        // but it is more secure.
+        ApiError::Forbidden("JWT authentication error".to_string())
+    }
+}
+
 pub struct JwtAuth {
    decoding_keys: Vec<DecodingKey>,
    validation: Validation,
--- a/libs/utils/src/backoff.rs
+++ b/libs/utils/src/backoff.rs
@@ -1,5 +1,4 @@
 use std::fmt::{Debug, Display};
-use std::time::Duration;

 use futures::Future;
 use tokio_util::sync::CancellationToken;
@@ -30,11 +29,6 @@ pub async fn exponential_backoff(
    }
 }

-pub fn exponential_backoff_duration(n: u32, base_increment: f64, max_seconds: f64) -> Duration {
-    let seconds = exponential_backoff_duration_seconds(n, base_increment, max_seconds);
-    Duration::from_secs_f64(seconds)
-}
-
 pub fn exponential_backoff_duration_seconds(n: u32, base_increment: f64, max_seconds: f64) -> f64 {
    if n == 0 {
        0.0
--- a/libs/utils/src/failpoint_support.rs
+++ b/libs/utils/src/failpoint_support.rs
@@ -1,6 +1,13 @@
 //! Failpoint support code shared between pageserver and safekeepers.

+use crate::http::{
+    error::ApiError,
+    json::{json_request, json_response},
+};
+use hyper::{Body, Request, Response, StatusCode};
+use serde::{Deserialize, Serialize};
 use tokio_util::sync::CancellationToken;
+use tracing::*;

 /// Declare a failpoint that can use to `pause` failpoint action.
 /// We don't want to block the executor thread, hence, spawn_blocking + await.
@@ -177,3 +184,45 @@ fn exit_failpoint() {
    tracing::info!("Exit requested by failpoint");
    std::process::exit(1);
 }
+
+pub type ConfigureFailpointsRequest = Vec<FailpointConfig>;
+
+/// Information for configuring a single fail point
+#[derive(Debug, Serialize, Deserialize)]
+pub struct FailpointConfig {
+    /// Name of the fail point
+    pub name: String,
+    /// List of actions to take, using the format described in `fail::cfg`
+    ///
+    /// We also support `actions = "exit"` to cause the fail point to immediately exit.
+    pub actions: String,
+}
+
+/// Configure failpoints through http.
+pub async fn failpoints_handler(
+    mut request: Request<Body>,
+    _cancel: CancellationToken,
+) -> Result<Response<Body>, ApiError> {
+    if !fail::has_failpoints() {
+        return Err(ApiError::BadRequest(anyhow::anyhow!(
+            "Cannot manage failpoints because neon was compiled without failpoints support"
+        )));
+    }
+
+    let failpoints: ConfigureFailpointsRequest = json_request(&mut request).await?;
+    for fp in failpoints {
+        info!("cfg failpoint: {} {}", fp.name, fp.actions);
+
+        // We recognize one extra "action" that's not natively recognized
+        // by the failpoints crate: exit, to immediately kill the process
+        let cfg_result = apply_failpoint(&fp.name, &fp.actions);
+
+        if let Err(err_msg) = cfg_result {
+            return Err(ApiError::BadRequest(anyhow::anyhow!(
+                "Failed to configure failpoints: {err_msg}"
+            )));
+        }
+    }
+
+    json_response(StatusCode::OK, ())
+}
--- a/libs/utils/src/http/endpoint.rs
+++ b/libs/utils/src/http/endpoint.rs
@@ -1,6 +1,7 @@
-use crate::error::{api_error_handler, route_error_handler, ApiError};
+use crate::auth::{AuthError, Claims, SwappableJwtAuth};
+use crate::http::error::{api_error_handler, route_error_handler, ApiError};
+use crate::http::request::{get_query_param, parse_query_param};
 use crate::pprof;
-use crate::request::{get_query_param, parse_query_param};
 use ::pprof::protos::Message as _;
 use ::pprof::ProfilerGuardBuilder;
 use anyhow::{anyhow, Context};
@@ -18,7 +19,6 @@ use tokio::sync::{mpsc, Mutex, Notify};
 use tokio_stream::wrappers::ReceiverStream;
 use tokio_util::io::ReaderStream;
 use tracing::{debug, info, info_span, warn, Instrument};
-use utils::auth::{AuthError, Claims, SwappableJwtAuth};

 use std::future::Future;
 use std::io::Write as _;
@@ -718,9 +718,9 @@ pub fn check_permission_with(
 #[cfg(test)]
 mod tests {
    use super::*;
+    use futures::future::poll_fn;
    use hyper::service::Service;
    use routerify::RequestServiceBuilder;
-    use std::future::poll_fn;
    use std::net::{IpAddr, SocketAddr};

    #[tokio::test]
--- a/libs/utils/src/http/error.rs
+++ b/libs/utils/src/http/error.rs
@@ -5,8 +5,6 @@ use std::error::Error as StdError;
 use thiserror::Error;
 use tracing::{error, info, warn};

-use utils::auth::AuthError;
-
 #[derive(Debug, Error)]
 pub enum ApiError {
    #[error("Bad request: {0:#?}")]
@@ -98,15 +96,6 @@ impl ApiError {
    }
 }

-impl From<AuthError> for ApiError {
-    fn from(_value: AuthError) -> Self {
-        // Don't pass on the value of the AuthError as a precautionary measure.
-        // Being intentionally vague in public error communication hurts debugability
-        // but it is more secure.
-        ApiError::Forbidden("JWT authentication error".to_string())
-    }
-}
-
 #[derive(Serialize, Deserialize)]
 pub struct HttpErrorBody {
    pub msg: String,
--- a/libs/utils/src/http/json.rs
+++ b/libs/utils/src/http/json.rs
--- a/libs/utils/src/http/mod.rs
+++ b/libs/utils/src/http/mod.rs
@@ -1,12 +1,8 @@
 pub mod endpoint;
 pub mod error;
-pub mod failpoints;
 pub mod json;
-pub mod pprof;
 pub mod request;

-extern crate hyper0 as hyper;
-
 /// Current fast way to apply simple http routing in various Neon binaries.
 /// Re-exported for sake of uniform approach, that could be later replaced with better alternatives, if needed.
 pub use routerify::{ext::RequestExt, RouterBuilder, RouterService};
--- a/libs/utils/src/http/request.rs
+++ b/libs/utils/src/http/request.rs
--- a/libs/utils/src/lib.rs
+++ b/libs/utils/src/lib.rs
@@ -2,6 +2,8 @@
 //! between other crates in this repository.
 #![deny(clippy::undocumented_unsafe_blocks)]

+extern crate hyper0 as hyper;
+
 pub mod backoff;

 /// `Lsn` type implements common tasks on Log Sequence Numbers
@@ -31,6 +33,9 @@ pub mod shard;
 mod hex;
 pub use hex::Hex;

+// http endpoint utils
+pub mod http;
+
 // definition of the Generation type for pageserver attachment APIs
 pub mod generation;

@@ -91,6 +96,8 @@ pub mod circuit_breaker;

 pub mod try_rcu;

+pub mod pprof;
+
 pub mod guard_arc_swap;

 // Re-export used in macro. Avoids adding git-version as dep in target crates.
--- a/libs/utils/src/logging.rs
+++ b/libs/utils/src/logging.rs
@@ -8,22 +8,19 @@ use strum_macros::{EnumString, VariantNames};
 /// Logs a critical error, similarly to `tracing::error!`. This will:
 ///
 /// * Emit an ERROR log message with prefix "CRITICAL:" and a backtrace.
-/// * Trigger a pageable alert (via the metric below).
 /// * Increment libmetrics_tracing_event_count{level="critical"}, and indirectly level="error".
+/// * Trigger a pageable alert (via the metric above).
 /// * In debug builds, panic the process.
-///
-/// When including errors in the message, please use {err:?} to include the error cause and original
-/// backtrace.
 #[macro_export]
 macro_rules! critical {
-    ($($arg:tt)*) => {{
+    ($($arg:tt)*) => {
        if cfg!(debug_assertions) {
            panic!($($arg)*);
        }
        $crate::logging::TRACING_EVENT_COUNT_METRIC.inc_critical();
        let backtrace = std::backtrace::Backtrace::capture();
        tracing::error!("CRITICAL: {}\n{backtrace}", format!($($arg)*));
-    }};
+    };
 }

 #[derive(EnumString, strum_macros::Display, VariantNames, Eq, PartialEq, Debug, Clone, Copy)]
--- a/libs/http-utils/src/pprof.rs
+++ b/libs/http-utils/src/pprof.rs
--- a/pageserver/Cargo.toml
+++ b/pageserver/Cargo.toml
@@ -79,7 +79,6 @@ pq_proto.workspace = true
 remote_storage.workspace = true
 storage_broker.workspace = true
 tenant_size_model.workspace = true
-http-utils.workspace = true
 utils.workspace = true
 workspace_hack.workspace = true
 reqwest.workspace = true
--- a/pageserver/client/Cargo.toml
+++ b/pageserver/client/Cargo.toml
@@ -11,7 +11,6 @@ testing = [ "pageserver_api/testing" ]
 pageserver_api.workspace = true
 thiserror.workspace = true
 reqwest = { workspace = true, features = [ "stream" ] }
-http-utils.workspace = true
 utils.workspace = true
 serde.workspace = true
 workspace_hack = { version = "0.1", path = "../../workspace_hack" }
--- a/pageserver/client/src/mgmt_api.rs
+++ b/pageserver/client/src/mgmt_api.rs
@@ -1,12 +1,11 @@
 use std::{collections::HashMap, error::Error as _};

 use bytes::Bytes;
-use reqwest::{IntoUrl, Method, StatusCode};
-
 use detach_ancestor::AncestorDetached;
-use http_utils::error::HttpErrorBody;
 use pageserver_api::{models::*, shard::TenantShardId};
+use reqwest::{IntoUrl, Method, StatusCode};
 use utils::{
+    http::error::HttpErrorBody,
    id::{TenantId, TimelineId},
    lsn::Lsn,
 };
--- a/pageserver/src/basebackup.rs
+++ b/pageserver/src/basebackup.rs
@@ -42,8 +42,8 @@ use utils::lsn::Lsn;
 pub enum BasebackupError {
    #[error("basebackup pageserver error {0:#}")]
    Server(#[from] anyhow::Error),
-    #[error("basebackup client error {0:#} when {1}")]
-    Client(#[source] io::Error, &'static str),
+    #[error("basebackup client error {0:#}")]
+    Client(#[source] io::Error),
 }

 /// Create basebackup with non-rel data in it.
@@ -234,7 +234,7 @@ where
        self.ar
            .append(&header, self.buf.as_slice())
            .await
-            .map_err(|e| BasebackupError::Client(e, "flush"))?;
+            .map_err(BasebackupError::Client)?;

        self.total_blocks += nblocks;
        debug!("Added to basebackup slru {} relsize {}", segname, nblocks);
@@ -273,9 +273,9 @@ where
        for dir in subdirs.iter() {
            let header = new_tar_header_dir(dir)?;
            self.ar
-                .append(&header, io::empty())
+                .append(&header, &mut io::empty())
                .await
-                .map_err(|e| BasebackupError::Client(e, "send_tarball"))?;
+                .context("could not add directory to basebackup tarball")?;
        }

        // Send config files.
@@ -286,13 +286,13 @@ where
                self.ar
                    .append(&header, data)
                    .await
-                    .map_err(|e| BasebackupError::Client(e, "send_tarball,pg_hba.conf"))?;
+                    .context("could not add config file to basebackup tarball")?;
            } else {
                let header = new_tar_header(filepath, 0)?;
                self.ar
-                    .append(&header, io::empty())
+                    .append(&header, &mut io::empty())
                    .await
-                    .map_err(|e| BasebackupError::Client(e, "send_tarball,add_config_file"))?;
+                    .context("could not add config file to basebackup tarball")?;
            }
        }
        if !lazy_slru_download {
@@ -406,7 +406,7 @@ where
            self.ar
                .append(&header, &*content)
                .await
-                .map_err(|e| BasebackupError::Client(e, "send_tarball,add_aux_file"))?;
+                .context("could not add aux file to basebackup tarball")?;
        }

        if min_restart_lsn != Lsn::MAX {
@@ -419,7 +419,7 @@ where
            self.ar
                .append(&header, &data[..])
                .await
-                .map_err(|e| BasebackupError::Client(e, "send_tarball,restart.lsn"))?;
+                .context("could not add restart.lsn file to basebackup tarball")?;
        }
        for xid in self
            .timeline
@@ -451,9 +451,9 @@ where
            let crc32 = crc32c::crc32c(&content);
            content.extend_from_slice(&crc32.to_le_bytes());
            let header = new_tar_header("pg_logical/replorigin_checkpoint", content.len() as u64)?;
-            self.ar.append(&header, &*content).await.map_err(|e| {
-                BasebackupError::Client(e, "send_tarball,pg_logical/replorigin_checkpoint")
-            })?;
+            self.ar.append(&header, &*content).await.context(
+                "could not add pg_logical/replorigin_checkpoint file to basebackup tarball",
+            )?;
        }

        fail_point!("basebackup-before-control-file", |_| {
@@ -464,10 +464,7 @@ where

        // Generate pg_control and bootstrap WAL segment.
        self.add_pgcontrol_file().await?;
-        self.ar
-            .finish()
-            .await
-            .map_err(|e| BasebackupError::Client(e, "send_tarball,finish"))?;
+        self.ar.finish().await.map_err(BasebackupError::Client)?;
        debug!("all tarred up!");
        Ok(())
    }
@@ -485,9 +482,9 @@ where
            let file_name = dst.to_segfile_name(0);
            let header = new_tar_header(&file_name, 0)?;
            self.ar
-                .append(&header, io::empty())
+                .append(&header, &mut io::empty())
                .await
-                .map_err(|e| BasebackupError::Client(e, "add_rel,empty"))?;
+                .map_err(BasebackupError::Client)?;
            return Ok(());
        }

@@ -518,7 +515,7 @@ where
            self.ar
                .append(&header, segment_data.as_slice())
                .await
-                .map_err(|e| BasebackupError::Client(e, "add_rel,segment"))?;
+                .map_err(BasebackupError::Client)?;

            seg += 1;
            startblk = endblk;
@@ -569,7 +566,7 @@ where
            self.ar
                .append(&header, pg_version_str.as_bytes())
                .await
-                .map_err(|e| BasebackupError::Client(e, "add_dbdir,PG_VERSION"))?;
+                .map_err(BasebackupError::Client)?;

            info!("timeline.pg_version {}", self.timeline.pg_version);

@@ -579,7 +576,7 @@ where
                self.ar
                    .append(&header, &img[..])
                    .await
-                    .map_err(|e| BasebackupError::Client(e, "add_dbdir,global/pg_filenode.map"))?;
+                    .map_err(BasebackupError::Client)?;
            } else {
                warn!("global/pg_filenode.map is missing");
            }
@@ -615,9 +612,9 @@ where
            let path = format!("base/{}", dbnode);
            let header = new_tar_header_dir(&path)?;
            self.ar
-                .append(&header, io::empty())
+                .append(&header, &mut io::empty())
                .await
-                .map_err(|e| BasebackupError::Client(e, "add_dbdir,base"))?;
+                .map_err(BasebackupError::Client)?;

            if let Some(img) = relmap_img {
                let dst_path = format!("base/{}/PG_VERSION", dbnode);
@@ -630,14 +627,14 @@ where
                self.ar
                    .append(&header, pg_version_str.as_bytes())
                    .await
-                    .map_err(|e| BasebackupError::Client(e, "add_dbdir,base/PG_VERSION"))?;
+                    .map_err(BasebackupError::Client)?;

                let relmap_path = format!("base/{}/pg_filenode.map", dbnode);
                let header = new_tar_header(&relmap_path, img.len() as u64)?;
                self.ar
                    .append(&header, &img[..])
                    .await
-                    .map_err(|e| BasebackupError::Client(e, "add_dbdir,base/pg_filenode.map"))?;
+                    .map_err(BasebackupError::Client)?;
            }
        };
        Ok(())
@@ -666,7 +663,7 @@ where
        self.ar
            .append(&header, &buf[..])
            .await
-            .map_err(|e| BasebackupError::Client(e, "add_twophase_file"))?;
+            .map_err(BasebackupError::Client)?;

        Ok(())
    }
@@ -696,7 +693,7 @@ where
                zenith_signal.as_bytes(),
            )
            .await
-            .map_err(|e| BasebackupError::Client(e, "add_pgcontrol_file,zenith.signal"))?;
+            .map_err(BasebackupError::Client)?;

        let checkpoint_bytes = self
            .timeline
@@ -721,7 +718,7 @@ where
        self.ar
            .append(&header, &pg_control_bytes[..])
            .await
-            .map_err(|e| BasebackupError::Client(e, "add_pgcontrol_file,pg_control"))?;
+            .map_err(BasebackupError::Client)?;

        //send wal segment
        let segno = self.lsn.segment_number(WAL_SEGMENT_SIZE);
@@ -745,7 +742,7 @@ where
        self.ar
            .append(&header, &wal_seg[..])
            .await
-            .map_err(|e| BasebackupError::Client(e, "add_pgcontrol_file,wal_segment"))?;
+            .map_err(BasebackupError::Client)?;
        Ok(())
    }
 }
--- a/pageserver/src/bin/pageserver.rs
+++ b/pageserver/src/bin/pageserver.rs
@@ -592,7 +592,7 @@ fn start_pageserver(
        let router = http::make_router(router_state, launch_ts, http_auth.clone())?
            .build()
            .map_err(|err| anyhow!(err))?;
-        let service = http_utils::RouterService::new(router).unwrap();
+        let service = utils::http::RouterService::new(router).unwrap();
        let server = hyper0::Server::from_tcp(http_listener)?
            .serve(service)
            .with_graceful_shutdown({
--- a/pageserver/src/config.rs
+++ b/pageserver/src/config.rs
@@ -193,10 +193,6 @@ pub struct PageServerConf {
    pub page_service_pipelining: pageserver_api::config::PageServicePipeliningConfig,

    pub get_vectored_concurrent_io: pageserver_api::config::GetVectoredConcurrentIo,
-
-    /// Enable read path debugging. If enabled, read key errors will print a backtrace of the layer
-    /// files read.
-    pub enable_read_path_debugging: bool,
 }

 /// Token for authentication to safekeepers
@@ -359,7 +355,6 @@ impl PageServerConf {
            wal_receiver_protocol,
            page_service_pipelining,
            get_vectored_concurrent_io,
-            enable_read_path_debugging,
        } = config_toml;

        let mut conf = PageServerConf {
@@ -445,7 +440,6 @@ impl PageServerConf {
                .unwrap_or_default(),
            virtual_file_io_mode: virtual_file_io_mode.unwrap_or(virtual_file::IoMode::preferred()),
            no_sync: no_sync.unwrap_or(false),
-            enable_read_path_debugging: enable_read_path_debugging.unwrap_or(false),
        };

        // ------------------------------------------------------------
--- a/pageserver/src/deletion_queue.rs
+++ b/pageserver/src/deletion_queue.rs
@@ -8,6 +8,7 @@ use std::time::Duration;

 use crate::controller_upcall_client::ControlPlaneGenerationsApi;
 use crate::metrics;
+use crate::tenant::remote_timeline_client::remote_layer_path;
 use crate::tenant::remote_timeline_client::remote_timeline_path;
 use crate::tenant::remote_timeline_client::LayerFileMetadata;
 use crate::virtual_file::MaybeFatalIo;
@@ -462,18 +463,45 @@ impl DeletionQueueClient {
    ///
    /// The `current_generation` is the generation of this pageserver's current attachment.  The
    /// generations in `layers` are the generations in which those layers were written.
-    pub(crate) fn push_layers(
+    pub(crate) async fn push_layers(
        &self,
        tenant_shard_id: TenantShardId,
        timeline_id: TimelineId,
        current_generation: Generation,
        layers: Vec<(LayerName, LayerFileMetadata)>,
    ) -> Result<(), DeletionQueueError> {
-        // None generations are not valid for attached tenants: they must always be attached in
-        // a known generation.  None generations are still permitted for layers in the index because
-        // they may be historical.
-        assert!(!current_generation.is_none());
+        if current_generation.is_none() {
+            debug!("Enqueuing deletions in legacy mode, skipping queue");

+            let mut layer_paths = Vec::new();
+            for (layer, meta) in layers {
+                layer_paths.push(remote_layer_path(
+                    &tenant_shard_id.tenant_id,
+                    &timeline_id,
+                    meta.shard,
+                    &layer,
+                    meta.generation,
+                ));
+            }
+            self.push_immediate(layer_paths).await?;
+            return self.flush_immediate().await;
+        }
+
+        self.push_layers_sync(tenant_shard_id, timeline_id, current_generation, layers)
+    }
+
+    /// When a Tenant has a generation, push_layers is always synchronous because
+    /// the ListValidator channel is an unbounded channel.
+    ///
+    /// This can be merged into push_layers when we remove the Generation-less mode
+    /// support (`<https://github.com/neondatabase/neon/issues/5395>`)
+    pub(crate) fn push_layers_sync(
+        &self,
+        tenant_shard_id: TenantShardId,
+        timeline_id: TimelineId,
+        current_generation: Generation,
+        layers: Vec<(LayerName, LayerFileMetadata)>,
+    ) -> Result<(), DeletionQueueError> {
        metrics::DELETION_QUEUE
            .keys_submitted
            .inc_by(layers.len() as u64);
@@ -929,12 +957,14 @@ mod test {

        // File should still be there after we push it to the queue (we haven't pushed enough to flush anything)
        info!("Pushing");
-        client.push_layers(
-            tenant_shard_id,
-            TIMELINE_ID,
-            now_generation,
-            [(layer_file_name_1.clone(), layer_metadata)].to_vec(),
-        )?;
+        client
+            .push_layers(
+                tenant_shard_id,
+                TIMELINE_ID,
+                now_generation,
+                [(layer_file_name_1.clone(), layer_metadata)].to_vec(),
+            )
+            .await?;
        assert_remote_files(&[&remote_layer_file_name_1], &remote_timeline_path);

        assert_local_files(&[], &deletion_prefix);
@@ -987,12 +1017,14 @@ mod test {
        assert_remote_files(&[&remote_layer_name], &remote_timeline_path);

        tracing::debug!("Pushing...");
-        client.push_layers(
-            tenant_shard_id,
-            TIMELINE_ID,
-            stale_generation,
-            [(EXAMPLE_LAYER_NAME.clone(), layer_metadata.clone())].to_vec(),
-        )?;
+        client
+            .push_layers(
+                tenant_shard_id,
+                TIMELINE_ID,
+                stale_generation,
+                [(EXAMPLE_LAYER_NAME.clone(), layer_metadata.clone())].to_vec(),
+            )
+            .await?;

        // We enqueued the operation in a stale generation: it should have failed validation
        tracing::debug!("Flushing...");
@@ -1000,12 +1032,14 @@ mod test {
        assert_remote_files(&[&remote_layer_name], &remote_timeline_path);

        tracing::debug!("Pushing...");
-        client.push_layers(
-            tenant_shard_id,
-            TIMELINE_ID,
-            latest_generation,
-            [(EXAMPLE_LAYER_NAME.clone(), layer_metadata.clone())].to_vec(),
-        )?;
+        client
+            .push_layers(
+                tenant_shard_id,
+                TIMELINE_ID,
+                latest_generation,
+                [(EXAMPLE_LAYER_NAME.clone(), layer_metadata.clone())].to_vec(),
+            )
+            .await?;

        // We enqueued the operation in a fresh generation: it should have passed validation
        tracing::debug!("Flushing...");
@@ -1040,24 +1074,28 @@ mod test {
        // generation gets that treatment)
        let remote_layer_file_name_historical =
            ctx.write_remote_layer(EXAMPLE_LAYER_NAME, layer_generation)?;
-        client.push_layers(
-            tenant_shard_id,
-            TIMELINE_ID,
-            now_generation.previous(),
-            [(EXAMPLE_LAYER_NAME.clone(), layer_metadata.clone())].to_vec(),
-        )?;
+        client
+            .push_layers(
+                tenant_shard_id,
+                TIMELINE_ID,
+                now_generation.previous(),
+                [(EXAMPLE_LAYER_NAME.clone(), layer_metadata.clone())].to_vec(),
+            )
+            .await?;

        // Inject a deletion in the generation before generation_now: after restart,
        // this deletion should get executed, because we execute deletions in the
        // immediately previous generation on the same node.
        let remote_layer_file_name_previous =
            ctx.write_remote_layer(EXAMPLE_LAYER_NAME_ALT, layer_generation)?;
-        client.push_layers(
-            tenant_shard_id,
-            TIMELINE_ID,
-            now_generation,
-            [(EXAMPLE_LAYER_NAME_ALT.clone(), layer_metadata.clone())].to_vec(),
-        )?;
+        client
+            .push_layers(
+                tenant_shard_id,
+                TIMELINE_ID,
+                now_generation,
+                [(EXAMPLE_LAYER_NAME_ALT.clone(), layer_metadata.clone())].to_vec(),
+            )
+            .await?;

        client.flush().await?;
        assert_remote_files(
@@ -1101,7 +1139,6 @@ pub(crate) mod mock {
    use tracing::info;

    use super::*;
-    use crate::tenant::remote_timeline_client::remote_layer_path;
    use std::sync::atomic::{AtomicUsize, Ordering};

    pub struct ConsumerState {
--- a/pageserver/src/disk_usage_eviction_task.rs
+++ b/pageserver/src/disk_usage_eviction_task.rs
@@ -61,7 +61,6 @@ use crate::{
        remote_timeline_client::LayerFileMetadata,
        secondary::SecondaryTenant,
        storage_layer::{AsLayerDesc, EvictionError, Layer, LayerName, LayerVisibilityHint},
-        tasks::sleep_random,
    },
    CancellableTask, DiskUsageEvictionTask,
 };
@@ -211,8 +210,14 @@ async fn disk_usage_eviction_task(
        info!("disk usage based eviction task finishing");
    };

-    if sleep_random(task_config.period, &cancel).await.is_err() {
-        return;
+    use crate::tenant::tasks::random_init_delay;
+    {
+        if random_init_delay(task_config.period, &cancel)
+            .await
+            .is_err()
+        {
+            return;
+        }
    }

    let mut iteration_no = 0;
--- a/pageserver/src/http/openapi_spec.yml
+++ b/pageserver/src/http/openapi_spec.yml
@@ -1080,10 +1080,7 @@ components:
          type: integer
        state:
          type: string
-        min_readable_lsn:
-          type: string
-          format: hex
-        applied_gc_cutoff_lsn:
+        latest_gc_cutoff_lsn:
          type: string
          format: hex

--- a/pageserver/src/http/routes.rs
+++ b/pageserver/src/http/routes.rs
@@ -13,12 +13,6 @@ use enumset::EnumSet;
 use futures::future::join_all;
 use futures::StreamExt;
 use futures::TryFutureExt;
-use http_utils::endpoint::{
-    profile_cpu_handler, profile_heap_handler, prometheus_metrics_handler, request_span,
-};
-use http_utils::failpoints::failpoints_handler;
-use http_utils::request::must_parse_query_param;
-use http_utils::request::{get_request_param, must_get_query_param, parse_query_param};
 use humantime::format_rfc3339;
 use hyper::header;
 use hyper::StatusCode;
@@ -66,6 +60,13 @@ use tokio::time::Instant;
 use tokio_util::io::StreamReader;
 use tokio_util::sync::CancellationToken;
 use tracing::*;
+use utils::auth::JwtAuth;
+use utils::failpoint_support::failpoints_handler;
+use utils::http::endpoint::{
+    profile_cpu_handler, profile_heap_handler, prometheus_metrics_handler, request_span,
+};
+use utils::http::request::must_parse_query_param;
+use utils::http::request::{get_request_param, must_get_query_param, parse_query_param};

 use crate::config::PageServerConf;
 use crate::context::{DownloadBehavior, RequestContext};
@@ -103,13 +104,6 @@ use crate::tenant::OffloadedTimeline;
 use crate::tenant::{LogicalSizeCalculationCause, PageReconstructError};
 use crate::DEFAULT_PG_VERSION;
 use crate::{disk_usage_eviction_task, tenant};
-use http_utils::{
-    endpoint::{self, attach_openapi_ui, auth_middleware, check_permission_with},
-    error::{ApiError, HttpErrorBody},
-    json::{json_request, json_request_maybe, json_response},
-    request::parse_request_param,
-    RequestExt, RouterBuilder,
-};
 use pageserver_api::models::{
    StatusResponse, TenantConfigRequest, TenantInfo, TimelineCreateRequest, TimelineGcRequest,
    TimelineInfo,
@@ -117,6 +111,13 @@ use pageserver_api::models::{
 use utils::{
    auth::SwappableJwtAuth,
    generation::Generation,
+    http::{
+        endpoint::{self, attach_openapi_ui, auth_middleware, check_permission_with},
+        error::{ApiError, HttpErrorBody},
+        json::{json_request, json_request_maybe, json_response},
+        request::parse_request_param,
+        RequestExt, RouterBuilder,
+    },
    id::{TenantId, TimelineId},
    lsn::Lsn,
 };
@@ -482,11 +483,6 @@ async fn build_timeline_info_common(

    let (pitr_history_size, within_ancestor_pitr) = timeline.get_pitr_history_stats();

-    let min_readable_lsn = std::cmp::max(
-        timeline.get_gc_cutoff_lsn(),
-        *timeline.get_applied_gc_cutoff_lsn(),
-    );
-
    let info = TimelineInfo {
        tenant_id: timeline.tenant_shard_id,
        timeline_id: timeline.timeline_id,
@@ -498,12 +494,7 @@ async fn build_timeline_info_common(
        initdb_lsn,
        last_record_lsn,
        prev_record_lsn: Some(timeline.get_prev_record_lsn()),
-        // Externally, expose the lowest LSN that can be used to create a branch as the "GC cutoff", although internally
-        // we distinguish between the "planned" GC cutoff (PITR point) and the "latest" GC cutoff (where we
-        // actually trimmed data to), which can pass each other when PITR is changed.
-        latest_gc_cutoff_lsn: min_readable_lsn,
-        min_readable_lsn,
-        applied_gc_cutoff_lsn: *timeline.get_applied_gc_cutoff_lsn(),
+        latest_gc_cutoff_lsn: *timeline.get_latest_gc_cutoff_lsn(),
        current_logical_size: current_logical_size.size_dont_care_about_accuracy(),
        current_logical_size_is_accurate: match current_logical_size.accuracy() {
            tenant::timeline::logical_size::Accuracy::Approximate => false,
@@ -570,7 +561,7 @@ async fn reload_auth_validation_keys_handler(
    let key_path = config.auth_validation_public_key_path.as_ref().unwrap();
    info!("Reloading public key(s) for verifying JWT tokens from {key_path:?}");

-    match utils::auth::JwtAuth::from_key_path(key_path) {
+    match JwtAuth::from_key_path(key_path) {
        Ok(new_auth) => {
            shared_auth.swap(new_auth);
            json_response(StatusCode::OK, ())
@@ -2161,7 +2152,6 @@ async fn timeline_compact_handler(
    let state = get_state(&request);

    let mut flags = EnumSet::empty();
-    flags |= CompactFlags::NoYield; // run compaction to completion

    if Some(true) == parse_query_param::<_, bool>(&request, "force_l0_compaction")? {
        flags |= CompactFlags::ForceL0Compaction;
--- a/pageserver/src/metrics.rs
+++ b/pageserver/src/metrics.rs
@@ -6,7 +6,7 @@ use std::sync::{Arc, Mutex};
 use std::task::{Context, Poll};
 use std::time::{Duration, Instant};

-use enum_map::{Enum as _, EnumMap};
+use enum_map::EnumMap;
 use futures::Future;
 use metrics::{
    register_counter_vec, register_gauge_vec, register_histogram, register_histogram_vec,
@@ -32,7 +32,6 @@ use utils::id::TimelineId;

 use crate::config::PageServerConf;
 use crate::context::{PageContentKind, RequestContext};
-use crate::pgdatadir_mapping::DatadirModificationStats;
 use crate::task_mgr::TaskKind;
 use crate::tenant::layer_map::LayerMap;
 use crate::tenant::mgr::TenantSlot;
@@ -104,7 +103,7 @@ pub(crate) static STORAGE_TIME_COUNT_PER_TIMELINE: Lazy<IntCounterVec> = Lazy::n
    .expect("failed to define a metric")
 });

-// Buckets for background operation duration in seconds, like compaction, GC, size calculation.
+// Buckets for background operations like compaction, GC, size calculation
 const STORAGE_OP_BUCKETS: &[f64] = &[0.010, 0.100, 1.0, 10.0, 100.0, 1000.0];

 pub(crate) static STORAGE_TIME_GLOBAL: Lazy<HistogramVec> = Lazy::new(|| {
@@ -236,7 +235,7 @@ pub(crate) static GET_VECTORED_LATENCY: Lazy<GetVectoredLatency> = Lazy::new(||

    GetVectoredLatency {
        map: EnumMap::from_array(std::array::from_fn(|task_kind_idx| {
-            let task_kind = TaskKind::from_usize(task_kind_idx);
+            let task_kind = <TaskKind as enum_map::Enum>::from_usize(task_kind_idx);

            if GetVectoredLatency::TRACKED_TASK_KINDS.contains(&task_kind) {
                let task_kind = task_kind.into();
@@ -259,7 +258,7 @@ pub(crate) static SCAN_LATENCY: Lazy<ScanLatency> = Lazy::new(|| {

    ScanLatency {
        map: EnumMap::from_array(std::array::from_fn(|task_kind_idx| {
-            let task_kind = TaskKind::from_usize(task_kind_idx);
+            let task_kind = <TaskKind as enum_map::Enum>::from_usize(task_kind_idx);

            if ScanLatency::TRACKED_TASK_KINDS.contains(&task_kind) {
                let task_kind = task_kind.into();
@@ -300,10 +299,10 @@ static PAGE_CACHE_READ_ACCESSES: Lazy<IntCounterVec> = Lazy::new(|| {

 pub(crate) static PAGE_CACHE: Lazy<PageCacheMetrics> = Lazy::new(|| PageCacheMetrics {
    map: EnumMap::from_array(std::array::from_fn(|task_kind| {
-        let task_kind = TaskKind::from_usize(task_kind);
+        let task_kind = <TaskKind as enum_map::Enum>::from_usize(task_kind);
        let task_kind: &'static str = task_kind.into();
        EnumMap::from_array(std::array::from_fn(|content_kind| {
-            let content_kind = PageContentKind::from_usize(content_kind);
+            let content_kind = <PageContentKind as enum_map::Enum>::from_usize(content_kind);
            let content_kind: &'static str = content_kind.into();
            PageCacheMetricsForTaskKind {
                read_accesses_immutable: {
@@ -1366,7 +1365,10 @@ impl SmgrOpTimer {
    /// The first callers receives Some, subsequent ones None.
    ///
    /// See [`SmgrOpTimerState`] for more context.
-    pub(crate) fn observe_execution_end(&mut self, at: Instant) -> Option<SmgrOpFlushInProgress> {
+    pub(crate) fn observe_execution_end_flush_start(
+        &mut self,
+        at: Instant,
+    ) -> Option<SmgrOpFlushInProgress> {
        // NB: unlike the other observe_* methods, this one take()s.
        #[allow(clippy::question_mark)] // maintain similar code pattern.
        let Some(mut inner) = self.0.take() else {
@@ -1400,6 +1402,7 @@ impl SmgrOpTimer {
            ..
        } = inner;
        Some(SmgrOpFlushInProgress {
+            flush_started_at: at,
            global_micros: global_flush_in_progress_micros,
            per_timeline_micros: per_timeline_flush_in_progress_micros,
        })
@@ -1415,6 +1418,7 @@ impl SmgrOpTimer {
 /// add another `observe_*` method to [`SmgrOpTimer`], follow the existing pattern there,
 /// and remove this struct from the code base.
 pub(crate) struct SmgrOpFlushInProgress {
+    flush_started_at: Instant,
    global_micros: IntCounter,
    per_timeline_micros: IntCounter,
 }
@@ -1433,13 +1437,12 @@ impl Drop for SmgrOpTimer {
        self.observe_throttle_start(now);
        self.observe_throttle_done(ThrottleResult::NotThrottled { end: now });
        self.observe_execution_start(now);
-        let maybe_flush_timer = self.observe_execution_end(now);
-        drop(maybe_flush_timer);
+        self.observe_execution_end_flush_start(now);
    }
 }

 impl SmgrOpFlushInProgress {
-    pub(crate) async fn measure<Fut, O>(self, mut started_at: Instant, mut fut: Fut) -> O
+    pub(crate) async fn measure<Fut, O>(mut self, mut fut: Fut) -> O
    where
        Fut: std::future::Future<Output = O>,
    {
@@ -1451,12 +1454,12 @@ impl SmgrOpFlushInProgress {
        let mut observe_guard = scopeguard::guard(
            || {
                let now = Instant::now();
-                let elapsed = now - started_at;
+                let elapsed = now - self.flush_started_at;
                self.global_micros
                    .inc_by(u64::try_from(elapsed.as_micros()).unwrap());
                self.per_timeline_micros
                    .inc_by(u64::try_from(elapsed.as_micros()).unwrap());
-                started_at = now;
+                self.flush_started_at = now;
            },
            |mut observe| {
                observe();
@@ -1909,7 +1912,7 @@ pub(crate) static COMPUTE_COMMANDS_COUNTERS: Lazy<ComputeCommandCounters> = Lazy

    ComputeCommandCounters {
        map: EnumMap::from_array(std::array::from_fn(|i| {
-            let command = ComputeCommandKind::from_usize(i);
+            let command = <ComputeCommandKind as enum_map::Enum>::from_usize(i);
            let command_str: &'static str = command.into();
            inner.with_label_values(&[command_str])
        })),
@@ -2209,13 +2212,11 @@ pub(crate) static TENANT_TASK_EVENTS: Lazy<IntCounterVec> = Lazy::new(|| {

 pub struct BackgroundLoopSemaphoreMetrics {
    counters: EnumMap<BackgroundLoopKind, IntCounterPair>,
-    durations: EnumMap<BackgroundLoopKind, Histogram>,
-    waiting_tasks: EnumMap<BackgroundLoopKind, IntGauge>,
-    running_tasks: EnumMap<BackgroundLoopKind, IntGauge>,
+    durations: EnumMap<BackgroundLoopKind, Counter>,
 }

-pub(crate) static BACKGROUND_LOOP_SEMAPHORE: Lazy<BackgroundLoopSemaphoreMetrics> =
-    Lazy::new(|| {
+pub(crate) static BACKGROUND_LOOP_SEMAPHORE: Lazy<BackgroundLoopSemaphoreMetrics> = Lazy::new(
+    || {
        let counters = register_int_counter_pair_vec!(
            "pageserver_background_loop_semaphore_wait_start_count",
            "Counter for background loop concurrency-limiting semaphore acquire calls started",
@@ -2225,101 +2226,45 @@ pub(crate) static BACKGROUND_LOOP_SEMAPHORE: Lazy<BackgroundLoopSemaphoreMetrics
        )
        .unwrap();

-        let durations = register_histogram_vec!(
-            "pageserver_background_loop_semaphore_wait_seconds",
-            "Seconds spent waiting on background loop semaphore acquisition",
-            &["task"],
-            vec![0.01, 1.0, 5.0, 10.0, 30.0, 60.0, 180.0, 300.0, 600.0],
-        )
-        .unwrap();
-
-        let waiting_tasks = register_int_gauge_vec!(
-            "pageserver_background_loop_semaphore_waiting_tasks",
-            "Number of background loop tasks waiting for semaphore",
-            &["task"],
-        )
-        .unwrap();
-
-        let running_tasks = register_int_gauge_vec!(
-            "pageserver_background_loop_semaphore_running_tasks",
-            "Number of background loop tasks running concurrently",
+        let durations = register_counter_vec!(
+            "pageserver_background_loop_semaphore_wait_duration_seconds",
+            "Sum of wall clock time spent waiting on the background loop concurrency-limiting semaphore acquire calls",
            &["task"],
        )
        .unwrap();

        BackgroundLoopSemaphoreMetrics {
-            counters: EnumMap::from_array(std::array::from_fn(|i| {
-                let kind = BackgroundLoopKind::from_usize(i);
+            counters: enum_map::EnumMap::from_array(std::array::from_fn(|i| {
+                let kind = <BackgroundLoopKind as enum_map::Enum>::from_usize(i);
                counters.with_label_values(&[kind.into()])
            })),
-            durations: EnumMap::from_array(std::array::from_fn(|i| {
-                let kind = BackgroundLoopKind::from_usize(i);
+            durations: enum_map::EnumMap::from_array(std::array::from_fn(|i| {
+                let kind = <BackgroundLoopKind as enum_map::Enum>::from_usize(i);
                durations.with_label_values(&[kind.into()])
            })),
-            waiting_tasks: EnumMap::from_array(std::array::from_fn(|i| {
-                let kind = BackgroundLoopKind::from_usize(i);
-                waiting_tasks.with_label_values(&[kind.into()])
-            })),
-            running_tasks: EnumMap::from_array(std::array::from_fn(|i| {
-                let kind = BackgroundLoopKind::from_usize(i);
-                running_tasks.with_label_values(&[kind.into()])
-            })),
        }
-    });
+    },
+);

 impl BackgroundLoopSemaphoreMetrics {
-    /// Starts recording semaphore metrics. Call `acquired()` on the returned recorder when the
-    /// semaphore is acquired, and drop it when the task completes or is cancelled.
-    pub(crate) fn record(
-        &self,
-        task: BackgroundLoopKind,
-    ) -> BackgroundLoopSemaphoreMetricsRecorder {
-        BackgroundLoopSemaphoreMetricsRecorder::start(self, task)
-    }
-}
-
-/// Records metrics for a background task.
-pub struct BackgroundLoopSemaphoreMetricsRecorder<'a> {
-    metrics: &'a BackgroundLoopSemaphoreMetrics,
-    task: BackgroundLoopKind,
-    start: Instant,
-    wait_counter_guard: Option<metrics::IntCounterPairGuard>,
-}
-
-impl<'a> BackgroundLoopSemaphoreMetricsRecorder<'a> {
-    /// Starts recording semaphore metrics, by recording wait time and incrementing
-    /// `wait_start_count` and `waiting_tasks`.
-    fn start(metrics: &'a BackgroundLoopSemaphoreMetrics, task: BackgroundLoopKind) -> Self {
-        metrics.waiting_tasks[task].inc();
-        Self {
-            metrics,
-            task,
-            start: Instant::now(),
-            wait_counter_guard: Some(metrics.counters[task].guard()),
+    pub(crate) fn measure_acquisition(&self, task: BackgroundLoopKind) -> impl Drop + '_ {
+        struct Record<'a> {
+            metrics: &'a BackgroundLoopSemaphoreMetrics,
+            task: BackgroundLoopKind,
+            _counter_guard: metrics::IntCounterPairGuard,
+            start: Instant,
        }
-    }
-
-    /// Signals that the semaphore has been acquired, and updates relevant metrics.
-    pub fn acquired(&mut self) -> Duration {
-        let waited = self.start.elapsed();
-        self.wait_counter_guard.take().expect("already acquired");
-        self.metrics.durations[self.task].observe(waited.as_secs_f64());
-        self.metrics.waiting_tasks[self.task].dec();
-        self.metrics.running_tasks[self.task].inc();
-        waited
-    }
-}
-
-impl Drop for BackgroundLoopSemaphoreMetricsRecorder<'_> {
-    /// The task either completed or was cancelled.
-    fn drop(&mut self) {
-        if self.wait_counter_guard.take().is_some() {
-            // Waiting.
-            self.metrics.durations[self.task].observe(self.start.elapsed().as_secs_f64());
-            self.metrics.waiting_tasks[self.task].dec();
-        } else {
-            // Running.
-            self.metrics.running_tasks[self.task].dec();
+        impl Drop for Record<'_> {
+            fn drop(&mut self) {
+                let elapsed = self.start.elapsed().as_secs_f64();
+                self.metrics.durations[self.task].inc_by(elapsed);
+            }
+        }
+        Record {
+            metrics: self,
+            task,
+            _counter_guard: self.counters[task].guard(),
+            start: Instant::now(),
        }
    }
 }
@@ -2433,40 +2378,11 @@ pub(crate) struct WalIngestMetrics {
    pub(crate) records_observed: IntCounter,
    pub(crate) records_committed: IntCounter,
    pub(crate) records_filtered: IntCounter,
-    pub(crate) values_committed_metadata_images: IntCounter,
-    pub(crate) values_committed_metadata_deltas: IntCounter,
-    pub(crate) values_committed_data_images: IntCounter,
-    pub(crate) values_committed_data_deltas: IntCounter,
    pub(crate) gap_blocks_zeroed_on_rel_extend: IntCounter,
-}
-
-impl WalIngestMetrics {
-    pub(crate) fn inc_values_committed(&self, stats: &DatadirModificationStats) {
-        if stats.metadata_images > 0 {
-            self.values_committed_metadata_images
-                .inc_by(stats.metadata_images);
-        }
-        if stats.metadata_deltas > 0 {
-            self.values_committed_metadata_deltas
-                .inc_by(stats.metadata_deltas);
-        }
-        if stats.data_images > 0 {
-            self.values_committed_data_images.inc_by(stats.data_images);
-        }
-        if stats.data_deltas > 0 {
-            self.values_committed_data_deltas.inc_by(stats.data_deltas);
-        }
-    }
+    pub(crate) clear_vm_bits_unknown: IntCounterVec,
 }

 pub(crate) static WAL_INGEST: Lazy<WalIngestMetrics> = Lazy::new(|| {
-    let values_committed = register_int_counter_vec!(
-        "pageserver_wal_ingest_values_committed",
-        "Number of values committed to pageserver storage from WAL records",
-        &["class", "kind"],
-    )
-    .expect("failed to define a metric");
-
    WalIngestMetrics {
    bytes_received: register_int_counter!(
        "pageserver_wal_ingest_bytes_received",
@@ -2493,15 +2409,17 @@ pub(crate) static WAL_INGEST: Lazy<WalIngestMetrics> = Lazy::new(|| {
        "Number of WAL records filtered out due to sharding"
    )
    .expect("failed to define a metric"),
-    values_committed_metadata_images: values_committed.with_label_values(&["metadata", "image"]),
-    values_committed_metadata_deltas: values_committed.with_label_values(&["metadata", "delta"]),
-    values_committed_data_images: values_committed.with_label_values(&["data", "image"]),
-    values_committed_data_deltas: values_committed.with_label_values(&["data", "delta"]),
    gap_blocks_zeroed_on_rel_extend: register_int_counter!(
        "pageserver_gap_blocks_zeroed_on_rel_extend",
        "Total number of zero gap blocks written on relation extends"
    )
    .expect("failed to define a metric"),
+    clear_vm_bits_unknown: register_int_counter_vec!(
+        "pageserver_wal_ingest_clear_vm_bits_unknown",
+        "Number of ignored ClearVmBits operations due to unknown pages/relations",
+        &["entity"],
+    )
+    .expect("failed to define a metric"),
 }
 });

@@ -2568,7 +2486,7 @@ pub(crate) static WAL_REDO_PROCESS_LAUNCH_DURATION_HISTOGRAM: Lazy<Histogram> =

 pub(crate) struct WalRedoProcessCounters {
    pub(crate) started: IntCounter,
-    pub(crate) killed_by_cause: EnumMap<WalRedoKillCause, IntCounter>,
+    pub(crate) killed_by_cause: enum_map::EnumMap<WalRedoKillCause, IntCounter>,
    pub(crate) active_stderr_logger_tasks_started: IntCounter,
    pub(crate) active_stderr_logger_tasks_finished: IntCounter,
 }
@@ -2610,7 +2528,7 @@ impl Default for WalRedoProcessCounters {
        Self {
            started,
            killed_by_cause: EnumMap::from_array(std::array::from_fn(|i| {
-                let cause = WalRedoKillCause::from_usize(i);
+                let cause = <WalRedoKillCause as enum_map::Enum>::from_usize(i);
                let cause_str: &'static str = cause.into();
                killed.with_label_values(&[cause_str])
            })),
--- a/pageserver/src/page_service.rs
+++ b/pageserver/src/page_service.rs
@@ -489,6 +489,7 @@ impl timeline::handle::TenantManager<TenantManagerTypes> for TenantManagerWrappe
        let timeline = tenant_shard
            .get_timeline(timeline_id, true)
            .map_err(GetActiveTimelineError::Timeline)?;
+        set_tracing_field_shard_id(&timeline);
        Ok(timeline)
    }
 }
@@ -773,11 +774,11 @@ impl PageServerHandler {

        let batched_msg = match neon_fe_msg {
            PagestreamFeMessage::Exists(req) => {
+                let span = tracing::info_span!(parent: parent_span, "handle_get_rel_exists_request", rel = %req.rel, req_lsn = %req.hdr.request_lsn);
                let shard = timeline_handles
                    .get(tenant_id, timeline_id, ShardSelector::Zero)
+                    .instrument(span.clone()) // sets `shard_id` field
                    .await?;
-                debug_assert_current_span_has_tenant_and_timeline_id_no_shard_id();
-                let span = tracing::info_span!(parent: &parent_span, "handle_get_rel_exists_request", rel = %req.rel, req_lsn = %req.hdr.request_lsn, shard_id = %shard.tenant_shard_id.shard_slug());
                let timer = record_op_start_and_throttle(
                    &shard,
                    metrics::SmgrQueryType::GetRelExists,
@@ -792,10 +793,11 @@ impl PageServerHandler {
                }
            }
            PagestreamFeMessage::Nblocks(req) => {
+                let span = tracing::info_span!(parent: parent_span, "handle_get_nblocks_request", rel = %req.rel, req_lsn = %req.hdr.request_lsn);
                let shard = timeline_handles
                    .get(tenant_id, timeline_id, ShardSelector::Zero)
+                    .instrument(span.clone()) // sets `shard_id` field
                    .await?;
-                let span = tracing::info_span!(parent: &parent_span, "handle_get_nblocks_request", rel = %req.rel, req_lsn = %req.hdr.request_lsn, shard_id = %shard.tenant_shard_id.shard_slug());
                let timer = record_op_start_and_throttle(
                    &shard,
                    metrics::SmgrQueryType::GetRelSize,
@@ -810,10 +812,11 @@ impl PageServerHandler {
                }
            }
            PagestreamFeMessage::DbSize(req) => {
+                let span = tracing::info_span!(parent: parent_span, "handle_db_size_request", dbnode = %req.dbnode, req_lsn = %req.hdr.request_lsn);
                let shard = timeline_handles
                    .get(tenant_id, timeline_id, ShardSelector::Zero)
+                    .instrument(span.clone()) // sets `shard_id` field
                    .await?;
-                let span = tracing::info_span!(parent: &parent_span, "handle_db_size_request", dbnode = %req.dbnode, req_lsn = %req.hdr.request_lsn, shard_id = %shard.tenant_shard_id.shard_slug());
                let timer = record_op_start_and_throttle(
                    &shard,
                    metrics::SmgrQueryType::GetDbSize,
@@ -828,10 +831,11 @@ impl PageServerHandler {
                }
            }
            PagestreamFeMessage::GetSlruSegment(req) => {
+                let span = tracing::info_span!(parent: parent_span, "handle_get_slru_segment_request", kind = %req.kind, segno = %req.segno, req_lsn = %req.hdr.request_lsn);
                let shard = timeline_handles
                    .get(tenant_id, timeline_id, ShardSelector::Zero)
+                    .instrument(span.clone()) // sets `shard_id` field
                    .await?;
-                let span = tracing::info_span!(parent: &parent_span, "handle_get_slru_segment_request", kind = %req.kind, segno = %req.segno, req_lsn = %req.hdr.request_lsn, shard_id = %shard.tenant_shard_id.shard_slug());
                let timer = record_op_start_and_throttle(
                    &shard,
                    metrics::SmgrQueryType::GetSlruSegment,
@@ -846,20 +850,12 @@ impl PageServerHandler {
                }
            }
            PagestreamFeMessage::GetPage(req) => {
-                // avoid a somewhat costly Span::record() by constructing the entire span in one go.
-                macro_rules! mkspan {
-                    (before shard routing) => {{
-                        tracing::info_span!(parent: &parent_span, "handle_get_page_request", rel = %req.rel, blkno = %req.blkno, req_lsn = %req.hdr.request_lsn)
-                    }};
-                    ($shard_id:expr) => {{
-                        tracing::info_span!(parent: &parent_span, "handle_get_page_request", rel = %req.rel, blkno = %req.blkno, req_lsn = %req.hdr.request_lsn, shard_id = %$shard_id)
-                    }};
-                }
+                let span = tracing::info_span!(parent: parent_span, "handle_get_page_at_lsn_request_batched", req_lsn = %req.hdr.request_lsn);

                macro_rules! respond_error {
-                    ($span:expr, $error:expr) => {{
+                    ($error:expr) => {{
                        let error = BatchedFeMessage::RespondError {
-                            span: $span,
+                            span,
                            error: BatchedPageStreamError {
                                req: req.hdr,
                                err: $error,
@@ -872,35 +868,27 @@ impl PageServerHandler {
                let key = rel_block_to_key(req.rel, req.blkno);
                let shard = match timeline_handles
                    .get(tenant_id, timeline_id, ShardSelector::Page(key))
+                    .instrument(span.clone()) // sets `shard_id` field
                    .await
                {
                    Ok(tl) => tl,
+                    Err(GetActiveTimelineError::Tenant(GetActiveTenantError::NotFound(_))) => {
+                        // We already know this tenant exists in general, because we resolved it at
+                        // start of connection.  Getting a NotFound here indicates that the shard containing
+                        // the requested page is not present on this node: the client's knowledge of shard->pageserver
+                        // mapping is out of date.
+                        //
+                        // Closing the connection by returning ``::Reconnect` has the side effect of rate-limiting above message, via
+                        // client's reconnect backoff, as well as hopefully prompting the client to load its updated configuration
+                        // and talk to a different pageserver.
+                        return respond_error!(PageStreamError::Reconnect(
+                            "getpage@lsn request routed to wrong shard".into()
+                        ));
+                    }
                    Err(e) => {
-                        let span = mkspan!(before shard routing);
-                        match e {
-                            GetActiveTimelineError::Tenant(GetActiveTenantError::NotFound(_)) => {
-                                // We already know this tenant exists in general, because we resolved it at
-                                // start of connection.  Getting a NotFound here indicates that the shard containing
-                                // the requested page is not present on this node: the client's knowledge of shard->pageserver
-                                // mapping is out of date.
-                                //
-                                // Closing the connection by returning ``::Reconnect` has the side effect of rate-limiting above message, via
-                                // client's reconnect backoff, as well as hopefully prompting the client to load its updated configuration
-                                // and talk to a different pageserver.
-                                return respond_error!(
-                                    span,
-                                    PageStreamError::Reconnect(
-                                        "getpage@lsn request routed to wrong shard".into()
-                                    )
-                                );
-                            }
-                            e => {
-                                return respond_error!(span, e.into());
-                            }
-                        }
+                        return respond_error!(e.into());
                    }
                };
-                let span = mkspan!(shard.tenant_shard_id.shard_slug());

                let timer = record_op_start_and_throttle(
                    &shard,
@@ -914,7 +902,7 @@ impl PageServerHandler {
                    &shard,
                    req.hdr.request_lsn,
                    req.hdr.not_modified_since,
-                    &shard.get_applied_gc_cutoff_lsn(),
+                    &shard.get_latest_gc_cutoff_lsn(),
                    ctx,
                )
                // TODO: if we actually need to wait for lsn here, it delays the entire batch which doesn't need to wait
@@ -922,7 +910,7 @@ impl PageServerHandler {
                {
                    Ok(lsn) => lsn,
                    Err(e) => {
-                        return respond_error!(span, e);
+                        return respond_error!(e);
                    }
                };
                BatchedFeMessage::GetPage {
@@ -934,10 +922,11 @@ impl PageServerHandler {
            }
            #[cfg(feature = "testing")]
            PagestreamFeMessage::Test(req) => {
+                let span = tracing::info_span!(parent: parent_span, "handle_test_request");
                let shard = timeline_handles
                    .get(tenant_id, timeline_id, ShardSelector::Zero)
+                    .instrument(span.clone()) // sets `shard_id` field
                    .await?;
-                let span = tracing::info_span!(parent: &parent_span, "handle_test_request", shard_id = %shard.tenant_shard_id.shard_slug());
                let timer =
                    record_op_start_and_throttle(&shard, metrics::SmgrQueryType::Test, received_at)
                        .await?;
@@ -1074,7 +1063,7 @@ impl PageServerHandler {
        };

        // invoke handler function
-        let (mut handler_results, span): (
+        let (handler_results, span): (
            Vec<Result<(PagestreamBeMessage, SmgrOpTimer), BatchedPageStreamError>>,
            _,
        ) = match batch {
@@ -1201,49 +1190,11 @@ impl PageServerHandler {
            }
        };

-        // We purposefully don't count flush time into the smgr operation timer.
-        //
-        // The reason is that current compute client will not perform protocol processing
-        // if the postgres backend process is doing things other than `->smgr_read()`.
-        // This is especially the case for prefetch.
-        //
-        // If the compute doesn't read from the connection, eventually TCP will backpressure
-        // all the way into our flush call below.
-        //
-        // The timer's underlying metric is used for a storage-internal latency SLO and
-        // we don't want to include latency in it that we can't control.
-        // And as pointed out above, in this case, we don't control the time that flush will take.
-        //
-        // We put each response in the batch onto the wire in a separate pgb_writer.flush()
-        // call, which (all unmeasured) adds syscall overhead but reduces time to first byte
-        // and avoids building up a "giant" contiguous userspace buffer to hold the entire response.
-        // TODO: vectored socket IO would be great, but pgb_writer doesn't support that.
-        let flush_timers = {
-            let flushing_start_time = Instant::now();
-            let mut flush_timers = Vec::with_capacity(handler_results.len());
-            for handler_result in &mut handler_results {
-                let flush_timer = match handler_result {
-                    Ok((_, timer)) => Some(
-                        timer
-                            .observe_execution_end(flushing_start_time)
-                            .expect("we are the first caller"),
-                    ),
-                    Err(_) => {
-                        // TODO: measure errors
-                        None
-                    }
-                };
-                flush_timers.push(flush_timer);
-            }
-            assert_eq!(flush_timers.len(), handler_results.len());
-            flush_timers
-        };
-
        // Map handler result to protocol behavior.
        // Some handler errors cause exit from pagestream protocol.
        // Other handler errors are sent back as an error message and we stay in pagestream protocol.
-        for (handler_result, flushing_timer) in handler_results.into_iter().zip(flush_timers) {
-            let response_msg = match handler_result {
+        for handler_result in handler_results {
+            let (response_msg, timer) = match handler_result {
                Err(e) => match &e.err {
                    PageStreamError::Shutdown => {
                        // If we fail to fulfil a request during shutdown, which may be _because_ of
@@ -1267,14 +1218,16 @@ impl PageServerHandler {
                        span.in_scope(|| {
                            error!("error reading relation or page version: {full:#}")
                        });
-
-                        PagestreamBeMessage::Error(PagestreamErrorResponse {
-                            req: e.req,
-                            message: e.err.to_string(),
-                        })
+                        (
+                            PagestreamBeMessage::Error(PagestreamErrorResponse {
+                                req: e.req,
+                                message: e.err.to_string(),
+                            }),
+                            None, // TODO: measure errors
+                        )
                    }
                },
-                Ok((response_msg, _op_timer_already_observed)) => response_msg,
+                Ok((response_msg, timer)) => (response_msg, Some(timer)),
            };

            //
@@ -1285,12 +1238,30 @@ impl PageServerHandler {
                &response_msg.serialize(protocol_version),
            ))?;

+            // We purposefully don't count flush time into the timer.
+            //
+            // The reason is that current compute client will not perform protocol processing
+            // if the postgres backend process is doing things other than `->smgr_read()`.
+            // This is especially the case for prefetch.
+            //
+            // If the compute doesn't read from the connection, eventually TCP will backpressure
+            // all the way into our flush call below.
+            //
+            // The timer's underlying metric is used for a storage-internal latency SLO and
+            // we don't want to include latency in it that we can't control.
+            // And as pointed out above, in this case, we don't control the time that flush will take.
+            let flushing_timer = timer.map(|mut timer| {
+                timer
+                    .observe_execution_end_flush_start(Instant::now())
+                    .expect("we are the first caller")
+            });
+
            // what we want to do
            let flush_fut = pgb_writer.flush();
            // metric for how long flushing takes
            let flush_fut = match flushing_timer {
                Some(flushing_timer) => {
-                    futures::future::Either::Left(flushing_timer.measure(Instant::now(), flush_fut))
+                    futures::future::Either::Left(flushing_timer.measure(flush_fut))
                }
                None => futures::future::Either::Right(flush_fut),
            };
@@ -1309,6 +1280,8 @@ impl PageServerHandler {
                }
                Ok(())
            }
+            // and log the info! line inside the request span
+            .instrument(span.clone())
            .await?;
        }
        Ok(())
@@ -1369,7 +1342,7 @@ impl PageServerHandler {
            .take()
            .expect("implementation error: timeline_handles should not be locked");

-        let request_span = info_span!("request");
+        let request_span = info_span!("request", shard_id = tracing::field::Empty);
        let ((pgb_reader, timeline_handles), result) = match self.pipelining_config.clone() {
            PageServicePipeliningConfig::Pipelined(pipelining_config) => {
                self.handle_pagerequests_pipelined(
@@ -1719,7 +1692,7 @@ impl PageServerHandler {
        // to distinguish a misbehaving client (asking for old LSN) from a storage issue (data missing at a legitimate LSN).
        if request_lsn < **latest_gc_cutoff_lsn && !timeline.is_gc_blocked_by_lsn_lease_deadline() {
            let gc_info = &timeline.gc_info.read().unwrap();
-            if !gc_info.lsn_covered_by_lease(request_lsn) {
+            if !gc_info.leases.contains_key(&request_lsn) {
                return Err(
                    PageStreamError::BadRequest(format!(
                        "tried to request a page version that was garbage collected. requested at {} gc cutoff {}",
@@ -1810,7 +1783,7 @@ impl PageServerHandler {
        req: &PagestreamExistsRequest,
        ctx: &RequestContext,
    ) -> Result<PagestreamBeMessage, PageStreamError> {
-        let latest_gc_cutoff_lsn = timeline.get_applied_gc_cutoff_lsn();
+        let latest_gc_cutoff_lsn = timeline.get_latest_gc_cutoff_lsn();
        let lsn = Self::wait_or_get_last_lsn(
            timeline,
            req.hdr.request_lsn,
@@ -1837,7 +1810,7 @@ impl PageServerHandler {
        req: &PagestreamNblocksRequest,
        ctx: &RequestContext,
    ) -> Result<PagestreamBeMessage, PageStreamError> {
-        let latest_gc_cutoff_lsn = timeline.get_applied_gc_cutoff_lsn();
+        let latest_gc_cutoff_lsn = timeline.get_latest_gc_cutoff_lsn();
        let lsn = Self::wait_or_get_last_lsn(
            timeline,
            req.hdr.request_lsn,
@@ -1864,7 +1837,7 @@ impl PageServerHandler {
        req: &PagestreamDbSizeRequest,
        ctx: &RequestContext,
    ) -> Result<PagestreamBeMessage, PageStreamError> {
-        let latest_gc_cutoff_lsn = timeline.get_applied_gc_cutoff_lsn();
+        let latest_gc_cutoff_lsn = timeline.get_latest_gc_cutoff_lsn();
        let lsn = Self::wait_or_get_last_lsn(
            timeline,
            req.hdr.request_lsn,
@@ -1954,7 +1927,7 @@ impl PageServerHandler {
        req: &PagestreamGetSlruSegmentRequest,
        ctx: &RequestContext,
    ) -> Result<PagestreamBeMessage, PageStreamError> {
-        let latest_gc_cutoff_lsn = timeline.get_applied_gc_cutoff_lsn();
+        let latest_gc_cutoff_lsn = timeline.get_latest_gc_cutoff_lsn();
        let lsn = Self::wait_or_get_last_lsn(
            timeline,
            req.hdr.request_lsn,
@@ -2050,8 +2023,7 @@ impl PageServerHandler {
    {
        fn map_basebackup_error(err: BasebackupError) -> QueryError {
            match err {
-                // TODO: passthrough the error site to the final error message?
-                BasebackupError::Client(e, _) => QueryError::Disconnected(ConnectionError::Io(e)),
+                BasebackupError::Client(e) => QueryError::Disconnected(ConnectionError::Io(e)),
                BasebackupError::Server(e) => QueryError::Other(e),
            }
        }
@@ -2064,15 +2036,8 @@ impl PageServerHandler {
            .unwrap()
            .get(tenant_id, timeline_id, ShardSelector::Zero)
            .await?;
-        set_tracing_field_shard_id(&timeline);

-        if timeline.is_archived() == Some(true) {
-            // TODO after a grace period, turn this log line into a hard error
-            tracing::warn!("timeline {tenant_id}/{timeline_id} is archived, but got basebackup request for it.");
-            //return Err(QueryError::NotFound("timeline is archived".into()))
-        }
-
-        let latest_gc_cutoff_lsn = timeline.get_applied_gc_cutoff_lsn();
+        let latest_gc_cutoff_lsn = timeline.get_latest_gc_cutoff_lsn();
        if let Some(lsn) = lsn {
            // Backup was requested at a particular LSN. Wait for it to arrive.
            info!("waiting for {}", lsn);
@@ -2152,12 +2117,10 @@ impl PageServerHandler {
                .await
                .map_err(map_basebackup_error)?;
            }
-            writer.flush().await.map_err(|e| {
-                map_basebackup_error(BasebackupError::Client(
-                    e,
-                    "handle_basebackup_request,flush",
-                ))
-            })?;
+            writer
+                .flush()
+                .await
+                .map_err(|e| map_basebackup_error(BasebackupError::Client(e)))?;
        }

        pgb.write_message_noflush(&BeMessage::CopyDone)
--- a/pageserver/src/pgdatadir_mapping.rs
+++ b/pageserver/src/pgdatadir_mapping.rs
@@ -48,7 +48,7 @@ use tracing::{debug, trace, warn};
 use utils::bin_ser::DeserializeError;
 use utils::pausable_failpoint;
 use utils::{bin_ser::BeSer, lsn::Lsn};
-use wal_decoder::serialized_batch::{SerializedValueBatch, ValueMeta};
+use wal_decoder::serialized_batch::SerializedValueBatch;

 /// Max delta records appended to the AUX_FILES_KEY (for aux v1). The write path will write a full image once this threshold is reached.
 pub const MAX_AUX_FILE_DELTAS: usize = 1024;
@@ -611,19 +611,12 @@ impl Timeline {
    ) -> Result<LsnForTimestamp, PageReconstructError> {
        pausable_failpoint!("find-lsn-for-timestamp-pausable");

-        let gc_cutoff_lsn_guard = self.get_applied_gc_cutoff_lsn();
-        let gc_cutoff_planned = {
-            let gc_info = self.gc_info.read().unwrap();
-            gc_info.min_cutoff()
-        };
-        // Usually the planned cutoff is newer than the cutoff of the last gc run,
-        // but let's be defensive.
-        let gc_cutoff = gc_cutoff_planned.max(*gc_cutoff_lsn_guard);
+        let gc_cutoff_lsn_guard = self.get_latest_gc_cutoff_lsn();
        // We use this method to figure out the branching LSN for the new branch, but the
        // GC cutoff could be before the branching point and we cannot create a new branch
        // with LSN < `ancestor_lsn`. Thus, pick the maximum of these two to be
        // on the safe side.
-        let min_lsn = std::cmp::max(gc_cutoff, self.get_ancestor_lsn());
+        let min_lsn = std::cmp::max(*gc_cutoff_lsn_guard, self.get_ancestor_lsn());
        let max_lsn = self.get_last_record_lsn();

        // LSNs are always 8-byte aligned. low/mid/high represent the
@@ -1304,26 +1297,6 @@ impl DatadirModification<'_> {
            .is_some_and(|b| b.has_data())
    }

-    /// Returns statistics about the currently pending modifications.
-    pub(crate) fn stats(&self) -> DatadirModificationStats {
-        let mut stats = DatadirModificationStats::default();
-        for (_, _, value) in self.pending_metadata_pages.values().flatten() {
-            match value {
-                Value::Image(_) => stats.metadata_images += 1,
-                Value::WalRecord(r) if r.will_init() => stats.metadata_images += 1,
-                Value::WalRecord(_) => stats.metadata_deltas += 1,
-            }
-        }
-        for valuemeta in self.pending_data_batch.iter().flat_map(|b| &b.metadata) {
-            match valuemeta {
-                ValueMeta::Serialized(s) if s.will_init => stats.data_images += 1,
-                ValueMeta::Serialized(_) => stats.data_deltas += 1,
-                ValueMeta::Observed(_) => {}
-            }
-        }
-        stats
-    }
-
    /// Set the current lsn
    pub(crate) fn set_lsn(&mut self, lsn: Lsn) -> anyhow::Result<()> {
        ensure!(
@@ -2344,15 +2317,6 @@ impl DatadirModification<'_> {
    }
 }

-/// Statistics for a DatadirModification.
-#[derive(Default)]
-pub struct DatadirModificationStats {
-    pub metadata_images: u64,
-    pub metadata_deltas: u64,
-    pub data_images: u64,
-    pub data_deltas: u64,
-}
-
 /// This struct facilitates accessing either a committed key from the timeline at a
 /// specific LSN, or the latest uncommitted key from a pending modification.
 ///
--- a/pageserver/src/task_mgr.rs
+++ b/pageserver/src/task_mgr.rs
@@ -328,8 +328,8 @@ pub enum TaskKind {
    // Eviction. One per timeline.
    Eviction,

-    // Tenant housekeeping (flush idle ephemeral layers, shut down idle walredo, etc.).
-    TenantHousekeeping,
+    // Ingest housekeeping (flushing ephemeral layers on time threshold or disk pressure)
+    IngestHousekeeping,

    /// See [`crate::disk_usage_eviction_task`].
    DiskUsageEviction,
--- a/pageserver/src/tenant.rs
+++ b/pageserver/src/tenant.rs
@@ -20,7 +20,6 @@ use chrono::NaiveDateTime;
 use enumset::EnumSet;
 use futures::stream::FuturesUnordered;
 use futures::StreamExt;
-use itertools::Itertools as _;
 use pageserver_api::models;
 use pageserver_api::models::CompactInfoResponse;
 use pageserver_api::models::LsnLease;
@@ -40,8 +39,6 @@ use remote_timeline_client::manifest::{
 use remote_timeline_client::UploadQueueNotReadyError;
 use remote_timeline_client::FAILED_REMOTE_OP_RETRIES;
 use remote_timeline_client::FAILED_UPLOAD_WARN_THRESHOLD;
-use secondary::heatmap::HeatMapTenant;
-use secondary::heatmap::HeatMapTimeline;
 use std::collections::BTreeMap;
 use std::fmt;
 use std::future::Future;
@@ -54,14 +51,10 @@ use timeline::compaction::GcCompactionQueue;
 use timeline::import_pgdata;
 use timeline::offload::offload_timeline;
 use timeline::offload::OffloadError;
-use timeline::CompactFlags;
 use timeline::CompactOptions;
-use timeline::CompactionError;
-use timeline::PreviousHeatmap;
 use timeline::ShutdownMode;
 use tokio::io::BufReader;
 use tokio::sync::watch;
-use tokio::sync::Notify;
 use tokio::task::JoinSet;
 use tokio_util::sync::CancellationToken;
 use tracing::*;
@@ -265,7 +258,6 @@ struct TimelinePreload {
    timeline_id: TimelineId,
    client: RemoteTimelineClient,
    index_part: Result<MaybeDeletedIndexPart, DownloadError>,
-    previous_heatmap: Option<PreviousHeatmap>,
 }

 pub(crate) struct TenantPreload {
@@ -357,9 +349,6 @@ pub struct Tenant {
    /// Overhead of mutex is acceptable because compaction is done with a multi-second period.
    compaction_circuit_breaker: std::sync::Mutex<CircuitBreaker>,

-    /// Signals the tenant compaction loop that there is L0 compaction work to be done.
-    pub(crate) l0_compaction_trigger: Arc<Notify>,
-
    /// Scheduled gc-compaction tasks.
    scheduled_compaction_tasks: std::sync::Mutex<HashMap<TimelineId, Arc<GcCompactionQueue>>>,

@@ -1132,7 +1121,6 @@ impl Tenant {
        resources: TimelineResources,
        mut index_part: IndexPart,
        metadata: TimelineMetadata,
-        previous_heatmap: Option<PreviousHeatmap>,
        ancestor: Option<Arc<Timeline>>,
        cause: LoadTimelineCause,
        ctx: &RequestContext,
@@ -1163,7 +1151,6 @@ impl Tenant {
        let timeline = self.create_timeline_struct(
            timeline_id,
            &metadata,
-            previous_heatmap,
            ancestor.clone(),
            resources,
            CreateTimelineCause::Load,
@@ -1563,18 +1550,8 @@ impl Tenant {
            }
        }

-        // TODO(vlad): Could go to S3 if the secondary is freezing cold and hasn't even
-        // pulled the first heatmap. Not entirely necessary since the storage controller
-        // will kick the secondary in any case and cause a download.
-        let maybe_heatmap_at = self.read_on_disk_heatmap().await;
-
        let timelines = self
-            .load_timelines_metadata(
-                remote_timeline_ids,
-                remote_storage,
-                maybe_heatmap_at,
-                cancel,
-            )
+            .load_timelines_metadata(remote_timeline_ids, remote_storage, cancel)
            .await?;

        Ok(TenantPreload {
@@ -1587,26 +1564,6 @@ impl Tenant {
        })
    }

-    async fn read_on_disk_heatmap(&self) -> Option<(HeatMapTenant, std::time::Instant)> {
-        let on_disk_heatmap_path = self.conf.tenant_heatmap_path(&self.tenant_shard_id);
-        match tokio::fs::read_to_string(on_disk_heatmap_path).await {
-            Ok(heatmap) => match serde_json::from_str::<HeatMapTenant>(&heatmap) {
-                Ok(heatmap) => Some((heatmap, std::time::Instant::now())),
-                Err(err) => {
-                    error!("Failed to deserialize old heatmap: {err}");
-                    None
-                }
-            },
-            Err(err) => match err.kind() {
-                std::io::ErrorKind::NotFound => None,
-                _ => {
-                    error!("Unexpected IO error reading old heatmap: {err}");
-                    None
-                }
-            },
-        }
-    }
-
    ///
    /// Background task that downloads all data for a tenant and brings it to Active state.
    ///
@@ -1694,10 +1651,7 @@ impl Tenant {
            match index_part {
                MaybeDeletedIndexPart::IndexPart(index_part) => {
                    timeline_ancestors.insert(timeline_id, index_part.metadata.clone());
-                    remote_index_and_client.insert(
-                        timeline_id,
-                        (index_part, preload.client, preload.previous_heatmap),
-                    );
+                    remote_index_and_client.insert(timeline_id, (index_part, preload.client));
                }
                MaybeDeletedIndexPart::Deleted(index_part) => {
                    info!(
@@ -1716,7 +1670,7 @@ impl Tenant {
        // layer file.
        let sorted_timelines = tree_sort_timelines(timeline_ancestors, |m| m.ancestor_timeline())?;
        for (timeline_id, remote_metadata) in sorted_timelines {
-            let (index_part, remote_client, previous_heatmap) = remote_index_and_client
+            let (index_part, remote_client) = remote_index_and_client
                .remove(&timeline_id)
                .expect("just put it in above");

@@ -1736,8 +1690,12 @@ impl Tenant {
                    timeline_id,
                    index_part,
                    remote_metadata,
-                    previous_heatmap,
-                    self.get_timeline_resources_for(remote_client),
+                    TimelineResources {
+                        remote_client,
+                        pagestream_throttle: self.pagestream_throttle.clone(),
+                        pagestream_throttle_metrics: self.pagestream_throttle_metrics.clone(),
+                        l0_flush_global_state: self.l0_flush_global_state.clone(),
+                    },
                    LoadTimelineCause::Attach,
                    ctx,
                )
@@ -1886,13 +1844,11 @@ impl Tenant {
    }

    #[instrument(skip_all, fields(timeline_id=%timeline_id))]
-    #[allow(clippy::too_many_arguments)]
    async fn load_remote_timeline(
        self: &Arc<Self>,
        timeline_id: TimelineId,
        index_part: IndexPart,
        remote_metadata: TimelineMetadata,
-        previous_heatmap: Option<PreviousHeatmap>,
        resources: TimelineResources,
        cause: LoadTimelineCause,
        ctx: &RequestContext,
@@ -1922,7 +1878,6 @@ impl Tenant {
            resources,
            index_part,
            remote_metadata,
-            previous_heatmap,
            ancestor,
            cause,
            ctx,
@@ -1934,29 +1889,14 @@ impl Tenant {
        self: &Arc<Tenant>,
        timeline_ids: HashSet<TimelineId>,
        remote_storage: &GenericRemoteStorage,
-        heatmap: Option<(HeatMapTenant, std::time::Instant)>,
        cancel: CancellationToken,
    ) -> anyhow::Result<HashMap<TimelineId, TimelinePreload>> {
-        let mut timeline_heatmaps = heatmap.map(|h| (h.0.into_timelines_index(), h.1));
-
        let mut part_downloads = JoinSet::new();
        for timeline_id in timeline_ids {
            let cancel_clone = cancel.clone();
-
-            let previous_timeline_heatmap = timeline_heatmaps.as_mut().and_then(|hs| {
-                hs.0.remove(&timeline_id).map(|h| PreviousHeatmap::Active {
-                    heatmap: h,
-                    read_at: hs.1,
-                })
-            });
            part_downloads.spawn(
-                self.load_timeline_metadata(
-                    timeline_id,
-                    remote_storage.clone(),
-                    previous_timeline_heatmap,
-                    cancel_clone,
-                )
-                .instrument(info_span!("download_index_part", %timeline_id)),
+                self.load_timeline_metadata(timeline_id, remote_storage.clone(), cancel_clone)
+                    .instrument(info_span!("download_index_part", %timeline_id)),
            );
        }

@@ -2004,7 +1944,6 @@ impl Tenant {
        self: &Arc<Tenant>,
        timeline_id: TimelineId,
        remote_storage: GenericRemoteStorage,
-        previous_heatmap: Option<PreviousHeatmap>,
        cancel: CancellationToken,
    ) -> impl Future<Output = TimelinePreload> {
        let client = self.build_timeline_client(timeline_id, remote_storage);
@@ -2020,7 +1959,6 @@ impl Tenant {
                client,
                timeline_id,
                index_part,
-                previous_heatmap,
            }
        }
    }
@@ -2132,12 +2070,7 @@ impl Tenant {
            })?;

        let timeline_preload = self
-            .load_timeline_metadata(
-                timeline_id,
-                self.remote_storage.clone(),
-                None,
-                cancel.clone(),
-            )
+            .load_timeline_metadata(timeline_id, self.remote_storage.clone(), cancel.clone())
            .await;

        let index_part = match timeline_preload.index_part {
@@ -2171,7 +2104,6 @@ impl Tenant {
            timeline_id,
            index_part,
            remote_metadata,
-            None,
            timeline_resources,
            LoadTimelineCause::Unoffload,
            &ctx,
@@ -2887,7 +2819,7 @@ impl Tenant {
        };
        let metadata = index_part.metadata.clone();
        self
-            .load_remote_timeline(timeline_id, index_part, metadata, None, resources, LoadTimelineCause::ImportPgdata{
+            .load_remote_timeline(timeline_id, index_part, metadata, resources, LoadTimelineCause::ImportPgdata{
                create_guard: timeline_create_guard, activate, }, &ctx)
            .await?
            .ready_to_activate()
@@ -2966,194 +2898,150 @@ impl Tenant {
            .await
    }

-    /// Performs one compaction iteration. Called periodically from the compaction loop. Returns
-    /// whether another compaction is needed, if we still have pending work or if we yield for
-    /// immediate L0 compaction.
+    /// Perform one compaction iteration.
+    /// This function is periodically called by compactor task.
+    /// Also it can be explicitly requested per timeline through page server
+    /// api's 'compact' command.
    ///
-    /// Compaction can also be explicitly requested for a timeline via the HTTP API.
+    /// Returns whether we have pending compaction task.
    async fn compaction_iteration(
        self: &Arc<Self>,
        cancel: &CancellationToken,
        ctx: &RequestContext,
-    ) -> Result<CompactionOutcome, CompactionError> {
-        // Don't compact inactive tenants.
+    ) -> Result<CompactionOutcome, timeline::CompactionError> {
+        // Don't start doing work during shutdown, or when broken, we do not need those in the logs
        if !self.is_active() {
-            return Ok(CompactionOutcome::Skipped);
+            return Ok(CompactionOutcome::Done);
        }

-        // Don't compact tenants that can't upload layers. We don't check `may_delete_layers_hint`,
-        // since we need to compact L0 even in AttachedMulti to bound read amplification.
-        let location = self.tenant_conf.load().location;
-        if !location.may_upload_layers_hint() {
-            info!("skipping compaction in location state {location:?}");
-            return Ok(CompactionOutcome::Skipped);
-        }
-
-        // Don't compact if the circuit breaker is tripped.
-        if self.compaction_circuit_breaker.lock().unwrap().is_broken() {
-            info!("skipping compaction due to previous failures");
-            return Ok(CompactionOutcome::Skipped);
-        }
-
-        // Collect all timelines to compact, along with offload instructions and L0 counts.
-        let mut compact: Vec<Arc<Timeline>> = Vec::new();
-        let mut offload: HashSet<TimelineId> = HashSet::new();
-        let mut l0_counts: HashMap<TimelineId, usize> = HashMap::new();
-
        {
-            let offload_enabled = self.get_timeline_offloading_enabled();
+            let conf = self.tenant_conf.load();
+
+            // Note that compaction usually requires deletions, but we don't respect
+            // may_delete_layers_hint here: that is because tenants in AttachedMulti
+            // should proceed with compaction even if they can't do deletion, to avoid
+            // accumulating dangerously deep stacks of L0 layers.  Deletions will be
+            // enqueued inside RemoteTimelineClient, and executed layer if/when we transition
+            // to AttachedSingle state.
+            if !conf.location.may_upload_layers_hint() {
+                info!("Skipping compaction in location state {:?}", conf.location);
+                return Ok(CompactionOutcome::Done);
+            }
+        }
+
+        // Scan through the hashmap and collect a list of all the timelines,
+        // while holding the lock. Then drop the lock and actually perform the
+        // compactions.  We don't want to block everything else while the
+        // compaction runs.
+        let timelines_to_compact_or_offload;
+        {
            let timelines = self.timelines.lock().unwrap();
-            for (&timeline_id, timeline) in timelines.iter() {
-                // Skip inactive timelines.
-                if !timeline.is_active() {
-                    continue;
-                }
-
-                // Schedule the timeline for compaction.
-                compact.push(timeline.clone());
-
-                // Schedule the timeline for offloading if eligible.
-                let can_offload = offload_enabled
-                    && timeline.can_offload().0
-                    && !timelines
-                        .iter()
-                        .any(|(_, tli)| tli.get_ancestor_timeline_id() == Some(timeline_id));
-                if can_offload {
-                    offload.insert(timeline_id);
-                }
-            }
-        } // release timelines lock
-
-        for timeline in &compact {
-            // Collect L0 counts. Can't await while holding lock above.
-            if let Ok(lm) = timeline.layers.read().await.layer_map() {
-                l0_counts.insert(timeline.timeline_id, lm.level0_deltas().len());
-            }
-        }
-
-        // Pass 1: L0 compaction across all timelines, in order of L0 count. We prioritize this to
-        // bound read amplification.
-        //
-        // TODO: this may spin on one or more ingest-heavy timelines, starving out image/GC
-        // compaction and offloading. We leave that as a potential problem to solve later. Consider
-        // splitting L0 and image/GC compaction to separate background jobs.
-        if self.get_compaction_l0_first() {
-            let compaction_threshold = self.get_compaction_threshold();
-            let compact_l0 = compact
+            timelines_to_compact_or_offload = timelines
                .iter()
-                .map(|tli| (tli, l0_counts.get(&tli.timeline_id).copied().unwrap_or(0)))
-                .filter(|&(_, l0)| l0 >= compaction_threshold)
-                .sorted_by_key(|&(_, l0)| l0)
-                .rev()
-                .map(|(tli, _)| tli.clone())
-                .collect_vec();
-
-            let mut has_pending_l0 = false;
-            for timeline in compact_l0 {
-                let outcome = timeline
-                    .compact(cancel, CompactFlags::OnlyL0Compaction.into(), ctx)
-                    .instrument(info_span!("compact_timeline", timeline_id = %timeline.timeline_id))
-                    .await
-                    .inspect_err(|err| self.maybe_trip_compaction_breaker(err))?;
-                match outcome {
-                    CompactionOutcome::Done => {}
-                    CompactionOutcome::Skipped => {}
-                    CompactionOutcome::Pending => has_pending_l0 = true,
-                    CompactionOutcome::YieldForL0 => has_pending_l0 = true,
-                }
-            }
-            if has_pending_l0 {
-                return Ok(CompactionOutcome::YieldForL0); // do another pass
-            }
+                .filter_map(|(timeline_id, timeline)| {
+                    let (is_active, (can_offload, _)) =
+                        (timeline.is_active(), timeline.can_offload());
+                    let has_no_unoffloaded_children = {
+                        !timelines
+                            .iter()
+                            .any(|(_id, tl)| tl.get_ancestor_timeline_id() == Some(*timeline_id))
+                    };
+                    let config_allows_offload = self.conf.timeline_offloading
+                        || self
+                            .tenant_conf
+                            .load()
+                            .tenant_conf
+                            .timeline_offloading
+                            .unwrap_or_default();
+                    let can_offload =
+                        can_offload && has_no_unoffloaded_children && config_allows_offload;
+                    if (is_active, can_offload) == (false, false) {
+                        None
+                    } else {
+                        Some((*timeline_id, timeline.clone(), (is_active, can_offload)))
+                    }
+                })
+                .collect::<Vec<_>>();
+            drop(timelines);
        }

-        // Pass 2: image compaction and timeline offloading. If any timelines have accumulated
-        // more L0 layers, they may also be compacted here.
-        //
-        // NB: image compaction may yield if there is pending L0 compaction.
-        //
-        // TODO: it will only yield if there is pending L0 compaction on the same timeline. If a
-        // different timeline needs compaction, it won't. It should check `l0_compaction_trigger`.
-        // We leave this for a later PR.
-        //
-        // TODO: consider ordering timelines by some priority, e.g. time since last full compaction,
-        // amount of L1 delta debt or garbage, offload-eligible timelines first, etc.
-        let mut has_pending = false;
-        for timeline in compact {
-            if !timeline.is_active() {
-                continue;
-            }
+        // Before doing any I/O work, check our circuit breaker
+        if self.compaction_circuit_breaker.lock().unwrap().is_broken() {
+            info!("Skipping compaction due to previous failures");
+            return Ok(CompactionOutcome::Done);
+        }

-            let mut outcome = timeline
-                .compact(cancel, EnumSet::default(), ctx)
-                .instrument(info_span!("compact_timeline", timeline_id = %timeline.timeline_id))
-                .await
-                .inspect_err(|err| self.maybe_trip_compaction_breaker(err))?;
+        let mut has_pending_task = false;

-            // If we're done compacting, check the scheduled GC compaction queue for more work.
-            if outcome == CompactionOutcome::Done {
-                let queue = self
-                    .scheduled_compaction_tasks
-                    .lock()
-                    .unwrap()
-                    .get(&timeline.timeline_id)
-                    .cloned();
-                if let Some(queue) = queue {
-                    outcome = queue
-                        .iteration(cancel, ctx, &self.gc_block, &timeline)
-                        .await?;
-                }
-            }
-
-            // If we're done compacting, offload the timeline if requested.
-            if outcome == CompactionOutcome::Done && offload.contains(&timeline.timeline_id) {
-                pausable_failpoint!("before-timeline-auto-offload");
-                offload_timeline(self, &timeline)
-                    .instrument(info_span!("offload_timeline", timeline_id = %timeline.timeline_id))
+        for (timeline_id, timeline, (can_compact, can_offload)) in &timelines_to_compact_or_offload
+        {
+            // pending_task_left == None: cannot compact, maybe still pending tasks
+            // pending_task_left == Some(Pending): compaction task left
+            // pending_task_left == Some(Done): no compaction task left
+            let pending_task_left = if *can_compact {
+                let compaction_outcome = timeline
+                    .compact(cancel, EnumSet::empty(), ctx)
+                    .instrument(info_span!("compact_timeline", %timeline_id))
                    .await
-                    .or_else(|err| match err {
-                        // Ignore this, we likely raced with unarchival.
-                        OffloadError::NotArchived => Ok(()),
-                        err => Err(err),
+                    .inspect_err(|e| match e {
+                        timeline::CompactionError::ShuttingDown => (),
+                        timeline::CompactionError::Offload(_) => {
+                            // Failures to offload timelines do not trip the circuit breaker, because
+                            // they do not do lots of writes the way compaction itself does: it is cheap
+                            // to retry, and it would be bad to stop all compaction because of an issue with offloading.
+                        }
+                        timeline::CompactionError::Other(e) => {
+                            self.compaction_circuit_breaker
+                                .lock()
+                                .unwrap()
+                                .fail(&CIRCUIT_BREAKERS_BROKEN, e);
+                        }
                    })?;
-            }
-
-            match outcome {
-                CompactionOutcome::Done => {}
-                CompactionOutcome::Skipped => {}
-                CompactionOutcome::Pending => has_pending = true,
-                // This mostly makes sense when the L0-only pass above is enabled, since there's
-                // otherwise no guarantee that we'll start with the timeline that has high L0.
-                CompactionOutcome::YieldForL0 => return Ok(CompactionOutcome::YieldForL0),
+                if let CompactionOutcome::Pending = compaction_outcome {
+                    Some(CompactionOutcome::Pending)
+                } else {
+                    let queue = {
+                        let guard = self.scheduled_compaction_tasks.lock().unwrap();
+                        guard.get(timeline_id).cloned()
+                    };
+                    if let Some(queue) = queue {
+                        let outcome = queue
+                            .iteration(cancel, ctx, &self.gc_block, timeline)
+                            .await?;
+                        Some(outcome)
+                    } else {
+                        Some(CompactionOutcome::Done)
+                    }
+                }
+            } else {
+                None
+            };
+            has_pending_task |= pending_task_left == Some(CompactionOutcome::Pending);
+            if pending_task_left == Some(CompactionOutcome::Done) && *can_offload {
+                pausable_failpoint!("before-timeline-auto-offload");
+                match offload_timeline(self, timeline)
+                    .instrument(info_span!("offload_timeline", %timeline_id))
+                    .await
+                {
+                    Err(OffloadError::NotArchived) => {
+                        // Ignore this, we likely raced with unarchival
+                        Ok(())
+                    }
+                    other => other,
+                }?;
            }
        }

-        // Success! Untrip the breaker if necessary.
        self.compaction_circuit_breaker
            .lock()
            .unwrap()
            .success(&CIRCUIT_BREAKERS_UNBROKEN);

-        match has_pending {
-            true => Ok(CompactionOutcome::Pending),
-            false => Ok(CompactionOutcome::Done),
-        }
-    }
-
-    /// Trips the compaction circuit breaker if appropriate.
-    pub(crate) fn maybe_trip_compaction_breaker(&self, err: &CompactionError) {
-        match err {
-            CompactionError::ShuttingDown => (),
-            // Offload failures don't trip the circuit breaker, since they're cheap to retry and
-            // shouldn't block compaction.
-            CompactionError::Offload(_) => {}
-            CompactionError::Other(err) => {
-                self.compaction_circuit_breaker
-                    .lock()
-                    .unwrap()
-                    .fail(&CIRCUIT_BREAKERS_BROKEN, err);
-            }
-        }
+        Ok(if has_pending_task {
+            CompactionOutcome::Pending
+        } else {
+            CompactionOutcome::Done
+        })
    }

    /// Cancel scheduled compaction tasks
@@ -3200,28 +3088,32 @@ impl Tenant {
        Ok(rx)
    }

-    /// Performs periodic housekeeping, via the tenant housekeeping background task.
-    async fn housekeeping(&self) {
-        // Call through to all timelines to freeze ephemeral layers as needed. This usually happens
-        // during ingest, but we don't want idle timelines to hold open layers for too long.
-        let timelines = self
-            .timelines
-            .lock()
-            .unwrap()
-            .values()
-            .filter(|tli| tli.is_active())
-            .cloned()
-            .collect_vec();
+    // Call through to all timelines to freeze ephemeral layers if needed.  Usually
+    // this happens during ingest: this background housekeeping is for freezing layers
+    // that are open but haven't been written to for some time.
+    async fn ingest_housekeeping(&self) {
+        // Scan through the hashmap and collect a list of all the timelines,
+        // while holding the lock. Then drop the lock and actually perform the
+        // compactions.  We don't want to block everything else while the
+        // compaction runs.
+        let timelines = {
+            self.timelines
+                .lock()
+                .unwrap()
+                .values()
+                .filter_map(|timeline| {
+                    if timeline.is_active() {
+                        Some(timeline.clone())
+                    } else {
+                        None
+                    }
+                })
+                .collect::<Vec<_>>()
+        };

-        for timeline in timelines {
+        for timeline in &timelines {
            timeline.maybe_freeze_ephemeral_layer().await;
        }
-
-        // Shut down walredo if idle.
-        const WALREDO_IDLE_TIMEOUT: Duration = Duration::from_secs(180);
-        if let Some(ref walredo_mgr) = self.walredo_mgr {
-            walredo_mgr.maybe_quiesce(WALREDO_IDLE_TIMEOUT);
-        }
    }

    pub fn timeline_has_no_attached_children(&self, timeline_id: TimelineId) -> bool {
@@ -3931,13 +3823,6 @@ impl Tenant {
            .unwrap_or(self.conf.default_tenant_conf.compaction_upper_limit)
    }

-    pub fn get_compaction_l0_first(&self) -> bool {
-        let tenant_conf = self.tenant_conf.load().tenant_conf.clone();
-        tenant_conf
-            .compaction_l0_first
-            .unwrap_or(self.conf.default_tenant_conf.compaction_l0_first)
-    }
-
    pub fn get_gc_horizon(&self) -> u64 {
        let tenant_conf = self.tenant_conf.load().tenant_conf.clone();
        tenant_conf
@@ -3992,16 +3877,6 @@ impl Tenant {
            .unwrap_or(self.conf.default_tenant_conf.lsn_lease_length)
    }

-    pub fn get_timeline_offloading_enabled(&self) -> bool {
-        if self.conf.timeline_offloading {
-            return true;
-        }
-        let tenant_conf = self.tenant_conf.load().tenant_conf.clone();
-        tenant_conf
-            .timeline_offloading
-            .unwrap_or(self.conf.default_tenant_conf.timeline_offloading)
-    }
-
    /// Generate an up-to-date TenantManifest based on the state of this Tenant.
    fn build_tenant_manifest(&self) -> TenantManifest {
        let timelines_offloaded = self.timelines_offloaded.lock().unwrap();
@@ -4096,7 +3971,6 @@ impl Tenant {
        &self,
        new_timeline_id: TimelineId,
        new_metadata: &TimelineMetadata,
-        previous_heatmap: Option<PreviousHeatmap>,
        ancestor: Option<Arc<Timeline>>,
        resources: TimelineResources,
        cause: CreateTimelineCause,
@@ -4120,7 +3994,6 @@ impl Tenant {
            self.conf,
            Arc::clone(&self.tenant_conf),
            new_metadata,
-            previous_heatmap,
            ancestor,
            new_timeline_id,
            self.tenant_shard_id,
@@ -4242,7 +4115,6 @@ impl Tenant {
                // use an extremely long backoff.
                Some(Duration::from_secs(3600 * 24)),
            )),
-            l0_compaction_trigger: Arc::new(Notify::new()),
            scheduled_compaction_tasks: Mutex::new(Default::default()),
            activate_now_sem: tokio::sync::Semaphore::new(0),
            attach_wal_lag_cooldown: Arc::new(std::sync::OnceLock::new()),
@@ -4763,33 +4635,29 @@ impl Tenant {
        // We check it against both the planned GC cutoff stored in 'gc_info',
        // and the 'latest_gc_cutoff' of the last GC that was performed.  The
        // planned GC cutoff in 'gc_info' is normally larger than
-        // 'applied_gc_cutoff_lsn', but beware of corner cases like if you just
+        // 'latest_gc_cutoff_lsn', but beware of corner cases like if you just
        // changed the GC settings for the tenant to make the PITR window
        // larger, but some of the data was already removed by an earlier GC
        // iteration.

        // check against last actual 'latest_gc_cutoff' first
-        let applied_gc_cutoff_lsn = src_timeline.get_applied_gc_cutoff_lsn();
+        let latest_gc_cutoff_lsn = src_timeline.get_latest_gc_cutoff_lsn();
+        src_timeline
+            .check_lsn_is_in_scope(start_lsn, &latest_gc_cutoff_lsn)
+            .context(format!(
+                "invalid branch start lsn: less than latest GC cutoff {}",
+                *latest_gc_cutoff_lsn,
+            ))
+            .map_err(CreateTimelineError::AncestorLsn)?;
+
+        // and then the planned GC cutoff
        {
            let gc_info = src_timeline.gc_info.read().unwrap();
-            let planned_cutoff = gc_info.min_cutoff();
-            if gc_info.lsn_covered_by_lease(start_lsn) {
-                tracing::info!("skipping comparison of {start_lsn} with gc cutoff {} and planned gc cutoff {planned_cutoff} due to lsn lease", *applied_gc_cutoff_lsn);
-            } else {
-                src_timeline
-                    .check_lsn_is_in_scope(start_lsn, &applied_gc_cutoff_lsn)
-                    .context(format!(
-                        "invalid branch start lsn: less than latest GC cutoff {}",
-                        *applied_gc_cutoff_lsn,
-                    ))
-                    .map_err(CreateTimelineError::AncestorLsn)?;
-
-                // and then the planned GC cutoff
-                if start_lsn < planned_cutoff {
-                    return Err(CreateTimelineError::AncestorLsn(anyhow::anyhow!(
-                        "invalid branch start lsn: less than planned GC cutoff {planned_cutoff}"
-                    )));
-                }
+            let cutoff = gc_info.min_cutoff();
+            if start_lsn < cutoff {
+                return Err(CreateTimelineError::AncestorLsn(anyhow::anyhow!(
+                    "invalid branch start lsn: less than planned GC cutoff {cutoff}"
+                )));
            }
        }

@@ -4819,7 +4687,7 @@ impl Tenant {
            dst_prev,
            Some(src_id),
            start_lsn,
-            *src_timeline.applied_gc_cutoff_lsn.read(), // FIXME: should we hold onto this guard longer?
+            *src_timeline.latest_gc_cutoff_lsn.read(), // FIXME: should we hold onto this guard longer?
            src_timeline.initdb_lsn,
            src_timeline.pg_version,
        );
@@ -5151,19 +5019,12 @@ impl Tenant {
        )
    }

-    /// Builds required resources for a new timeline.
+    /// Call this before constructing a timeline, to build its required structures
    fn build_timeline_resources(&self, timeline_id: TimelineId) -> TimelineResources {
-        let remote_client = self.build_timeline_remote_client(timeline_id);
-        self.get_timeline_resources_for(remote_client)
-    }
-
-    /// Builds timeline resources for the given remote client.
-    fn get_timeline_resources_for(&self, remote_client: RemoteTimelineClient) -> TimelineResources {
        TimelineResources {
-            remote_client,
+            remote_client: self.build_timeline_remote_client(timeline_id),
            pagestream_throttle: self.pagestream_throttle.clone(),
            pagestream_throttle_metrics: self.pagestream_throttle_metrics.clone(),
-            l0_compaction_trigger: self.l0_compaction_trigger.clone(),
            l0_flush_global_state: self.l0_flush_global_state.clone(),
        }
    }
@@ -5192,7 +5053,6 @@ impl Tenant {
            .create_timeline_struct(
                new_timeline_id,
                new_metadata,
-                None,
                ancestor,
                resources,
                CreateTimelineCause::Load,
@@ -5610,8 +5470,6 @@ pub(crate) mod harness {
                compaction_threshold: Some(tenant_conf.compaction_threshold),
                compaction_upper_limit: Some(tenant_conf.compaction_upper_limit),
                compaction_algorithm: Some(tenant_conf.compaction_algorithm),
-                compaction_l0_first: Some(tenant_conf.compaction_l0_first),
-                compaction_l0_semaphore: Some(tenant_conf.compaction_l0_semaphore),
                l0_flush_delay_threshold: tenant_conf.l0_flush_delay_threshold,
                l0_flush_stall_threshold: tenant_conf.l0_flush_stall_threshold,
                l0_flush_wait_upload: Some(tenant_conf.l0_flush_wait_upload),
@@ -6199,8 +6057,8 @@ mod tests {
        make_some_layers(tline.as_ref(), Lsn(0x20), &ctx).await?;

        repo.gc_iteration(Some(TIMELINE_ID), 0x10, Duration::ZERO)?;
-        let applied_gc_cutoff_lsn = tline.get_applied_gc_cutoff_lsn();
-        assert!(*applied_gc_cutoff_lsn > Lsn(0x25));
+        let latest_gc_cutoff_lsn = tline.get_latest_gc_cutoff_lsn();
+        assert!(*latest_gc_cutoff_lsn > Lsn(0x25));
        match tline.get(*TEST_KEY, Lsn(0x25)) {
            Ok(_) => panic!("request for page should have failed"),
            Err(err) => assert!(err.to_string().contains("not found at")),
@@ -7839,18 +7697,6 @@ mod tests {
            }

            tline.freeze_and_flush().await?;
-            // Force layers to L1
-            tline
-                .compact(
-                    &cancel,
-                    {
-                        let mut flags = EnumSet::new();
-                        flags.insert(CompactFlags::ForceL0Compaction);
-                        flags
-                    },
-                    &ctx,
-                )
-                .await?;

            if iter % 5 == 0 {
                let (_, before_delta_file_accessed) =
@@ -7863,7 +7709,6 @@ mod tests {
                            let mut flags = EnumSet::new();
                            flags.insert(CompactFlags::ForceImageLayerCreation);
                            flags.insert(CompactFlags::ForceRepartition);
-                            flags.insert(CompactFlags::ForceL0Compaction);
                            flags
                        },
                        &ctx,
@@ -8310,8 +8155,6 @@ mod tests {

        let cancel = CancellationToken::new();

-        // Image layer creation happens on the disk_consistent_lsn so we need to force set it now.
-        tline.force_set_disk_consistent_lsn(Lsn(0x40));
        tline
            .compact(
                &cancel,
@@ -8325,7 +8168,8 @@ mod tests {
            )
            .await
            .unwrap();
-        // Image layers are created at repartition LSN
+
+        // Image layers are created at last_record_lsn
        let images = tline
            .inspect_image_layers(Lsn(0x40), &ctx, io_concurrency.clone())
            .await
@@ -8496,7 +8340,7 @@ mod tests {
            .await?;
        {
            tline
-                .applied_gc_cutoff_lsn
+                .latest_gc_cutoff_lsn
                .lock_for_write()
                .store_and_unlock(Lsn(0x30))
                .wait()
@@ -8604,7 +8448,7 @@ mod tests {
        // increase GC horizon and compact again
        {
            tline
-                .applied_gc_cutoff_lsn
+                .latest_gc_cutoff_lsn
                .lock_for_write()
                .store_and_unlock(Lsn(0x40))
                .wait()
@@ -8772,8 +8616,8 @@ mod tests {

        // Force set disk consistent lsn so we can get the cutoff at `end_lsn`.
        info!(
-            "applied_gc_cutoff_lsn: {}",
-            *timeline.get_applied_gc_cutoff_lsn()
+            "latest_gc_cutoff_lsn: {}",
+            *timeline.get_latest_gc_cutoff_lsn()
        );
        timeline.force_set_disk_consistent_lsn(end_lsn);

@@ -8799,7 +8643,7 @@ mod tests {

        // Make lease on a already GC-ed LSN.
        // 0/80 does not have a valid lease + is below latest_gc_cutoff
-        assert!(Lsn(0x80) < *timeline.get_applied_gc_cutoff_lsn());
+        assert!(Lsn(0x80) < *timeline.get_latest_gc_cutoff_lsn());
        timeline
            .init_lsn_lease(Lsn(0x80), timeline.get_lsn_lease_length(), &ctx)
            .expect_err("lease request on GC-ed LSN should fail");
@@ -8990,7 +8834,7 @@ mod tests {
        };
        {
            tline
-                .applied_gc_cutoff_lsn
+                .latest_gc_cutoff_lsn
                .lock_for_write()
                .store_and_unlock(Lsn(0x30))
                .wait()
@@ -9077,7 +8921,7 @@ mod tests {
        // increase GC horizon and compact again
        {
            tline
-                .applied_gc_cutoff_lsn
+                .latest_gc_cutoff_lsn
                .lock_for_write()
                .store_and_unlock(Lsn(0x40))
                .wait()
@@ -9530,7 +9374,7 @@ mod tests {
            .await?;
        {
            tline
-                .applied_gc_cutoff_lsn
+                .latest_gc_cutoff_lsn
                .lock_for_write()
                .store_and_unlock(Lsn(0x30))
                .wait()
@@ -9677,7 +9521,7 @@ mod tests {
        // increase GC horizon and compact again
        {
            tline
-                .applied_gc_cutoff_lsn
+                .latest_gc_cutoff_lsn
                .lock_for_write()
                .store_and_unlock(Lsn(0x38))
                .wait()
@@ -9778,7 +9622,7 @@ mod tests {
            .await?;
        {
            tline
-                .applied_gc_cutoff_lsn
+                .latest_gc_cutoff_lsn
                .lock_for_write()
                .store_and_unlock(Lsn(0x30))
                .wait()
@@ -10029,7 +9873,7 @@ mod tests {

        {
            parent_tline
-                .applied_gc_cutoff_lsn
+                .latest_gc_cutoff_lsn
                .lock_for_write()
                .store_and_unlock(Lsn(0x10))
                .wait()
@@ -10049,7 +9893,7 @@ mod tests {

        {
            branch_tline
-                .applied_gc_cutoff_lsn
+                .latest_gc_cutoff_lsn
                .lock_for_write()
                .store_and_unlock(Lsn(0x50))
                .wait()
@@ -10405,7 +10249,7 @@ mod tests {

        {
            tline
-                .applied_gc_cutoff_lsn
+                .latest_gc_cutoff_lsn
                .lock_for_write()
                .store_and_unlock(Lsn(0x30))
                .wait()
@@ -10790,7 +10634,7 @@ mod tests {
            .await?;
        {
            tline
-                .applied_gc_cutoff_lsn
+                .latest_gc_cutoff_lsn
                .lock_for_write()
                .store_and_unlock(Lsn(0x30))
                .wait()
@@ -11041,7 +10885,7 @@ mod tests {
            .await?;
        {
            tline
-                .applied_gc_cutoff_lsn
+                .latest_gc_cutoff_lsn
                .lock_for_write()
                .store_and_unlock(Lsn(0x30))
                .wait()
--- a/pageserver/src/tenant/config.rs
+++ b/pageserver/src/tenant/config.rs
@@ -285,14 +285,6 @@ pub struct TenantConfOpt {
    #[serde(default)]
    pub compaction_algorithm: Option<CompactionAlgorithmSettings>,

-    #[serde(skip_serializing_if = "Option::is_none")]
-    #[serde(default)]
-    pub compaction_l0_first: Option<bool>,
-
-    #[serde(skip_serializing_if = "Option::is_none")]
-    #[serde(default)]
-    pub compaction_l0_semaphore: Option<bool>,
-
    #[serde(skip_serializing_if = "Option::is_none")]
    #[serde(default)]
    pub l0_flush_delay_threshold: Option<usize>,
@@ -424,12 +416,6 @@ impl TenantConfOpt {
                .as_ref()
                .unwrap_or(&global_conf.compaction_algorithm)
                .clone(),
-            compaction_l0_first: self
-                .compaction_l0_first
-                .unwrap_or(global_conf.compaction_l0_first),
-            compaction_l0_semaphore: self
-                .compaction_l0_semaphore
-                .unwrap_or(global_conf.compaction_l0_semaphore),
            l0_flush_delay_threshold: self
                .l0_flush_delay_threshold
                .or(global_conf.l0_flush_delay_threshold),
@@ -480,7 +466,7 @@ impl TenantConfOpt {
                .lsn_lease_length_for_ts
                .unwrap_or(global_conf.lsn_lease_length_for_ts),
            timeline_offloading: self
-                .timeline_offloading
+                .lazy_slru_download
                .unwrap_or(global_conf.timeline_offloading),
            wal_receiver_protocol_override: self
                .wal_receiver_protocol_override
@@ -507,8 +493,6 @@ impl TenantConfOpt {
            mut compaction_threshold,
            mut compaction_upper_limit,
            mut compaction_algorithm,
-            mut compaction_l0_first,
-            mut compaction_l0_semaphore,
            mut l0_flush_delay_threshold,
            mut l0_flush_stall_threshold,
            mut l0_flush_wait_upload,
@@ -554,10 +538,6 @@ impl TenantConfOpt {
            .compaction_upper_limit
            .apply(&mut compaction_upper_limit);
        patch.compaction_algorithm.apply(&mut compaction_algorithm);
-        patch.compaction_l0_first.apply(&mut compaction_l0_first);
-        patch
-            .compaction_l0_semaphore
-            .apply(&mut compaction_l0_semaphore);
        patch
            .l0_flush_delay_threshold
            .apply(&mut l0_flush_delay_threshold);
@@ -639,8 +619,6 @@ impl TenantConfOpt {
            compaction_threshold,
            compaction_upper_limit,
            compaction_algorithm,
-            compaction_l0_first,
-            compaction_l0_semaphore,
            l0_flush_delay_threshold,
            l0_flush_stall_threshold,
            l0_flush_wait_upload,
@@ -703,8 +681,6 @@ impl From<TenantConfOpt> for models::TenantConfig {
            compaction_period: value.compaction_period.map(humantime),
            compaction_threshold: value.compaction_threshold,
            compaction_upper_limit: value.compaction_upper_limit,
-            compaction_l0_first: value.compaction_l0_first,
-            compaction_l0_semaphore: value.compaction_l0_semaphore,
            l0_flush_delay_threshold: value.l0_flush_delay_threshold,
            l0_flush_stall_threshold: value.l0_flush_stall_threshold,
            l0_flush_wait_upload: value.l0_flush_wait_upload,
--- a/pageserver/src/tenant/metadata.rs
+++ b/pageserver/src/tenant/metadata.rs
@@ -130,10 +130,7 @@ struct TimelineMetadataBodyV2 {
    prev_record_lsn: Option<Lsn>,
    ancestor_timeline: Option<TimelineId>,
    ancestor_lsn: Lsn,
-
-    // The LSN at which GC was last executed.  Synonym of [`Timeline::applied_gc_cutoff_lsn`].
    latest_gc_cutoff_lsn: Lsn,
-
    initdb_lsn: Lsn,
    pg_version: u32,
 }
--- a/pageserver/src/tenant/mgr.rs
+++ b/pageserver/src/tenant/mgr.rs
@@ -2816,8 +2816,8 @@ where
 }

 use {
-    crate::tenant::gc_result::GcResult, http_utils::error::ApiError,
-    pageserver_api::models::TimelineGcRequest,
+    crate::tenant::gc_result::GcResult, pageserver_api::models::TimelineGcRequest,
+    utils::http::error::ApiError,
 };

 #[cfg(test)]
--- a/pageserver/src/tenant/remote_timeline_client.rs
+++ b/pageserver/src/tenant/remote_timeline_client.rs
@@ -437,7 +437,8 @@ impl RemoteTimelineClient {
            .conf
            .remote_storage_config
            .as_ref()
-            .map_or(0, |r| r.concurrency_limit());
+            .and_then(|r| r.concurrency_limit())
+            .unwrap_or(0);
        let mut upload_queue = self.upload_queue.lock().unwrap();
        upload_queue.initialize_with_current_remote_index_part(index_part, inprogress_limit)?;
        self.update_remote_physical_size_gauge(Some(index_part));
@@ -460,7 +461,8 @@ impl RemoteTimelineClient {
            .conf
            .remote_storage_config
            .as_ref()
-            .map_or(0, |r| r.concurrency_limit());
+            .and_then(|r| r.concurrency_limit())
+            .unwrap_or(0);
        let mut upload_queue = self.upload_queue.lock().unwrap();
        upload_queue.initialize_empty_remote(local_metadata, inprogress_limit)?;
        self.update_remote_physical_size_gauge(None);
@@ -482,7 +484,8 @@ impl RemoteTimelineClient {
            .conf
            .remote_storage_config
            .as_ref()
-            .map_or(0, |r| r.concurrency_limit());
+            .and_then(|r| r.concurrency_limit())
+            .unwrap_or(0);

        let mut upload_queue = self.upload_queue.lock().unwrap();
        upload_queue.initialize_with_current_remote_index_part(index_part, inprogress_limit)?;
@@ -517,7 +520,7 @@ impl RemoteTimelineClient {
            if let Ok(queue) = queue_locked.initialized_mut() {
                let blocked_deletions = std::mem::take(&mut queue.blocked_deletions);
                for d in blocked_deletions {
-                    if let Err(e) = self.deletion_queue_client.push_layers(
+                    if let Err(e) = self.deletion_queue_client.push_layers_sync(
                        self.tenant_shard_id,
                        self.timeline_id,
                        self.generation,
@@ -2151,6 +2154,7 @@ impl RemoteTimelineClient {
                                self.generation,
                                delete.layers.clone(),
                            )
+                            .await
                            .map_err(|e| anyhow::anyhow!(e))
                    }
                }
--- a/pageserver/src/tenant/secondary/heatmap.rs
+++ b/pageserver/src/tenant/secondary/heatmap.rs
@@ -1,4 +1,4 @@
-use std::{collections::HashMap, time::SystemTime};
+use std::time::SystemTime;

 use crate::tenant::{remote_timeline_client::index::LayerFileMetadata, storage_layer::LayerName};

@@ -8,7 +8,7 @@ use serde_with::{serde_as, DisplayFromStr, TimestampSeconds};
 use utils::{generation::Generation, id::TimelineId};

 #[derive(Serialize, Deserialize)]
-pub(crate) struct HeatMapTenant {
+pub(super) struct HeatMapTenant {
    /// Generation of the attached location that uploaded the heatmap: this is not required
    /// for correctness, but acts as a hint to secondary locations in order to detect thrashing
    /// in the unlikely event that two attached locations are both uploading conflicting heatmaps.
@@ -25,17 +25,8 @@ pub(crate) struct HeatMapTenant {
    pub(super) upload_period_ms: Option<u128>,
 }

-impl HeatMapTenant {
-    pub(crate) fn into_timelines_index(self) -> HashMap<TimelineId, HeatMapTimeline> {
-        self.timelines
-            .into_iter()
-            .map(|htl| (htl.timeline_id, htl))
-            .collect()
-    }
-}
-
 #[serde_as]
-#[derive(Serialize, Deserialize, Clone)]
+#[derive(Serialize, Deserialize)]
 pub(crate) struct HeatMapTimeline {
    #[serde_as(as = "DisplayFromStr")]
    pub(crate) timeline_id: TimelineId,
@@ -44,13 +35,13 @@ pub(crate) struct HeatMapTimeline {
 }

 #[serde_as]
-#[derive(Serialize, Deserialize, Clone)]
+#[derive(Serialize, Deserialize)]
 pub(crate) struct HeatMapLayer {
    pub(crate) name: LayerName,
    pub(crate) metadata: LayerFileMetadata,

    #[serde_as(as = "TimestampSeconds<i64>")]
-    pub(crate) access_time: SystemTime,
+    pub(super) access_time: SystemTime,
    // TODO: an actual 'heat' score that would let secondary locations prioritize downloading
    // the hottest layers, rather than trying to simply mirror whatever layers are on-disk on the primary.
 }
--- a/pageserver/src/tenant/secondary/heatmap_uploader.rs
+++ b/pageserver/src/tenant/secondary/heatmap_uploader.rs
@@ -9,14 +9,13 @@ use crate::{
    metrics::SECONDARY_MODE,
    tenant::{
        config::AttachmentMode,
-        mgr::{GetTenantError, TenantManager},
+        mgr::GetTenantError,
+        mgr::TenantManager,
        remote_timeline_client::remote_heatmap_path,
        span::debug_assert_current_span_has_tenant_id,
        tasks::{warn_when_period_overrun, BackgroundLoopKind},
        Tenant,
    },
-    virtual_file::VirtualFile,
-    TEMP_FILE_SUFFIX,
 };

 use futures::Future;
@@ -33,10 +32,7 @@ use super::{
 };
 use tokio_util::sync::CancellationToken;
 use tracing::{info_span, instrument, Instrument};
-use utils::{
-    backoff, completion::Barrier, crashsafe::path_with_suffix_extension,
-    yielding_loop::yielding_loop,
-};
+use utils::{backoff, completion::Barrier, yielding_loop::yielding_loop};

 pub(super) async fn heatmap_uploader_task(
    tenant_manager: Arc<TenantManager>,
@@ -465,18 +461,6 @@ async fn upload_tenant_heatmap(
        }
    }

-    // After a successful upload persist the fresh heatmap to disk.
-    // When restarting, the tenant will read the heatmap from disk
-    // and additively generate a new heatmap (see [`Timeline::generate_heatmap`]).
-    // If the heatmap is stale, the additive generation can lead to keeping previously
-    // evicted timelines on the secondarie's disk.
-    let tenant_shard_id = tenant.get_tenant_shard_id();
-    let heatmap_path = tenant.conf.tenant_heatmap_path(tenant_shard_id);
-    let temp_path = path_with_suffix_extension(&heatmap_path, TEMP_FILE_SUFFIX);
-    if let Err(err) = VirtualFile::crashsafe_overwrite(heatmap_path, temp_path, bytes).await {
-        tracing::warn!("Non fatal IO error writing to disk after heatmap upload: {err}");
-    }
-
    tracing::info!("Successfully uploaded {size} byte heatmap to {path}");

    Ok(UploadHeatmapOutcome::Uploaded(LastUploadState {
--- a/pageserver/src/tenant/size.rs
+++ b/pageserver/src/tenant/size.rs
@@ -394,7 +394,7 @@ pub(super) async fn gather_inputs(
            ancestor_lsn,
            last_record: last_record_lsn,
            // this is not used above, because it might not have updated recently enough
-            latest_gc_cutoff: *timeline.get_applied_gc_cutoff_lsn(),
+            latest_gc_cutoff: *timeline.get_latest_gc_cutoff_lsn(),
            next_pitr_cutoff,
            retention_param_cutoff,
            lease_points,
--- a/pageserver/src/tenant/storage_layer.rs
+++ b/pageserver/src/tenant/storage_layer.rs
@@ -44,7 +44,7 @@ pub(crate) use layer::{EvictionError, Layer, ResidentLayer};

 use self::inmemory_layer::InMemoryLayerFileId;

-use super::timeline::{GetVectoredError, ReadPath};
+use super::timeline::GetVectoredError;
 use super::PageReconstructError;

 pub fn range_overlaps<T>(a: &Range<T>, b: &Range<T>) -> bool
@@ -262,8 +262,6 @@ pub(crate) struct ValuesReconstructState {

    pub(crate) io_concurrency: IoConcurrency,
    num_active_ios: Arc<AtomicUsize>,
-
-    pub(crate) read_path: Option<ReadPath>,
 }

 /// The level of IO concurrency to be used on the read path
@@ -611,7 +609,6 @@ impl ValuesReconstructState {
            delta_layers_visited: 0,
            io_concurrency,
            num_active_ios: Arc::new(AtomicUsize::new(0)),
-            read_path: None,
        }
    }

--- a/pageserver/src/tenant/storage_layer/layer.rs
+++ b/pageserver/src/tenant/storage_layer/layer.rs
@@ -136,22 +136,6 @@ pub(crate) fn local_layer_path(
    }
 }

-pub(crate) enum LastEviction {
-    Never,
-    At(std::time::Instant),
-    Evicting,
-}
-
-impl LastEviction {
-    pub(crate) fn happened_after(&self, timepoint: std::time::Instant) -> bool {
-        match self {
-            LastEviction::Never => false,
-            LastEviction::At(evicted_at) => evicted_at > &timepoint,
-            LastEviction::Evicting => true,
-        }
-    }
-}
-
 impl Layer {
    /// Creates a layer value for a file we know to not be resident.
    pub(crate) fn for_evicted(
@@ -369,6 +353,7 @@ impl Layer {
    /// while the guard exists.
    ///
    /// Returns None if the layer is currently evicted or becoming evicted.
+    #[cfg(test)]
    pub(crate) async fn keep_resident(&self) -> Option<ResidentLayer> {
        let downloaded = self.0.inner.get().and_then(|rowe| rowe.get())?;

@@ -421,17 +406,6 @@ impl Layer {
        self.0.metadata()
    }

-    pub(crate) fn last_evicted_at(&self) -> LastEviction {
-        match self.0.last_evicted_at.try_lock() {
-            Ok(lock) => match *lock {
-                None => LastEviction::Never,
-                Some(at) => LastEviction::At(at),
-            },
-            Err(std::sync::TryLockError::WouldBlock) => LastEviction::Evicting,
-            Err(std::sync::TryLockError::Poisoned(p)) => panic!("Lock poisoned: {p}"),
-        }
-    }
-
    pub(crate) fn get_timeline_id(&self) -> Option<TimelineId> {
        self.0
            .timeline
@@ -556,6 +530,7 @@ impl ResidentOrWantedEvicted {
    /// This is not used on the read path (anything that calls
    /// [`LayerInner::get_or_maybe_download`]) because it was decided that reads always win
    /// evictions, and part of that winning is using [`ResidentOrWantedEvicted::get_and_upgrade`].
+    #[cfg(test)]
    fn get(&self) -> Option<Arc<DownloadedLayer>> {
        match self {
            ResidentOrWantedEvicted::Resident(strong) => Some(strong.clone()),
@@ -683,9 +658,7 @@ struct LayerInner {

    /// When the Layer was last evicted but has not been downloaded since.
    ///
-    /// This is used for skipping evicted layers from the previous heatmap (see
-    /// `[Timeline::generate_heatmap]`) and for updating metrics
-    /// (see [`LayerImplMetrics::redownload_after`]).
+    /// This is used solely for updating metrics. See [`LayerImplMetrics::redownload_after`].
    last_evicted_at: std::sync::Mutex<Option<std::time::Instant>>,

    #[cfg(test)]
--- a/Show More
+++ b/Show More