Refactor LayerMap::search to also search open and frozen layers.

This allows refactoring in get_reconstruct_data() which is needed by PR #3228. That PR turns some functions async, and you cannot hold a RwLock over 'await'. And it's enough to "drop(guard)" the lock guard, it has to actually go out-of-scope to placate the compiler. This refactoring allows dropping the lock on 'layers' at end of scope.
Refactor send_tarball()
2026-08-12 17:23:42 +00:00 · 2023-01-03 22:59:18 +02:00 · 2023-01-03 20:47:38 +02:00
165 changed files with 3837 additions and 7832 deletions
@@ -4,7 +4,7 @@
 hakari-package = "workspace_hack"

 # Format for `workspace-hack = ...` lines in other Cargo.tomls. Requires cargo-hakari 0.9.8 or above.
-dep-format-version = "3"
+dep-format-version = "2"

 # Setting workspace.resolver = "2" in the root Cargo.toml is HIGHLY recommended.
 # Hakari works much better with the new feature resolver.
@@ -1,10 +0,0 @@
-## Describe your changes
-
-## Issue ticket number and link
-
-## Checklist before requesting a review
- [ ] I have performed a self-review of my code.
- [ ] If it is a core feature, I have added thorough tests.
- [ ] Do we need to implement analytics? if so did you add the relevant metrics to the dashboard?
- [ ] If this PR requires public announcement, mark it with /release-notes label and add several sentences in this section.
-
@@ -123,8 +123,8 @@ runs:
          exit 1
        fi
        if [[ "${{ inputs.run_in_parallel }}" == "true" ]]; then
-          # -n4 uses four processes to run tests via pytest-xdist
-          EXTRA_PARAMS="-n4 $EXTRA_PARAMS"
+          # -n8 uses eight processes to run tests via pytest-xdist
+          EXTRA_PARAMS="-n8 $EXTRA_PARAMS"

          # --dist=loadgroup points tests marked with @pytest.mark.xdist_group
          # to the same worker to make @pytest.mark.order work with xdist
@@ -117,8 +117,7 @@
      shell:
        cmd: |
          INSTANCE_ID=$(curl -s http://169.254.169.254/latest/meta-data/instance-id)
-          curl -sfS -H "Authorization: Bearer {{ CONSOLE_API_TOKEN }}" {{ console_mgmt_base_url }}/management/api/v2/pageservers/$INSTANCE_ID | jq '.version = {{ current_version }}' > /tmp/new_version
-          curl -sfS -H "Authorization: Bearer {{ CONSOLE_API_TOKEN }}" -X POST -d@/tmp/new_version {{ console_mgmt_base_url }}/management/api/v2/pageservers
+          curl -sfS -d '{"version": {{ current_version }} }' -X PATCH {{ console_mgmt_base_url }}/api/v1/pageservers/$INSTANCE_ID
      tags:
      - pageserver

@@ -187,7 +186,6 @@
      shell:
        cmd: |
          INSTANCE_ID=$(curl -s http://169.254.169.254/latest/meta-data/instance-id)
-          curl -sfS -H "Authorization: Bearer {{ CONSOLE_API_TOKEN }}" {{ console_mgmt_base_url }}/management/api/v2/safekeepers/$INSTANCE_ID | jq '.version = {{ current_version }}' > /tmp/new_version
-          curl -sfS -H "Authorization: Bearer {{ CONSOLE_API_TOKEN }}" -X POST -d@/tmp/new_version {{ console_mgmt_base_url }}/management/api/v2/safekeepers
+          curl -sfS -d '{"version": {{ current_version }} }' -X PATCH {{ console_mgmt_base_url }}/api/v1/safekeepers/$INSTANCE_ID
      tags:
      - safekeeper
@@ -6,8 +6,6 @@ storage:
    broker_endpoint: http://storage-broker-lb.epsilon.ap-southeast-1.internal.aws.neon.tech:50051
    pageserver_config_stub:
      pg_distrib_dir: /usr/local
-      metric_collection_endpoint: http://console-release.local/billing/api/v1/usage_events
-      metric_collection_interval: 10min
      remote_storage:
        bucket_name: "{{ bucket_name }}"
        bucket_region: "{{ bucket_region }}"
@@ -6,8 +6,6 @@ storage:
    broker_endpoint: http://storage-broker-lb.gamma.eu-central-1.internal.aws.neon.tech:50051
    pageserver_config_stub:
      pg_distrib_dir: /usr/local
-      metric_collection_endpoint: http://console-release.local/billing/api/v1/usage_events
-      metric_collection_interval: 10min
      remote_storage:
        bucket_name: "{{ bucket_name }}"
        bucket_region: "{{ bucket_region }}"
@@ -6,8 +6,6 @@ storage:
    broker_endpoint: http://storage-broker-lb.delta.us-east-2.internal.aws.neon.tech:50051
    pageserver_config_stub:
      pg_distrib_dir: /usr/local
-      metric_collection_endpoint: http://console-release.local/billing/api/v1/usage_events
-      metric_collection_interval: 10min
      remote_storage:
        bucket_name: "{{ bucket_name }}"
        bucket_region: "{{ bucket_region }}"
@@ -36,4 +34,4 @@ storage:
          ansible_host:  i-06d113fb73bfddeb0
        safekeeper-2.us-east-2.aws.neon.tech:
          ansible_host:  i-09f66c8e04afff2e8
-
+          
@@ -6,8 +6,6 @@ storage:
    broker_endpoint: http://storage-broker-lb.eta.us-west-2.internal.aws.neon.tech:50051
    pageserver_config_stub:
      pg_distrib_dir: /usr/local
-      metric_collection_endpoint: http://console-release.local/billing/api/v1/usage_events
-      metric_collection_interval: 10min
      remote_storage:
        bucket_name: "{{ bucket_name }}"
        bucket_region: "{{ bucket_region }}"
@@ -7,8 +7,6 @@ storage:
    broker_endpoint: http://storage-broker.prod.local:50051
    pageserver_config_stub:
      pg_distrib_dir: /usr/local
-      metric_collection_endpoint: http://console-release.local/billing/api/v1/usage_events
-      metric_collection_interval: 10min
      remote_storage:
        bucket_name: "{{ bucket_name }}"
        bucket_region: "{{ bucket_region }}"
@@ -18,7 +18,7 @@ storage:
    ansible_aws_ssm_region: eu-west-1
    ansible_aws_ssm_bucket_name: neon-dev-storage-eu-west-1
    console_region_id: aws-eu-west-1
-    sentry_environment: staging
+    sentry_environment: development

  children:
    pageservers:
@@ -18,7 +18,7 @@ storage:
    ansible_aws_ssm_region: us-east-2
    ansible_aws_ssm_bucket_name: neon-staging-storage-us-east-2
    console_region_id: aws-us-east-2
-    sentry_environment: staging
+    sentry_environment: development

  children:
    pageservers:
@@ -29,8 +29,6 @@ storage:
          ansible_host: i-0565a8b4008aa3f40
        pageserver-2.us-east-2.aws.neon.build:
          ansible_host: i-01e31cdf7e970586a
-        pageserver-3.us-east-2.aws.neon.build:
-          ansible_host: i-0602a0291365ef7cc

    safekeepers:
      hosts:
@@ -8,10 +8,7 @@ settings:
  authBackend: "console"
  authEndpoint: "http://console-staging.local/management/api/v2"
  domain: "*.eu-west-1.aws.neon.build"
-  sentryEnvironment: "staging"
-  wssPort: 8443
-  metricCollectionEndpoint: "http://console-staging.local/billing/api/v1/usage_events"
-  metricCollectionInterval: "1min"
+  sentryEnvironment: "development"

 # -- Additional labels for neon-proxy pods
 podLabels:
@@ -26,7 +23,6 @@ exposedService:
    service.beta.kubernetes.io/aws-load-balancer-nlb-target-type: ip
    service.beta.kubernetes.io/aws-load-balancer-scheme: internet-facing
    external-dns.alpha.kubernetes.io/hostname: eu-west-1.aws.neon.build
-  httpsPort: 443

 #metrics:
 #  enabled: true
@@ -49,4 +49,4 @@ extraManifests:
          - "{{ .Release.Namespace }}"

 settings:
-  sentryEnvironment: "staging"
+  sentryEnvironment: "development"
@@ -8,9 +8,7 @@ settings:
  authBackend: "link"
  authEndpoint: "https://console.stage.neon.tech/authenticate_proxy_request/"
  uri: "https://console.stage.neon.tech/psql_session/"
-  sentryEnvironment: "staging"
-  metricCollectionEndpoint: "http://console-staging.local/billing/api/v1/usage_events"
-  metricCollectionInterval: "1min"
+  sentryEnvironment: "development"

 # -- Additional labels for neon-proxy-link pods
 podLabels:
@@ -8,10 +8,7 @@ settings:
  authBackend: "console"
  authEndpoint: "http://console-staging.local/management/api/v2"
  domain: "*.cloud.stage.neon.tech"
-  sentryEnvironment: "staging"
-  wssPort: 8443
-  metricCollectionEndpoint: "http://console-staging.local/billing/api/v1/usage_events"
-  metricCollectionInterval: "1min"
+  sentryEnvironment: "development"

 # -- Additional labels for neon-proxy pods
 podLabels:
@@ -26,7 +23,6 @@ exposedService:
    service.beta.kubernetes.io/aws-load-balancer-nlb-target-type: ip
    service.beta.kubernetes.io/aws-load-balancer-scheme: internet-facing
    external-dns.alpha.kubernetes.io/hostname: neon-proxy-scram-legacy.beta.us-east-2.aws.neon.build
-  httpsPort: 443

 #metrics:
 #  enabled: true
@@ -8,10 +8,7 @@ settings:
  authBackend: "console"
  authEndpoint: "http://console-staging.local/management/api/v2"
  domain: "*.us-east-2.aws.neon.build"
-  sentryEnvironment: "staging"
-  wssPort: 8443
-  metricCollectionEndpoint: "http://console-staging.local/billing/api/v1/usage_events"
-  metricCollectionInterval: "1min"
+  sentryEnvironment: "development"

 # -- Additional labels for neon-proxy pods
 podLabels:
@@ -26,7 +23,6 @@ exposedService:
    service.beta.kubernetes.io/aws-load-balancer-nlb-target-type: ip
    service.beta.kubernetes.io/aws-load-balancer-scheme: internet-facing
    external-dns.alpha.kubernetes.io/hostname: us-east-2.aws.neon.build
-  httpsPort: 443

 #metrics:
 #  enabled: true
@@ -49,4 +49,4 @@ extraManifests:
          - "{{ .Release.Namespace }}"

 settings:
-  sentryEnvironment: "staging"
+  sentryEnvironment: "development"
@@ -9,9 +9,6 @@ settings:
  authEndpoint: "http://console-release.local/management/api/v2"
  domain: "*.ap-southeast-1.aws.neon.tech"
  sentryEnvironment: "production"
-  wssPort: 8443
-  metricCollectionEndpoint: "http://console-release.local/billing/api/v1/usage_events"
-  metricCollectionInterval: "10min"

 # -- Additional labels for neon-proxy pods
 podLabels:
@@ -26,7 +23,6 @@ exposedService:
    service.beta.kubernetes.io/aws-load-balancer-nlb-target-type: ip
    service.beta.kubernetes.io/aws-load-balancer-scheme: internet-facing
    external-dns.alpha.kubernetes.io/hostname: ap-southeast-1.aws.neon.tech
-  httpsPort: 443

 #metrics:
 #  enabled: true
@@ -9,9 +9,6 @@ settings:
  authEndpoint: "http://console-release.local/management/api/v2"
  domain: "*.eu-central-1.aws.neon.tech"
  sentryEnvironment: "production"
-  wssPort: 8443
-  metricCollectionEndpoint: "http://console-release.local/billing/api/v1/usage_events"
-  metricCollectionInterval: "10min"

 # -- Additional labels for neon-proxy pods
 podLabels:
@@ -26,7 +23,6 @@ exposedService:
    service.beta.kubernetes.io/aws-load-balancer-nlb-target-type: ip
    service.beta.kubernetes.io/aws-load-balancer-scheme: internet-facing
    external-dns.alpha.kubernetes.io/hostname: eu-central-1.aws.neon.tech
-  httpsPort: 443

 #metrics:
 #  enabled: true
@@ -9,9 +9,6 @@ settings:
  authEndpoint: "http://console-release.local/management/api/v2"
  domain: "*.us-east-2.aws.neon.tech"
  sentryEnvironment: "production"
-  wssPort: 8443
-  metricCollectionEndpoint: "http://console-release.local/billing/api/v1/usage_events"
-  metricCollectionInterval: "10min"

 # -- Additional labels for neon-proxy pods
 podLabels:
@@ -26,7 +23,6 @@ exposedService:
    service.beta.kubernetes.io/aws-load-balancer-nlb-target-type: ip
    service.beta.kubernetes.io/aws-load-balancer-scheme: internet-facing
    external-dns.alpha.kubernetes.io/hostname: us-east-2.aws.neon.tech
-  httpsPort: 443

 #metrics:
 #  enabled: true
@@ -9,9 +9,6 @@ settings:
  authEndpoint: "http://console-release.local/management/api/v2"
  domain: "*.us-west-2.aws.neon.tech"
  sentryEnvironment: "production"
-  wssPort: 8443
-  metricCollectionEndpoint: "http://console-release.local/billing/api/v1/usage_events"
-  metricCollectionInterval: "10min"

 # -- Additional labels for neon-proxy pods
 podLabels:
@@ -26,7 +23,6 @@ exposedService:
    service.beta.kubernetes.io/aws-load-balancer-nlb-target-type: ip
    service.beta.kubernetes.io/aws-load-balancer-scheme: internet-facing
    external-dns.alpha.kubernetes.io/hostname: us-west-2.aws.neon.tech
-  httpsPort: 443

 #metrics:
 #  enabled: true
@@ -3,9 +3,6 @@ settings:
  authEndpoint: "http://console-release.local/management/api/v2"
  domain: "*.cloud.neon.tech"
  sentryEnvironment: "production"
-  wssPort: 8443
-  metricCollectionEndpoint: "http://console-release.local/billing/api/v1/usage_events"
-  metricCollectionInterval: "10min"

 podLabels:
  zenith_service: proxy-scram
@@ -19,7 +16,6 @@ exposedService:
    service.beta.kubernetes.io/aws-load-balancer-nlb-target-type: ip
    service.beta.kubernetes.io/aws-load-balancer-scheme: internet-facing
    external-dns.alpha.kubernetes.io/hostname: '*.cloud.neon.tech'
-  httpsPort: 443

 metrics:
  enabled: true
@@ -489,108 +489,3 @@ jobs:
        slack-message: "Periodic TPC-H perf testing ${{ matrix.platform }}: ${{ job.status }}\n${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
      env:
        SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
-
-  user-examples-compare:
-    if: success() || failure()
-    needs: [ tpch-compare ]
-
-    strategy:
-      fail-fast: false
-      matrix:
-        # neon-captest-prefetch: We have pre-created projects with prefetch enabled
-        # rds-aurora: Aurora Postgres Serverless v2 with autoscaling from 0.5 to 2 ACUs
-        # rds-postgres: RDS Postgres db.m5.large instance (2 vCPU, 8 GiB) with gp3 EBS storage
-        platform: [ neon-captest-prefetch, rds-postgres, rds-aurora ]
-
-    env:
-      POSTGRES_DISTRIB_DIR: /tmp/neon/pg_install
-      DEFAULT_PG_VERSION: 14
-      TEST_OUTPUT: /tmp/test_output
-      BUILD_TYPE: remote
-      SAVE_PERF_REPORT: ${{ github.event.inputs.save_perf_report || ( github.ref == 'refs/heads/main' ) }}
-      PLATFORM: ${{ matrix.platform }}
-
-    runs-on: [ self-hosted, us-east-2, x64 ]
-    container:
-      image: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/rust:pinned
-      options: --init
-
-    timeout-minutes: 360 # 6h
-
-    steps:
-    - uses: actions/checkout@v3
-
-    - name: Download Neon artifact
-      uses: ./.github/actions/download
-      with:
-        name: neon-${{ runner.os }}-release-artifact
-        path: /tmp/neon/
-        prefix: latest
-
-    - name: Add Postgres binaries to PATH
-      run: |
-        ${POSTGRES_DISTRIB_DIR}/v${DEFAULT_PG_VERSION}/bin/pgbench --version
-        echo "${POSTGRES_DISTRIB_DIR}/v${DEFAULT_PG_VERSION}/bin" >> $GITHUB_PATH
-
-    - name: Set up Connection String
-      id: set-up-connstr
-      run: |
-        case "${PLATFORM}" in
-          neon-captest-prefetch)
-            CONNSTR=${{ secrets.BENCHMARK_USER_EXAMPLE_CAPTEST_CONNSTR }}
-            ;;
-          rds-aurora)
-            CONNSTR=${{ secrets.BENCHMARK_USER_EXAMPLE_RDS_AURORA_CONNSTR }}
-            ;;
-          rds-postgres)
-            CONNSTR=${{ secrets.BENCHMARK_USER_EXAMPLE_RDS_POSTGRES_CONNSTR }}
-            ;;
-          *)
-            echo 2>&1 "Unknown PLATFORM=${PLATFORM}. Allowed only 'neon-captest-prefetch', 'rds-aurora', or 'rds-postgres'"
-            exit 1
-            ;;
-        esac
-
-        echo "connstr=${CONNSTR}" >> $GITHUB_OUTPUT
-
-        psql ${CONNSTR} -c "SELECT version();"
-
-    - name: Set database options
-      if: matrix.platform == 'neon-captest-prefetch'
-      run: |
-        DB_NAME=$(psql ${BENCHMARK_CONNSTR} --no-align --quiet -t -c "SELECT current_database()")
-
-        psql ${BENCHMARK_CONNSTR} -c "ALTER DATABASE ${DB_NAME} SET enable_seqscan_prefetch=on"
-        psql ${BENCHMARK_CONNSTR} -c "ALTER DATABASE ${DB_NAME} SET effective_io_concurrency=32"
-        psql ${BENCHMARK_CONNSTR} -c "ALTER DATABASE ${DB_NAME} SET maintenance_io_concurrency=32"
-      env:
-        BENCHMARK_CONNSTR: ${{ steps.set-up-connstr.outputs.connstr }}
-
-    - name: Run user examples
-      uses: ./.github/actions/run-python-test-set
-      with:
-        build_type: ${{ env.BUILD_TYPE }}
-        test_selection: performance/test_perf_olap.py
-        run_in_parallel: false
-        save_perf_report: ${{ env.SAVE_PERF_REPORT }}
-        extra_params: -m remote_cluster --timeout 21600 -k test_user_examples
-      env:
-        VIP_VAP_ACCESS_TOKEN: "${{ secrets.VIP_VAP_ACCESS_TOKEN }}"
-        PERF_TEST_RESULT_CONNSTR: "${{ secrets.PERF_TEST_RESULT_CONNSTR }}"
-        BENCHMARK_CONNSTR: ${{ steps.set-up-connstr.outputs.connstr }}
-
-    - name: Create Allure report
-      if: success() || failure()
-      uses: ./.github/actions/allure-report
-      with:
-        action: generate
-        build_type: ${{ env.BUILD_TYPE }}
-
-    - name: Post to a Slack channel
-      if: ${{ github.event.schedule && failure() }}
-      uses: slackapi/slack-github-action@v1
-      with:
-        channel-id: "C033QLM5P7D" # dev-staging-stream
-        slack-message: "Periodic TPC-H perf testing ${{ matrix.platform }}: ${{ job.status }}\n${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
-      env:
-        SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
@@ -595,8 +595,6 @@ jobs:
    defaults:
      run:
        shell: sh -eu {0}
-    env:
-      VM_INFORMANT_VERSION: 0.1.1

    steps:
      - name: Downloading latest vm-builder
@@ -608,22 +606,9 @@ jobs:
        run: |
          docker pull 369495373322.dkr.ecr.eu-central-1.amazonaws.com/compute-node-${{ matrix.version }}:${{needs.tag.outputs.build-tag}}

-      - name: Downloading VM informant version ${{ env.VM_INFORMANT_VERSION }}
-        run: |
-          curl -fL https://github.com/neondatabase/autoscaling/releases/download/${{ env.VM_INFORMANT_VERSION }}/vm-informant -o vm-informant
-          chmod +x vm-informant
-
-      - name: Adding VM informant to compute-node image
-        run: |
-          ID=$(docker create 369495373322.dkr.ecr.eu-central-1.amazonaws.com/compute-node-${{ matrix.version }}:${{needs.tag.outputs.build-tag}})
-          docker cp vm-informant $ID:/bin/vm-informant
-          docker commit $ID temp-vm-compute-node
-          docker rm -f $ID
-
      - name: Build vm image
        run: |
-          # note: as of 2023-01-12, vm-builder requires a trailing ":latest" for local images
-          ./vm-builder -src=temp-vm-compute-node:latest -dst=369495373322.dkr.ecr.eu-central-1.amazonaws.com/vm-compute-node-${{ matrix.version }}:${{needs.tag.outputs.build-tag}}
+          ./vm-builder -src=369495373322.dkr.ecr.eu-central-1.amazonaws.com/compute-node-${{ matrix.version }}:${{needs.tag.outputs.build-tag}} -dst=369495373322.dkr.ecr.eu-central-1.amazonaws.com/vm-compute-node-${{ matrix.version }}:${{needs.tag.outputs.build-tag}}

      - name: Pushing vm-compute-node image
        run: |
@@ -809,8 +794,6 @@ jobs:
    strategy:
      matrix:
        include: ${{fromJSON(needs.calculate-deploy-targets.outputs.matrix-include)}}
-    environment:
-      name: prod-old
    steps:
      - name: Checkout
        uses: actions/checkout@v3
@@ -856,9 +839,7 @@ jobs:
        shell: bash
    strategy:
      matrix:
-        target_region: [ eu-west-1, us-east-2 ]
-    environment:
-      name: dev-${{ matrix.target_region }}
+        target_region: [ us-east-2 ]
    steps:
      - name: Checkout
        uses: actions/checkout@v3
@@ -930,8 +911,6 @@ jobs:
    strategy:
      matrix:
        target_region: [ us-east-2, us-west-2, eu-central-1, ap-southeast-1 ]
-    environment:
-      name: prod-${{ matrix.target_region }}
    steps:
      - name: Checkout
        uses: actions/checkout@v3
@@ -971,8 +950,6 @@ jobs:
    strategy:
      matrix:
        include: ${{fromJSON(needs.calculate-deploy-targets.outputs.matrix-include)}}
-    environment:
-      name: prod-old
    env:
      KUBECONFIG: .kubeconfig
    steps:
@@ -998,8 +975,8 @@ jobs:
      - name: Re-deploy proxy
        run: |
          DOCKER_TAG=${{needs.tag.outputs.build-tag}}
-          helm upgrade ${{ matrix.proxy_job }}       neondatabase/neon-proxy --namespace neon-proxy --install --atomic -f .github/helm-values/${{ matrix.proxy_config }}.yaml       --set image.tag=${DOCKER_TAG} --set settings.sentryUrl=${{ secrets.SENTRY_URL_PROXY }} --wait --timeout 15m0s
-          helm upgrade ${{ matrix.proxy_job }}-scram neondatabase/neon-proxy --namespace neon-proxy --install --atomic -f .github/helm-values/${{ matrix.proxy_config }}-scram.yaml --set image.tag=${DOCKER_TAG} --set settings.sentryUrl=${{ secrets.SENTRY_URL_PROXY }} --wait --timeout 15m0s
+          helm upgrade ${{ matrix.proxy_job }}       neondatabase/neon-proxy --namespace neon-proxy --install -f .github/helm-values/${{ matrix.proxy_config }}.yaml       --set image.tag=${DOCKER_TAG} --set settings.sentryUrl=${{ secrets.SENTRY_URL_PROXY }} --wait --timeout 15m0s
+          helm upgrade ${{ matrix.proxy_job }}-scram neondatabase/neon-proxy --namespace neon-proxy --install -f .github/helm-values/${{ matrix.proxy_config }}-scram.yaml --set image.tag=${DOCKER_TAG} --set settings.sentryUrl=${{ secrets.SENTRY_URL_PROXY }} --wait --timeout 15m0s

  deploy-storage-broker:
    name: deploy storage broker on old staging and old prod
@@ -1016,8 +993,6 @@ jobs:
    strategy:
      matrix:
        include: ${{fromJSON(needs.calculate-deploy-targets.outputs.matrix-include)}}
-    environment:
-      name: prod-old
    env:
      KUBECONFIG: .kubeconfig
    steps:
@@ -1066,8 +1041,6 @@ jobs:
            target_cluster: dev-eu-west-1-zeta
            deploy_link_proxy: false
            deploy_legacy_scram_proxy: false
-    environment:
-      name: dev-${{ matrix.target_region }}
    steps:
      - name: Checkout
        uses: actions/checkout@v3
@@ -1083,19 +1056,19 @@ jobs:
      - name: Re-deploy scram proxy
        run: |
          DOCKER_TAG=${{needs.tag.outputs.build-tag}}
-          helm upgrade neon-proxy-scram neondatabase/neon-proxy --namespace neon-proxy --create-namespace --install --atomic -f .github/helm-values/${{ matrix.target_cluster }}.neon-proxy-scram.yaml --set image.tag=${DOCKER_TAG} --set settings.sentryUrl=${{ secrets.SENTRY_URL_PROXY }} --wait --timeout 15m0s
+          helm upgrade neon-proxy-scram neondatabase/neon-proxy --namespace neon-proxy --create-namespace --install -f .github/helm-values/${{ matrix.target_cluster }}.neon-proxy-scram.yaml --set image.tag=${DOCKER_TAG} --set settings.sentryUrl=${{ secrets.SENTRY_URL_PROXY }} --wait --timeout 15m0s

      - name: Re-deploy link proxy
        if: matrix.deploy_link_proxy
        run: |
          DOCKER_TAG=${{needs.tag.outputs.build-tag}}
-          helm upgrade neon-proxy-link neondatabase/neon-proxy --namespace neon-proxy --create-namespace --install --atomic -f .github/helm-values/${{ matrix.target_cluster }}.neon-proxy-link.yaml --set image.tag=${DOCKER_TAG} --set settings.sentryUrl=${{ secrets.SENTRY_URL_PROXY }} --wait --timeout 15m0s
+          helm upgrade neon-proxy-link neondatabase/neon-proxy --namespace neon-proxy --create-namespace --install -f .github/helm-values/${{ matrix.target_cluster }}.neon-proxy-link.yaml --set image.tag=${DOCKER_TAG} --set settings.sentryUrl=${{ secrets.SENTRY_URL_PROXY }} --wait --timeout 15m0s

      - name: Re-deploy legacy scram proxy
        if: matrix.deploy_legacy_scram_proxy
        run: |
          DOCKER_TAG=${{needs.tag.outputs.build-tag}}
-          helm upgrade neon-proxy-scram-legacy neondatabase/neon-proxy --namespace neon-proxy --create-namespace --install --atomic -f .github/helm-values/${{ matrix.target_cluster }}.neon-proxy-scram-legacy.yaml --set image.tag=${DOCKER_TAG} --set settings.sentryUrl=${{ secrets.SENTRY_URL_PROXY }} --wait --timeout 15m0s
+          helm upgrade neon-proxy-scram-legacy neondatabase/neon-proxy --namespace neon-proxy --create-namespace --install -f .github/helm-values/${{ matrix.target_cluster }}.neon-proxy-scram-legacy.yaml --set image.tag=${DOCKER_TAG} --set settings.sentryUrl=${{ secrets.SENTRY_URL_PROXY }} --wait --timeout 15m0s

  deploy-storage-broker-dev-new:
    runs-on: [ self-hosted, dev, x64 ]
@@ -1115,8 +1088,6 @@ jobs:
            target_cluster: dev-us-east-2-beta
          - target_region:  eu-west-1
            target_cluster: dev-eu-west-1-zeta
-    environment:
-      name: dev-${{ matrix.target_region }}
    steps:
      - name: Checkout
        uses: actions/checkout@v3
@@ -1155,8 +1126,6 @@ jobs:
            target_cluster: prod-eu-central-1-gamma
          - target_region: ap-southeast-1
            target_cluster: prod-ap-southeast-1-epsilon
-    environment:
-      name: prod-${{ matrix.target_region }}
    steps:
      - name: Checkout
        uses: actions/checkout@v3
@@ -1172,7 +1141,7 @@ jobs:
      - name: Re-deploy proxy
        run: |
          DOCKER_TAG=${{needs.tag.outputs.build-tag}}
-          helm upgrade neon-proxy-scram neondatabase/neon-proxy --namespace neon-proxy --create-namespace --install --atomic -f .github/helm-values/${{ matrix.target_cluster }}.neon-proxy-scram.yaml --set image.tag=${DOCKER_TAG} --set settings.sentryUrl=${{ secrets.SENTRY_URL_PROXY }} --wait --timeout 15m0s
+          helm upgrade neon-proxy-scram neondatabase/neon-proxy --namespace neon-proxy --create-namespace --install -f .github/helm-values/${{ matrix.target_cluster }}.neon-proxy-scram.yaml --set image.tag=${DOCKER_TAG} --set settings.sentryUrl=${{ secrets.SENTRY_URL_PROXY }} --wait --timeout 15m0s

  deploy-storage-broker-prod-new:
    runs-on: prod
@@ -1196,8 +1165,6 @@ jobs:
            target_cluster: prod-eu-central-1-gamma
          - target_region: ap-southeast-1
            target_cluster: prod-ap-southeast-1-epsilon
-    environment:
-      name: prod-${{ matrix.target_region }}
    steps:
      - name: Checkout
        uses: actions/checkout@v3
@@ -1,3 +1,14 @@
+# 'named-profiles' feature was stabilized in cargo 1.57. This line makes the
+# build work with older cargo versions.
+#
+# We have this because as of this writing, the latest cargo Debian package
+# that's available is 1.56. (Confusingly, the Debian package version number
+# is 0.57, whereas 'cargo --version' says 1.56.)
+#
+# See https://tracker.debian.org/pkg/cargo for the current status of the
+# package. When that gets updated, we can remove this.
+cargo-features = ["named-profiles"]
+
 [workspace]
 members = [
    "compute_tools",
@@ -10,140 +21,6 @@ members = [
    "libs/*",
 ]

-[workspace.package]
-edition = "2021"
-license = "Apache-2.0"
-
-## All dependency versions, used in the project
-[workspace.dependencies]
-anyhow = { version = "1.0", features = ["backtrace"] }
-async-stream = "0.3"
-async-trait = "0.1"
-atty = "0.2.14"
-aws-config = { version = "0.51.0", default-features = false, features=["rustls"] }
-aws-sdk-s3 = "0.21.0"
-aws-smithy-http = "0.51.0"
-aws-types = "0.51.0"
-base64 = "0.13.0"
-bincode = "1.3"
-bindgen = "0.61"
-bstr = "1.0"
-byteorder = "1.4"
-bytes = "1.0"
-chrono = { version = "0.4", default-features = false, features = ["clock"] }
-clap = "4.0"
-close_fds = "0.3.2"
-comfy-table = "6.1"
-const_format = "0.2"
-crc32c = "0.6"
-crossbeam-utils = "0.8.5"
-fail = "0.5.0"
-fs2 = "0.4.3"
-futures = "0.3"
-futures-core = "0.3"
-futures-util = "0.3"
-git-version = "0.3"
-hashbrown = "0.13"
-hex = "0.4"
-hex-literal = "0.3"
-hmac = "0.12.1"
-hostname = "0.3.1"
-humantime = "2.1"
-humantime-serde = "1.1.1"
-hyper = "0.14"
-hyper-tungstenite = "0.9"
-itertools = "0.10"
-jsonwebtoken = "8"
-libc = "0.2"
-md5 = "0.7.0"
-memoffset = "0.8"
-nix = "0.26"
-notify = "5.0.0"
-num-traits = "0.2.15"
-once_cell = "1.13"
-parking_lot = "0.12"
-pin-project-lite = "0.2"
-prometheus = {version = "0.13", default_features=false, features = ["process"]} # removes protobuf dependency
-prost = "0.11"
-rand = "0.8"
-regex = "1.4"
-reqwest = { version = "0.11", default-features = false, features = ["rustls-tls"] }
-routerify = "3"
-rpds = "0.12.0"
-rustls = "0.20"
-rustls-pemfile = "1"
-rustls-split = "0.3"
-scopeguard = "1.1"
-sentry = { version = "0.29", default-features = false, features = ["backtrace", "contexts", "panic", "rustls", "reqwest" ] }
-serde = { version = "1.0", features = ["derive"] }
-serde_json = "1"
-serde_with = "2.0"
-sha2 = "0.10.2"
-signal-hook = "0.3"
-socket2 = "0.4.4"
-strum = "0.24"
-strum_macros = "0.24"
-svg_fmt = "0.4.1"
-tar = "0.4"
-thiserror = "1.0"
-tls-listener = { version = "0.6", features = ["rustls", "hyper-h1"] }
-tokio = { version = "1.17", features = ["macros"] }
-tokio-postgres-rustls = "0.9.0"
-tokio-rustls = "0.23"
-tokio-stream = "0.1"
-tokio-util = { version = "0.7", features = ["io"] }
-toml = "0.5"
-toml_edit = { version = "0.17", features = ["easy"] }
-tonic = {version = "0.8", features = ["tls", "tls-roots"]}
-tracing = "0.1"
-tracing-subscriber = { version = "0.3", features = ["env-filter"] }
-url = "2.2"
-uuid = { version = "1.2", features = ["v4", "serde"] }
-walkdir = "2.3.2"
-webpki-roots = "0.22.5"
-x509-parser = "0.14"
-
-## TODO replace this with tracing
-env_logger = "0.10"
-log = "0.4"
-
-## Libraries from neondatabase/ git forks, ideally with changes to be upstreamed
-postgres = { git = "https://github.com/neondatabase/rust-postgres.git", rev="43e6db254a97fdecbce33d8bc0890accfd74495e" }
-postgres-protocol = { git = "https://github.com/neondatabase/rust-postgres.git", rev="43e6db254a97fdecbce33d8bc0890accfd74495e" }
-postgres-types = { git = "https://github.com/neondatabase/rust-postgres.git", rev="43e6db254a97fdecbce33d8bc0890accfd74495e" }
-tokio-postgres = { git = "https://github.com/neondatabase/rust-postgres.git", rev="43e6db254a97fdecbce33d8bc0890accfd74495e" }
-tokio-tar = { git = "https://github.com/neondatabase/tokio-tar.git", rev="404df61437de0feef49ba2ccdbdd94eb8ad6e142" }
-
-## Local libraries
-consumption_metrics = { version = "0.1", path = "./libs/consumption_metrics/" }
-metrics = { version = "0.1", path = "./libs/metrics/" }
-pageserver_api = { version = "0.1", path = "./libs/pageserver_api/" }
-postgres_connection = { version = "0.1", path = "./libs/postgres_connection/" }
-postgres_ffi = { version = "0.1", path = "./libs/postgres_ffi/" }
-pq_proto = { version = "0.1", path = "./libs/pq_proto/" }
-remote_storage = { version = "0.1", path = "./libs/remote_storage/" }
-safekeeper_api = { version = "0.1", path = "./libs/safekeeper_api" }
-storage_broker = { version = "0.1", path = "./storage_broker/" } # Note: main broker code is inside the binary crate, so linking with the library shouldn't be heavy.
-tenant_size_model = { version = "0.1", path = "./libs/tenant_size_model/" }
-utils = { version = "0.1", path = "./libs/utils/" }
-
-## Common library dependency
-workspace_hack = { version = "0.1", path = "./workspace_hack/" }
-
-## Build dependencies
-criterion = "0.4"
-rcgen = "0.10"
-rstest = "0.16"
-tempfile = "3.2"
-tonic-build = "0.8"
-
-# This is only needed for proxy's tests.
-# TODO: we should probably fork `tokio-postgres-rustls` instead.
-[patch.crates-io]
-tokio-postgres = { git = "https://github.com/neondatabase/rust-postgres.git", rev="43e6db254a97fdecbce33d8bc0890accfd74495e" }
-
-################# Binary contents sections
-
 [profile.release]
 # This is useful for profiling and, to some extent, debug.
 # Besides, debug info should not affect the performance.
@@ -204,3 +81,9 @@ inherits = "release"
 debug = false # true = 2 = all symbols, 1 = line only
 opt-level = "z"
 lto = true
+
+
+# This is only needed for proxy's tests.
+# TODO: we should probably fork `tokio-postgres-rustls` instead.
+[patch.crates-io]
+tokio-postgres = { git = "https://github.com/neondatabase/rust-postgres.git", rev="43e6db254a97fdecbce33d8bc0890accfd74495e" }
@@ -29,13 +29,7 @@ RUN cd postgres && \
    make MAKELEVEL=0 -j $(getconf _NPROCESSORS_ONLN) -s -C contrib/ install && \
    # Install headers
    make MAKELEVEL=0 -j $(getconf _NPROCESSORS_ONLN) -s -C src/include install && \
-    make MAKELEVEL=0 -j $(getconf _NPROCESSORS_ONLN) -s -C src/interfaces/libpq install && \
-    # Enable some of contrib extensions
-    echo 'trusted = true' >> /usr/local/pgsql/share/extension/bloom.control && \
-    echo 'trusted = true' >> /usr/local/pgsql/share/extension/pgrowlocks.control && \
-    echo 'trusted = true' >> /usr/local/pgsql/share/extension/intagg.control && \
-    echo 'trusted = true' >> /usr/local/pgsql/share/extension/pgstattuple.control && \
-    echo 'trusted = true' >> /usr/local/pgsql/share/extension/earthdistance.control
+    make MAKELEVEL=0 -j $(getconf _NPROCESSORS_ONLN) -s -C src/interfaces/libpq install

 #########################################################################################
 #
@@ -61,9 +55,7 @@ RUN wget https://download.osgeo.org/postgis/source/postgis-3.3.1.tar.gz && \
    echo 'trusted = true' >> /usr/local/pgsql/share/extension/postgis.control && \
    echo 'trusted = true' >> /usr/local/pgsql/share/extension/postgis_raster.control && \
    echo 'trusted = true' >> /usr/local/pgsql/share/extension/postgis_tiger_geocoder.control && \
-    echo 'trusted = true' >> /usr/local/pgsql/share/extension/postgis_topology.control && \
-    echo 'trusted = true' >> /usr/local/pgsql/share/extension/address_standardizer.control && \
-    echo 'trusted = true' >> /usr/local/pgsql/share/extension/address_standardizer_data_us.control
+    echo 'trusted = true' >> /usr/local/pgsql/share/extension/postgis_topology.control

 #########################################################################################
 #
@@ -29,13 +29,7 @@ RUN cd postgres && \
    make MAKELEVEL=0 -j $(getconf _NPROCESSORS_ONLN) -s -C contrib/ install && \
    # Install headers
    make MAKELEVEL=0 -j $(getconf _NPROCESSORS_ONLN) -s -C src/include install && \
-    make MAKELEVEL=0 -j $(getconf _NPROCESSORS_ONLN) -s -C src/interfaces/libpq install && \
-    # Enable some of contrib extensions
-    echo 'trusted = true' >> /usr/local/pgsql/share/extension/bloom.control && \
-    echo 'trusted = true' >> /usr/local/pgsql/share/extension/pgrowlocks.control && \
-    echo 'trusted = true' >> /usr/local/pgsql/share/extension/intagg.control && \
-    echo 'trusted = true' >> /usr/local/pgsql/share/extension/pgstattuple.control && \
-    echo 'trusted = true' >> /usr/local/pgsql/share/extension/earthdistance.control
+    make MAKELEVEL=0 -j $(getconf _NPROCESSORS_ONLN) -s -C src/interfaces/libpq install

 #########################################################################################
 #
@@ -61,9 +55,7 @@ RUN wget https://download.osgeo.org/postgis/source/postgis-3.3.1.tar.gz && \
    echo 'trusted = true' >> /usr/local/pgsql/share/extension/postgis.control && \
    echo 'trusted = true' >> /usr/local/pgsql/share/extension/postgis_raster.control && \
    echo 'trusted = true' >> /usr/local/pgsql/share/extension/postgis_tiger_geocoder.control && \
-    echo 'trusted = true' >> /usr/local/pgsql/share/extension/postgis_topology.control && \
-    echo 'trusted = true' >> /usr/local/pgsql/share/extension/address_standardizer.control && \
-    echo 'trusted = true' >> /usr/local/pgsql/share/extension/address_standardizer_data_us.control
+    echo 'trusted = true' >> /usr/local/pgsql/share/extension/postgis_topology.control

 #########################################################################################
 #
@@ -118,8 +118,11 @@ Python (3.9 or higher), and install python3 packages using `./scripts/pysync` (r
 # Later that would be responsibility of a package install script
 > ./target/debug/neon_local init
 Starting pageserver at '127.0.0.1:64000' in '.neon'.
+pageserver started, pid: 2545906
+Successfully initialized timeline de200bd42b49cc1814412c7e592dd6e9
+Stopped pageserver 1 process with pid 2545906

-# start pageserver, safekeeper, and broker for their intercommunication
+# start pageserver and safekeeper
 > ./target/debug/neon_local start
 Starting neon broker at 127.0.0.1:50051
 storage_broker started, pid: 2918372
@@ -128,12 +131,6 @@ pageserver started, pid: 2918386
 Starting safekeeper at '127.0.0.1:5454' in '.neon/safekeepers/sk1'.
 safekeeper 1 started, pid: 2918437

-# create initial tenant and use it as a default for every future neon_local invocation
-> ./target/debug/neon_local tenant create --set-default
-tenant 9ef87a5bf0d92544f6fafeeb3239695c successfully created on the pageserver
-Created an initial timeline 'de200bd42b49cc1814412c7e592dd6e9' at Lsn 0/16B5A50 for tenant: 9ef87a5bf0d92544f6fafeeb3239695c
-Setting tenant 9ef87a5bf0d92544f6fafeeb3239695c as a default one
-
 # start postgres compute node
 > ./target/debug/neon_local pg start main
 Starting new postgres (v14) main on timeline de200bd42b49cc1814412c7e592dd6e9 ...
@@ -1,25 +1,24 @@
 [package]
 name = "compute_tools"
 version = "0.1.0"
-edition.workspace = true
-license.workspace = true
+edition = "2021"
+license = "Apache-2.0"

 [dependencies]
-anyhow.workspace = true
-chrono.workspace = true
-clap.workspace = true
-futures.workspace = true
-hyper = { workspace = true, features = ["full"] }
-notify.workspace = true
-postgres.workspace = true
-regex.workspace = true
-serde.workspace = true
-serde_json.workspace = true
-tar.workspace = true
-tokio = { workspace = true, features = ["rt", "rt-multi-thread"] }
-tokio-postgres.workspace = true
-tracing.workspace = true
-tracing-subscriber.workspace = true
-url.workspace = true
-
-workspace_hack.workspace = true
+anyhow = "1.0"
+chrono = { version = "0.4", default-features = false, features = ["clock"] }
+clap = "4.0"
+env_logger = "0.9"
+futures = "0.3.13"
+hyper = { version = "0.14", features = ["full"] }
+log = { version = "0.4", features = ["std", "serde"] }
+notify = "5.0.0"
+postgres = { git = "https://github.com/neondatabase/rust-postgres.git", rev="43e6db254a97fdecbce33d8bc0890accfd74495e" }
+regex = "1"
+serde = { version = "1.0", features = ["derive"] }
+serde_json = "1"
+tar = "0.4"
+tokio = { version = "1.17", features = ["macros", "rt", "rt-multi-thread"] }
+tokio-postgres = { git = "https://github.com/neondatabase/rust-postgres.git", rev="43e6db254a97fdecbce33d8bc0890accfd74495e" }
+url = "2.2.2"
+workspace_hack = { version = "0.1", path = "../workspace_hack" }
@@ -19,10 +19,6 @@ Also `compute_ctl` spawns two separate service threads:
 - `http-endpoint` runs a Hyper HTTP API server, which serves readiness and the
  last activity requests.

-If the `vm-informant` binary is present at `/bin/vm-informant`, it will also be started. For VM
-compute nodes, `vm-informant` communicates with the VM autoscaling system. It coordinates
-downscaling and (eventually) will request immediate upscaling under resource pressure.
-
 Usage example:
 ```sh
 compute_ctl -D /var/db/postgres/compute \
@@ -18,10 +18,6 @@
 //! - `http-endpoint` runs a Hyper HTTP API server, which serves readiness and the
 //!   last activity requests.
 //!
-//! If the `vm-informant` binary is present at `/bin/vm-informant`, it will also be started. For VM
-//! compute nodes, `vm-informant` communicates with the VM autoscaling system. It coordinates
-//! downscaling and (eventually) will request immediate upscaling under resource pressure.
-//!
 //! Usage example:
 //! ```sh
 //! compute_ctl -D /var/db/postgres/compute \
@@ -40,11 +36,10 @@ use std::{thread, time::Duration};
 use anyhow::{Context, Result};
 use chrono::Utc;
 use clap::Arg;
-use tracing::{error, info};
+use log::{error, info};

 use compute_tools::compute::{ComputeMetrics, ComputeNode, ComputeState, ComputeStatus};
 use compute_tools::http::api::launch_http_server;
-use compute_tools::informant::spawn_vm_informant_if_present;
 use compute_tools::logger::*;
 use compute_tools::monitor::launch_monitor;
 use compute_tools::params::*;
@@ -53,6 +48,7 @@ use compute_tools::spec::*;
 use url::Url;

 fn main() -> Result<()> {
+    // TODO: re-use `utils::logging` later
    init_logger(DEFAULT_LOG_LEVEL)?;

    let matches = cli().get_matches();
@@ -118,48 +114,30 @@ fn main() -> Result<()> {
    // requests, while configuration is still in progress.
    let _http_handle = launch_http_server(&compute).expect("cannot launch http endpoint thread");
    let _monitor_handle = launch_monitor(&compute).expect("cannot launch compute monitor thread");
-    // Also spawn the thread responsible for handling the VM informant -- if it's present
-    let _vm_informant_handle = spawn_vm_informant_if_present().expect("cannot launch VM informant");

-    // Start Postgres
-    let mut delay_exit = false;
-    let mut exit_code = None;
-    let pg = match compute.start_compute() {
-        Ok(pg) => Some(pg),
-        Err(err) => {
-            error!("could not start the compute node: {:?}", err);
+    // Run compute (Postgres) and hang waiting on it.
+    match compute.prepare_and_run() {
+        Ok(ec) => {
+            let code = ec.code().unwrap_or(1);
+            info!("Postgres exited with code {}, shutting down", code);
+            exit(code)
+        }
+        Err(error) => {
+            error!("could not start the compute node: {:?}", error);
+
            let mut state = compute.state.write().unwrap();
-            state.error = Some(format!("{:?}", err));
+            state.error = Some(format!("{:?}", error));
            state.status = ComputeStatus::Failed;
            drop(state);
-            delay_exit = true;
-            None
+
+            // Keep serving HTTP requests, so the cloud control plane was able to
+            // get the actual error.
+            info!("giving control plane 30s to collect the error before shutdown");
+            thread::sleep(Duration::from_secs(30));
+            info!("shutting down");
+            Err(error)
        }
-    };
-
-    // Wait for the child Postgres process forever. In this state Ctrl+C will
-    // propagate to Postgres and it will be shut down as well.
-    if let Some(mut pg) = pg {
-        let ecode = pg
-            .wait()
-            .expect("failed to start waiting on Postgres process");
-        info!("Postgres exited with code {}, shutting down", ecode);
-        exit_code = ecode.code()
    }
-
-    if let Err(err) = compute.check_for_core_dumps() {
-        error!("error while checking for core dumps: {err:?}");
-    }
-
-    // If launch failed, keep serving HTTP requests for a while, so the cloud
-    // control plane can get the actual error.
-    if delay_exit {
-        info!("giving control plane 30s to collect the error before shutdown");
-        thread::sleep(Duration::from_secs(30));
-        info!("shutting down");
-    }
-
-    exit(exit_code.unwrap_or(1))
 }

 fn cli() -> clap::Command {
@@ -1,11 +1,10 @@
 use anyhow::{anyhow, Result};
+use log::error;
 use postgres::Client;
 use tokio_postgres::NoTls;
-use tracing::{error, instrument};

 use crate::compute::ComputeNode;

-#[instrument(skip_all)]
 pub fn create_writability_check_data(client: &mut Client) -> Result<()> {
    let query = "
    CREATE TABLE IF NOT EXISTS health_check (
@@ -22,7 +21,6 @@ pub fn create_writability_check_data(client: &mut Client) -> Result<()> {
    Ok(())
 }

-#[instrument(skip_all)]
 pub async fn check_writability(compute: &ComputeNode) -> Result<()> {
    let (client, connection) = tokio_postgres::connect(compute.connstr.as_str(), NoTls).await?;
    if client.is_closed() {
@@ -17,15 +17,15 @@
 use std::fs;
 use std::os::unix::fs::PermissionsExt;
 use std::path::Path;
-use std::process::{Command, Stdio};
+use std::process::{Command, ExitStatus, Stdio};
 use std::sync::atomic::{AtomicU64, Ordering};
 use std::sync::RwLock;

 use anyhow::{Context, Result};
 use chrono::{DateTime, Utc};
+use log::{info, warn};
 use postgres::{Client, NoTls};
 use serde::{Serialize, Serializer};
-use tracing::{info, instrument, warn};

 use crate::checker::create_writability_check_data;
 use crate::config;
@@ -121,7 +121,6 @@ impl ComputeNode {

    // Get basebackup from the libpq connection to pageserver using `connstr` and
    // unarchive it to `pgdata` directory overriding all its previous content.
-    #[instrument(skip(self))]
    fn get_basebackup(&self, lsn: &str) -> Result<()> {
        let start_time = Utc::now();

@@ -155,7 +154,6 @@ impl ComputeNode {

    // Run `postgres` in a special mode with `--sync-safekeepers` argument
    // and return the reported LSN back to the caller.
-    #[instrument(skip(self))]
    fn sync_safekeepers(&self) -> Result<String> {
        let start_time = Utc::now();

@@ -198,7 +196,6 @@ impl ComputeNode {

    /// Do all the preparations like PGDATA directory creation, configuration,
    /// safekeepers sync, basebackup, etc.
-    #[instrument(skip(self))]
    pub fn prepare_pgdata(&self) -> Result<()> {
        let spec = &self.spec;
        let pgdata_path = Path::new(&self.pgdata);
@@ -232,8 +229,9 @@ impl ComputeNode {

    /// Start Postgres as a child process and manage DBs/roles.
    /// After that this will hang waiting on the postmaster process to exit.
-    #[instrument(skip(self))]
-    pub fn start_postgres(&self) -> Result<std::process::Child> {
+    pub fn run(&self) -> Result<ExitStatus> {
+        let start_time = Utc::now();
+
        let pgdata_path = Path::new(&self.pgdata);

        // Run postgres as a child process.
@@ -244,15 +242,10 @@ impl ComputeNode {

        wait_for_postgres(&mut pg, pgdata_path)?;

-        Ok(pg)
-    }
-
-    #[instrument(skip(self))]
-    pub fn apply_config(&self) -> Result<()> {
        // If connection fails,
        // it may be the old node with `zenith_admin` superuser.
        //
-        // In this case we need to connect with old `zenith_admin` name
+        // In this case we need to connect with old `zenith_admin`name
        // and create new user. We cannot simply rename connected user,
        // but we can create a new one and grant it all privileges.
        let mut client = match Client::connect(self.connstr.as_str(), NoTls) {
@@ -278,7 +271,6 @@ impl ComputeNode {
            Ok(client) => client,
        };

-        // Proceed with post-startup configuration. Note, that order of operations is important.
        handle_roles(&self.spec, &mut client)?;
        handle_databases(&self.spec, &mut client)?;
        handle_role_deletions(self, &mut client)?;
@@ -287,34 +279,8 @@ impl ComputeNode {

        // 'Close' connection
        drop(client);
-
-        info!(
-            "finished configuration of compute for project {}",
-            self.spec.cluster.cluster_id
-        );
-
-        Ok(())
-    }
-
-    #[instrument(skip(self))]
-    pub fn start_compute(&self) -> Result<std::process::Child> {
-        info!(
-            "starting compute for project {}, operation {}, tenant {}, timeline {}",
-            self.spec.cluster.cluster_id,
-            self.spec.operation_uuid.as_ref().unwrap(),
-            self.tenant,
-            self.timeline,
-        );
-
-        self.prepare_pgdata()?;
-
-        let start_time = Utc::now();
-
-        let pg = self.start_postgres()?;
-
-        self.apply_config()?;
-
        let startup_end_time = Utc::now();
+
        self.metrics.config_ms.store(
            startup_end_time
                .signed_duration_since(start_time)
@@ -334,7 +300,34 @@ impl ComputeNode {

        self.set_status(ComputeStatus::Running);

-        Ok(pg)
+        info!(
+            "finished configuration of compute for project {}",
+            self.spec.cluster.cluster_id
+        );
+
+        // Wait for child Postgres process basically forever. In this state Ctrl+C
+        // will propagate to Postgres and it will be shut down as well.
+        let ecode = pg
+            .wait()
+            .expect("failed to start waiting on Postgres process");
+
+        self.check_for_core_dumps()
+            .expect("failed to check for core dumps");
+
+        Ok(ecode)
+    }
+
+    pub fn prepare_and_run(&self) -> Result<ExitStatus> {
+        info!(
+            "starting compute for project {}, operation {}, tenant {}, timeline {}",
+            self.spec.cluster.cluster_id,
+            self.spec.operation_uuid.as_ref().unwrap(),
+            self.tenant,
+            self.timeline,
+        );
+
+        self.prepare_pgdata()?;
+        self.run()
    }

    // Look for core dumps and collect backtraces.
@@ -347,7 +340,7 @@ impl ComputeNode {
    //
    // Use that as a default location and pattern, except macos where core dumps are written
    // to /cores/ directory by default.
-    pub fn check_for_core_dumps(&self) -> Result<()> {
+    fn check_for_core_dumps(&self) -> Result<()> {
        let core_dump_dir = match std::env::consts::OS {
            "macos" => Path::new("/cores/"),
            _ => Path::new(&self.pgdata),
@@ -6,8 +6,8 @@ use std::thread;
 use anyhow::Result;
 use hyper::service::{make_service_fn, service_fn};
 use hyper::{Body, Method, Request, Response, Server, StatusCode};
+use log::{error, info};
 use serde_json;
-use tracing::{error, info};

 use crate::compute::ComputeNode;

@@ -1,50 +0,0 @@
-use std::path::Path;
-use std::process;
-use std::thread;
-use std::time::Duration;
-use tracing::{info, warn};
-
-use anyhow::{Context, Result};
-
-const VM_INFORMANT_PATH: &str = "/bin/vm-informant";
-const RESTART_INFORMANT_AFTER_MILLIS: u64 = 5000;
-
-/// Launch a thread to start the VM informant if it's present (and restart, on failure)
-pub fn spawn_vm_informant_if_present() -> Result<Option<thread::JoinHandle<()>>> {
-    let exists = Path::new(VM_INFORMANT_PATH)
-        .try_exists()
-        .context("could not check if path exists")?;
-
-    if !exists {
-        return Ok(None);
-    }
-
-    Ok(Some(
-        thread::Builder::new()
-            .name("run-vm-informant".into())
-            .spawn(move || run_informant())?,
-    ))
-}
-
-fn run_informant() -> ! {
-    let restart_wait = Duration::from_millis(RESTART_INFORMANT_AFTER_MILLIS);
-
-    info!("starting VM informant");
-
-    loop {
-        let mut cmd = process::Command::new(VM_INFORMANT_PATH);
-        // Block on subprocess:
-        let result = cmd.status();
-
-        match result {
-            Err(e) => warn!("failed to run VM informant at {VM_INFORMANT_PATH:?}: {e}"),
-            Ok(status) if !status.success() => {
-                warn!("{VM_INFORMANT_PATH} exited with code {status:?}, retrying")
-            }
-            Ok(_) => info!("{VM_INFORMANT_PATH} ended gracefully (unexpectedly). Retrying"),
-        }
-
-        // Wait before retrying
-        thread::sleep(restart_wait);
-    }
-}
@@ -8,7 +8,6 @@ pub mod http;
 #[macro_use]
 pub mod logger;
 pub mod compute;
-pub mod informant;
 pub mod monitor;
 pub mod params;
 pub mod pg_helpers;
@@ -1,20 +1,42 @@
+use std::io::Write;
+
 use anyhow::Result;
-use tracing_subscriber::layer::SubscriberExt;
-use tracing_subscriber::prelude::*;
+use chrono::Utc;
+use env_logger::{Builder, Env};
+
+macro_rules! info_println {
+    ($($tts:tt)*) => {
+        if log_enabled!(Level::Info) {
+            println!($($tts)*);
+        }
+    }
+}
+
+macro_rules! info_print {
+    ($($tts:tt)*) => {
+        if log_enabled!(Level::Info) {
+            print!($($tts)*);
+        }
+    }
+}

 /// Initialize `env_logger` using either `default_level` or
 /// `RUST_LOG` environment variable as default log level.
 pub fn init_logger(default_level: &str) -> Result<()> {
-    let env_filter = tracing_subscriber::EnvFilter::try_from_default_env()
-        .unwrap_or_else(|_| tracing_subscriber::EnvFilter::new(default_level));
+    let env = Env::default().filter_or("RUST_LOG", default_level);

-    let fmt_layer = tracing_subscriber::fmt::layer()
-        .with_target(false)
-        .with_writer(std::io::stderr);
-
-    tracing_subscriber::registry()
-        .with(env_filter)
-        .with(fmt_layer)
+    Builder::from_env(env)
+        .format(|buf, record| {
+            let thread_handle = std::thread::current();
+            writeln!(
+                buf,
+                "{} [{}] {}: {}",
+                Utc::now().format("%Y-%m-%d %H:%M:%S%.3f %Z"),
+                thread_handle.name().unwrap_or("main"),
+                record.level(),
+                record.args()
+            )
+        })
        .init();

    Ok(())
@@ -3,8 +3,8 @@ use std::{thread, time};

 use anyhow::Result;
 use chrono::{DateTime, Utc};
+use log::{debug, info};
 use postgres::{Client, NoTls};
-use tracing::{debug, info};

 use crate::compute::ComputeNode;

@@ -52,16 +52,10 @@ fn watch_compute_activity(compute: &ComputeNode) {
                    let mut idle_backs: Vec<DateTime<Utc>> = vec![];

                    for b in backs.into_iter() {
-                        let state: String = match b.try_get("state") {
-                            Ok(state) => state,
-                            Err(_) => continue,
-                        };
+                        let state: String = b.get("state");
+                        let change: String = b.get("state_change");

                        if state == "idle" {
-                            let change: String = match b.try_get("state_change") {
-                                Ok(state_change) => state_change,
-                                Err(_) => continue,
-                            };
                            let change = DateTime::parse_from_rfc3339(&change);
                            match change {
                                Ok(t) => idle_backs.push(t.with_timezone(&Utc)),
@@ -1,9 +1,3 @@
 pub const DEFAULT_LOG_LEVEL: &str = "info";
-// From Postgres docs:
-//   To ease transition from the md5 method to the newer SCRAM method, if md5 is specified
-//   as a method in pg_hba.conf but the user's password on the server is encrypted for SCRAM
-//   (see below), then SCRAM-based authentication will automatically be chosen instead.
-//   https://www.postgresql.org/docs/15/auth-password.html
-//
-// So it's safe to set md5 here, as `control-plane` anyway uses SCRAM for all roles.
+pub const DEFAULT_CONNSTRING: &str = "host=localhost user=postgres";
 pub const PG_HBA_ALL_MD5: &str = "host\tall\t\tall\t\t0.0.0.0/0\t\tmd5";
@@ -11,7 +11,6 @@ use anyhow::{bail, Result};
 use notify::{RecursiveMode, Watcher};
 use postgres::{Client, Transaction};
 use serde::Deserialize;
-use tracing::{debug, instrument};

 const POSTGRES_WAIT_TIMEOUT: Duration = Duration::from_millis(60 * 1000); // milliseconds

@@ -130,8 +129,8 @@ impl Role {
    /// Serialize a list of role parameters into a Postgres-acceptable
    /// string of arguments.
    pub fn to_pg_options(&self) -> String {
-        // XXX: consider putting LOGIN as a default option somewhere higher, e.g. in control-plane.
-        // For now, we do not use generic `options` for roles. Once used, add
+        // XXX: consider putting LOGIN as a default option somewhere higher, e.g. in Rails.
+        // For now we do not use generic `options` for roles. Once used, add
        // `self.options.as_pg_options()` somewhere here.
        let mut params: String = "LOGIN".to_string();

@@ -230,7 +229,6 @@ pub fn get_existing_dbs(client: &mut Client) -> Result<Vec<Database>> {
 /// Wait for Postgres to become ready to accept connections. It's ready to
 /// accept connections when the state-field in `pgdata/postmaster.pid` says
 /// 'ready'.
-#[instrument(skip(pg))]
 pub fn wait_for_postgres(pg: &mut Child, pgdata: &Path) -> Result<()> {
    let pid_path = pgdata.join("postmaster.pid");

@@ -289,18 +287,18 @@ pub fn wait_for_postgres(pg: &mut Child, pgdata: &Path) -> Result<()> {
        }

        let res = rx.recv_timeout(Duration::from_millis(100));
-        debug!("woken up by notify: {res:?}");
+        log::debug!("woken up by notify: {res:?}");
        // If there are multiple events in the channel already, we only need to be
        // check once. Swallow the extra events before we go ahead to check the
        // pid file.
        while let Ok(res) = rx.try_recv() {
-            debug!("swallowing extra event: {res:?}");
+            log::debug!("swallowing extra event: {res:?}");
        }

        // Check that we can open pid file first.
        if let Ok(file) = File::open(&pid_path) {
            if !postmaster_pid_seen {
-                debug!("postmaster.pid appeared");
+                log::debug!("postmaster.pid appeared");
                watcher
                    .unwatch(pgdata)
                    .expect("Failed to remove pgdata dir watch");
@@ -316,7 +314,7 @@ pub fn wait_for_postgres(pg: &mut Child, pgdata: &Path) -> Result<()> {
            // Pid file could be there and we could read it, but it could be empty, for example.
            if let Some(Ok(line)) = last_line {
                let status = line.trim();
-                debug!("last line of postmaster.pid: {status:?}");
+                log::debug!("last line of postmaster.pid: {status:?}");

                // Now Postgres is ready to accept connections
                if status == "ready" {
@@ -332,7 +330,7 @@ pub fn wait_for_postgres(pg: &mut Child, pgdata: &Path) -> Result<()> {
        }
    }

-    tracing::info!("PostgreSQL is now running, continuing to configure it");
+    log::info!("PostgreSQL is now running, continuing to configure it");

    Ok(())
 }
@@ -2,10 +2,10 @@ use std::path::Path;
 use std::str::FromStr;

 use anyhow::Result;
+use log::{info, log_enabled, warn, Level};
 use postgres::config::Config;
 use postgres::{Client, NoTls};
 use serde::Deserialize;
-use tracing::{info, info_span, instrument, span_enabled, warn, Level};

 use crate::compute::ComputeNode;
 use crate::config;
@@ -79,25 +79,23 @@ pub fn update_pg_hba(pgdata_path: &Path) -> Result<()> {

 /// Given a cluster spec json and open transaction it handles roles creation,
 /// deletion and update.
-#[instrument(skip_all)]
 pub fn handle_roles(spec: &ComputeSpec, client: &mut Client) -> Result<()> {
    let mut xact = client.transaction()?;
    let existing_roles: Vec<Role> = get_existing_roles(&mut xact)?;

    // Print a list of existing Postgres roles (only in debug mode)
-    if span_enabled!(Level::INFO) {
-        info!("postgres roles:");
-        for r in &existing_roles {
-            info!(
-                "    - {}:{}",
-                r.name,
-                if r.encrypted_password.is_some() {
-                    "[FILTERED]"
-                } else {
-                    "(null)"
-                }
-            );
-        }
+    info!("postgres roles:");
+    for r in &existing_roles {
+        info_println!(
+            "{} - {}:{}",
+            " ".repeat(27 + 5),
+            r.name,
+            if r.encrypted_password.is_some() {
+                "[FILTERED]"
+            } else {
+                "(null)"
+            }
+        );
    }

    // Process delta operations first
@@ -138,80 +136,58 @@ pub fn handle_roles(spec: &ComputeSpec, client: &mut Client) -> Result<()> {
    info!("cluster spec roles:");
    for role in &spec.cluster.roles {
        let name = &role.name;
-        // XXX: with a limited number of roles it is fine, but consider making it a HashMap
-        let pg_role = existing_roles.iter().find(|r| r.name == *name);

-        enum RoleAction {
-            None,
-            Update,
-            Create,
-        }
-        let action = if let Some(r) = pg_role {
-            if (r.encrypted_password.is_none() && role.encrypted_password.is_some())
-                || (r.encrypted_password.is_some() && role.encrypted_password.is_none())
-            {
-                RoleAction::Update
-            } else if let Some(pg_pwd) = &r.encrypted_password {
-                // Check whether password changed or not (trim 'md5' prefix first if any)
-                //
-                // This is a backward compatibility hack, which comes from the times when we were using
-                // md5 for everyone and hashes were stored in the console db without md5 prefix. So when
-                // role comes from the control-plane (json spec) `Role.encrypted_password` doesn't have md5 prefix,
-                // but when role comes from Postgres (`get_existing_roles` / `existing_roles`) it has this prefix.
-                // Here is the only place so far where we compare hashes, so it seems to be the best candidate
-                // to place this compatibility layer.
-                let pg_pwd = if let Some(stripped) = pg_pwd.strip_prefix("md5") {
-                    stripped
-                } else {
-                    pg_pwd
-                };
-                if pg_pwd != *role.encrypted_password.as_ref().unwrap() {
-                    RoleAction::Update
-                } else {
-                    RoleAction::None
-                }
-            } else {
-                RoleAction::None
-            }
-        } else {
-            RoleAction::Create
-        };
-
-        match action {
-            RoleAction::None => {}
-            RoleAction::Update => {
-                let mut query: String = format!("ALTER ROLE {} ", name.pg_quote());
-                query.push_str(&role.to_pg_options());
-                xact.execute(query.as_str(), &[])?;
-            }
-            RoleAction::Create => {
-                let mut query: String = format!("CREATE ROLE {} ", name.pg_quote());
-                info!("role create query: '{}'", &query);
-                query.push_str(&role.to_pg_options());
-                xact.execute(query.as_str(), &[])?;
-
-                let grant_query = format!(
-                    "GRANT pg_read_all_data, pg_write_all_data TO {}",
-                    name.pg_quote()
-                );
-                xact.execute(grant_query.as_str(), &[])?;
-                info!("role grant query: '{}'", &grant_query);
-            }
-        }
-
-        if span_enabled!(Level::INFO) {
-            let pwd = if role.encrypted_password.is_some() {
+        info_print!(
+            "{} - {}:{}",
+            " ".repeat(27 + 5),
+            name,
+            if role.encrypted_password.is_some() {
                "[FILTERED]"
            } else {
                "(null)"
-            };
-            let action_str = match action {
-                RoleAction::None => "",
-                RoleAction::Create => " -> create",
-                RoleAction::Update => " -> update",
-            };
-            info!("   - {}:{}{}", name, pwd, action_str);
+            }
+        );
+
+        // XXX: with a limited number of roles it is fine, but consider making it a HashMap
+        let pg_role = existing_roles.iter().find(|r| r.name == *name);
+
+        if let Some(r) = pg_role {
+            let mut update_role = false;
+
+            if (r.encrypted_password.is_none() && role.encrypted_password.is_some())
+                || (r.encrypted_password.is_some() && role.encrypted_password.is_none())
+            {
+                update_role = true;
+            } else if let Some(pg_pwd) = &r.encrypted_password {
+                // Check whether password changed or not (trim 'md5:' prefix first)
+                update_role = pg_pwd[3..] != *role.encrypted_password.as_ref().unwrap();
+            }
+
+            if update_role {
+                let mut query: String = format!("ALTER ROLE {} ", name.pg_quote());
+                info_print!(" -> update");
+
+                query.push_str(&role.to_pg_options());
+                xact.execute(query.as_str(), &[])?;
+            }
+        } else {
+            info!("role name: '{}'", &name);
+            let mut query: String = format!("CREATE ROLE {} ", name.pg_quote());
+            info!("role create query: '{}'", &query);
+            info_print!(" -> create");
+
+            query.push_str(&role.to_pg_options());
+            xact.execute(query.as_str(), &[])?;
+
+            let grant_query = format!(
+                "GRANT pg_read_all_data, pg_write_all_data TO {}",
+                name.pg_quote()
+            );
+            xact.execute(grant_query.as_str(), &[])?;
+            info!("role grant query: '{}'", &grant_query);
        }
+
+        info_print!("\n");
    }

    xact.commit()?;
@@ -220,32 +196,23 @@ pub fn handle_roles(spec: &ComputeSpec, client: &mut Client) -> Result<()> {
 }

 /// Reassign all dependent objects and delete requested roles.
-#[instrument(skip_all)]
 pub fn handle_role_deletions(node: &ComputeNode, client: &mut Client) -> Result<()> {
-    if let Some(ops) = &node.spec.delta_operations {
-        // First, reassign all dependent objects to db owners.
+    let spec = &node.spec;
+
+    // First, reassign all dependent objects to db owners.
+    if let Some(ops) = &spec.delta_operations {
        info!("reassigning dependent objects of to-be-deleted roles");
-
-        // Fetch existing roles. We could've exported and used `existing_roles` from
-        // `handle_roles()`, but we only make this list there before creating new roles.
-        // Which is probably fine as we never create to-be-deleted roles, but that'd
-        // just look a bit untidy. Anyway, the entire `pg_roles` should be in shared
-        // buffers already, so this shouldn't be a big deal.
-        let mut xact = client.transaction()?;
-        let existing_roles: Vec<Role> = get_existing_roles(&mut xact)?;
-        xact.commit()?;
-
        for op in ops {
-            // Check that role is still present in Postgres, as this could be a
-            // restart with the same spec after role deletion.
-            if op.action == "delete_role" && existing_roles.iter().any(|r| r.name == op.name) {
+            if op.action == "delete_role" {
                reassign_owned_objects(node, &op.name)?;
            }
        }
+    }

-        // Second, proceed with role deletions.
+    // Second, proceed with role deletions.
+    let mut xact = client.transaction()?;
+    if let Some(ops) = &spec.delta_operations {
        info!("processing role deletions");
-        let mut xact = client.transaction()?;
        for op in ops {
            // We do not check either role exists or not,
            // Postgres will take care of it for us
@@ -256,7 +223,6 @@ pub fn handle_role_deletions(node: &ComputeNode, client: &mut Client) -> Result<
                xact.execute(query.as_str(), &[])?;
            }
        }
-        xact.commit()?;
    }

    Ok(())
@@ -297,16 +263,13 @@ fn reassign_owned_objects(node: &ComputeNode, role_name: &PgIdent) -> Result<()>
 /// like `CREATE DATABASE` and `DROP DATABASE` do not support it. Statement-level
 /// atomicity should be enough here due to the order of operations and various checks,
 /// which together provide us idempotency.
-#[instrument(skip_all)]
 pub fn handle_databases(spec: &ComputeSpec, client: &mut Client) -> Result<()> {
    let existing_dbs: Vec<Database> = get_existing_dbs(client)?;

    // Print a list of existing Postgres databases (only in debug mode)
-    if span_enabled!(Level::INFO) {
-        info!("postgres databases:");
-        for r in &existing_dbs {
-            info!("    {}:{}", r.name, r.owner);
-        }
+    info!("postgres databases:");
+    for r in &existing_dbs {
+        info_println!("{} - {}:{}", " ".repeat(27 + 5), r.name, r.owner);
    }

    // Process delta operations first
@@ -349,15 +312,12 @@ pub fn handle_databases(spec: &ComputeSpec, client: &mut Client) -> Result<()> {
    for db in &spec.cluster.databases {
        let name = &db.name;

+        info_print!("{} - {}:{}", " ".repeat(27 + 5), db.name, db.owner);
+
        // XXX: with a limited number of databases it is fine, but consider making it a HashMap
        let pg_db = existing_dbs.iter().find(|r| r.name == *name);

-        enum DatabaseAction {
-            None,
-            Update,
-            Create,
-        }
-        let action = if let Some(r) = pg_db {
+        if let Some(r) = pg_db {
            // XXX: db owner name is returned as quoted string from Postgres,
            // when quoting is needed.
            let new_owner = if r.owner.starts_with('"') {
@@ -367,42 +327,24 @@ pub fn handle_databases(spec: &ComputeSpec, client: &mut Client) -> Result<()> {
            };

            if new_owner != r.owner {
-                // Update the owner
-                DatabaseAction::Update
-            } else {
-                DatabaseAction::None
-            }
-        } else {
-            DatabaseAction::Create
-        };
-
-        match action {
-            DatabaseAction::None => {}
-            DatabaseAction::Update => {
                let query: String = format!(
                    "ALTER DATABASE {} OWNER TO {}",
                    name.pg_quote(),
                    db.owner.pg_quote()
                );
-                let _ = info_span!("executing", query).entered();
-                client.execute(query.as_str(), &[])?;
-            }
-            DatabaseAction::Create => {
-                let mut query: String = format!("CREATE DATABASE {} ", name.pg_quote());
-                query.push_str(&db.to_pg_options());
-                let _ = info_span!("executing", query).entered();
-                client.execute(query.as_str(), &[])?;
-            }
-        };
+                info_print!(" -> update");

-        if span_enabled!(Level::INFO) {
-            let action_str = match action {
-                DatabaseAction::None => "",
-                DatabaseAction::Create => " -> create",
-                DatabaseAction::Update => " -> update",
-            };
-            info!("   - {}:{}{}", db.name, db.owner, action_str);
+                client.execute(query.as_str(), &[])?;
+            }
+        } else {
+            let mut query: String = format!("CREATE DATABASE {} ", name.pg_quote());
+            info_print!(" -> create");
+
+            query.push_str(&db.to_pg_options());
+            client.execute(query.as_str(), &[])?;
        }
+
+        info_print!("\n");
    }

    Ok(())
@@ -410,7 +352,6 @@ pub fn handle_databases(spec: &ComputeSpec, client: &mut Client) -> Result<()> {

 /// Grant CREATE ON DATABASE to the database owner and do some other alters and grants
 /// to allow users creating trusted extensions and re-creating `public` schema, for example.
-#[instrument(skip_all)]
 pub fn handle_grants(node: &ComputeNode, client: &mut Client) -> Result<()> {
    let spec = &node.spec;

@@ -1,31 +1,32 @@
 [package]
 name = "control_plane"
 version = "0.1.0"
-edition.workspace = true
-license.workspace = true
+edition = "2021"
+license = "Apache-2.0"

 [dependencies]
-anyhow.workspace = true
-clap.workspace = true
-comfy-table.workspace = true
-git-version.workspace = true
-nix.workspace = true
-once_cell.workspace = true
-postgres.workspace = true
-regex.workspace = true
-reqwest = { workspace = true, features = ["blocking", "json"] }
-serde.workspace = true
-serde_with.workspace = true
-tar.workspace = true
-thiserror.workspace = true
-toml.workspace = true
-url.workspace = true
+anyhow = "1.0"
+clap = "4.0"
+comfy-table = "6.1"
+git-version = "0.3.5"
+nix = "0.25"
+once_cell = "1.13.0"
+postgres = { git = "https://github.com/neondatabase/rust-postgres.git", rev = "43e6db254a97fdecbce33d8bc0890accfd74495e" }
+regex = "1"
+reqwest = { version = "0.11", default-features = false, features = ["blocking", "json", "rustls-tls"] }
+serde = { version = "1.0", features = ["derive"] }
+serde_with = "2.0"
+tar = "0.4.38"
+thiserror = "1"
+toml = "0.5"
+url = "2.2.2"
+
 # Note: Do not directly depend on pageserver or safekeeper; use pageserver_api or safekeeper_api
 # instead, so that recompile times are better.
-pageserver_api.workspace = true
-safekeeper_api.workspace = true
-postgres_connection.workspace = true
-storage_broker.workspace = true
-utils.workspace = true
-
-workspace_hack.workspace = true
+pageserver_api = { path = "../libs/pageserver_api" }
+postgres_connection = { path = "../libs/postgres_connection" }
+safekeeper_api = { path = "../libs/safekeeper_api" }
+# Note: main broker code is inside the binary crate, so linking with the library shouldn't be heavy.
+storage_broker = { version = "0.1", path = "../storage_broker" }
+utils = { path = "../libs/utils" }
+workspace_hack = { version = "0.1", path = "../workspace_hack" }
@@ -136,6 +136,22 @@ where
    anyhow::bail!("{process_name} did not start in {RETRY_UNTIL_SECS} seconds");
 }

+/// Send SIGTERM to child process
+pub fn send_stop_child_process(child: &std::process::Child) -> anyhow::Result<()> {
+    let pid = child.id();
+    match kill(
+        nix::unistd::Pid::from_raw(pid.try_into().unwrap()),
+        Signal::SIGTERM,
+    ) {
+        Ok(()) => Ok(()),
+        Err(Errno::ESRCH) => {
+            println!("child process with pid {pid} does not exist");
+            Ok(())
+        }
+        Err(e) => anyhow::bail!("Failed to send signal to child process with pid {pid}: {e}"),
+    }
+}
+
 /// Stops the process, using the pid file given. Returns Ok also if the process is already not running.
 pub fn stop_process(immediate: bool, process_name: &str, pid_file: &Path) -> anyhow::Result<()> {
    let pid = match pid_file::read(pid_file)
@@ -263,7 +263,7 @@ fn get_tenant_id(sub_match: &ArgMatches, env: &local_env::LocalEnv) -> anyhow::R
    } else if let Some(default_id) = env.default_tenant_id {
        Ok(default_id)
    } else {
-        anyhow::bail!("No tenant id. Use --tenant-id, or set a default tenant");
+        bail!("No tenant id. Use --tenant-id, or set 'default_tenant_id' in the config file");
    }
 }

@@ -284,6 +284,8 @@ fn parse_timeline_id(sub_match: &ArgMatches) -> anyhow::Result<Option<TimelineId
 }

 fn handle_init(init_match: &ArgMatches) -> anyhow::Result<LocalEnv> {
+    let initial_timeline_id_arg = parse_timeline_id(init_match)?;
+
    // Create config file
    let toml_file: String = if let Some(config_path) = init_match.get_one::<PathBuf>("config") {
        // load and parse the file
@@ -307,16 +309,30 @@ fn handle_init(init_match: &ArgMatches) -> anyhow::Result<LocalEnv> {
        LocalEnv::parse_config(&toml_file).context("Failed to create neon configuration")?;
    env.init(pg_version)
        .context("Failed to initialize neon repository")?;
+    let initial_tenant_id = env
+        .default_tenant_id
+        .expect("default_tenant_id should be generated by the `env.init()` call above");

    // Initialize pageserver, create initial tenant and timeline.
    let pageserver = PageServerNode::from_env(&env);
-    pageserver
-        .initialize(&pageserver_config_overrides(init_match))
+    let initial_timeline_id = pageserver
+        .initialize(
+            Some(initial_tenant_id),
+            initial_timeline_id_arg,
+            &pageserver_config_overrides(init_match),
+            pg_version,
+        )
        .unwrap_or_else(|e| {
            eprintln!("pageserver init failed: {e:?}");
            exit(1);
        });

+    env.register_branch_mapping(
+        DEFAULT_BRANCH_NAME.to_owned(),
+        initial_tenant_id,
+        initial_timeline_id,
+    )?;
+
    Ok(env)
 }

@@ -372,17 +388,6 @@ fn handle_tenant(tenant_match: &ArgMatches, env: &mut local_env::LocalEnv) -> an
            println!(
                "Created an initial timeline '{new_timeline_id}' at Lsn {last_record_lsn} for tenant: {new_tenant_id}",
            );
-
-            if create_match.get_flag("set-default") {
-                println!("Setting tenant {new_tenant_id} as a default one");
-                env.default_tenant_id = Some(new_tenant_id);
-            }
-        }
-        Some(("set-default", set_default_match)) => {
-            let tenant_id =
-                parse_tenant_id(set_default_match)?.context("No tenant id specified")?;
-            println!("Setting tenant {tenant_id} as a default one");
-            env.default_tenant_id = Some(tenant_id);
        }
        Some(("config", create_match)) => {
            let tenant_id = get_tenant_id(create_match, env)?;
@@ -923,8 +928,9 @@ fn cli() -> Command {
        .version(GIT_VERSION)
        .subcommand(
            Command::new("init")
-                .about("Initialize a new Neon repository, preparing configs for services to start with")
+                .about("Initialize a new Neon repository")
                .arg(pageserver_config_args.clone())
+                .arg(timeline_id_arg.clone().help("Use a specific timeline id when creating a tenant and its initial timeline"))
                .arg(
                    Arg::new("config")
                        .long("config")
@@ -986,14 +992,11 @@ fn cli() -> Command {
                .arg(timeline_id_arg.clone().help("Use a specific timeline id when creating a tenant and its initial timeline"))
                .arg(Arg::new("config").short('c').num_args(1).action(ArgAction::Append).required(false))
                .arg(pg_version_arg.clone())
-                .arg(Arg::new("set-default").long("set-default").action(ArgAction::SetTrue).required(false)
-                    .help("Use this tenant in future CLI commands where tenant_id is needed, but not specified"))
                )
-            .subcommand(Command::new("set-default").arg(tenant_id_arg.clone().required(true))
-                .about("Set a particular tenant as default in future CLI commands where tenant_id is needed, but not specified"))
            .subcommand(Command::new("config")
                .arg(tenant_id_arg.clone())
-                .arg(Arg::new("config").short('c').num_args(1).action(ArgAction::Append).required(false)))
+                .arg(Arg::new("config").short('c').num_args(1).action(ArgAction::Append).required(false))
+                )
        )
        .subcommand(
            Command::new("pageserver")
@@ -296,6 +296,11 @@ impl LocalEnv {
            env.neon_distrib_dir = env::current_exe()?.parent().unwrap().to_owned();
        }

+        // If no initial tenant ID was given, generate it.
+        if env.default_tenant_id.is_none() {
+            env.default_tenant_id = Some(TenantId::generate());
+        }
+
        env.base_data_dir = base_path();

        Ok(env)
@@ -7,7 +7,7 @@ use std::path::PathBuf;
 use std::process::{Child, Command};
 use std::{io, result};

-use anyhow::{bail, Context};
+use anyhow::{bail, ensure, Context};
 use pageserver_api::models::{
    TenantConfigRequest, TenantCreateRequest, TenantInfo, TimelineCreateRequest, TimelineInfo,
 };
@@ -130,15 +130,83 @@ impl PageServerNode {
        overrides
    }

-    /// Initializes a pageserver node by creating its config with the overrides provided.
-    pub fn initialize(&self, config_overrides: &[&str]) -> anyhow::Result<()> {
+    /// Initializes a pageserver node by creating its config with the overrides provided,
+    /// and creating an initial tenant and timeline afterwards.
+    pub fn initialize(
+        &self,
+        create_tenant: Option<TenantId>,
+        initial_timeline_id: Option<TimelineId>,
+        config_overrides: &[&str],
+        pg_version: u32,
+    ) -> anyhow::Result<TimelineId> {
        // First, run `pageserver --init` and wait for it to write a config into FS and exit.
        self.pageserver_init(config_overrides).with_context(|| {
            format!(
                "Failed to run init for pageserver node {}",
                self.env.pageserver.id,
            )
-        })
+        })?;
+
+        // Then, briefly start it fully to run HTTP commands on it,
+        // to create initial tenant and timeline.
+        // We disable the remote storage, since we stop pageserver right after the timeline creation,
+        // hence most of the uploads will either aborted or not started: no point to start them at all.
+        let disabled_remote_storage_override = "remote_storage={}";
+        let mut pageserver_process = self
+            .start_node(
+                &[disabled_remote_storage_override],
+                // Previous overrides will be taken from the config created before, don't overwrite them.
+                false,
+            )
+            .with_context(|| {
+                format!(
+                    "Failed to start a process for pageserver node {}",
+                    self.env.pageserver.id,
+                )
+            })?;
+
+        let init_result = self
+            .try_init_timeline(create_tenant, initial_timeline_id, pg_version)
+            .context("Failed to create initial tenant and timeline for pageserver");
+        match &init_result {
+            Ok(initial_timeline_id) => {
+                println!("Successfully initialized timeline {initial_timeline_id}")
+            }
+            Err(e) => eprintln!("{e:#}"),
+        }
+        background_process::send_stop_child_process(&pageserver_process)?;
+
+        let exit_code = pageserver_process.wait()?;
+        ensure!(
+            exit_code.success(),
+            format!(
+                "pageserver init failed with exit code {:?}",
+                exit_code.code()
+            )
+        );
+        println!(
+            "Stopped pageserver {} process with pid {}",
+            self.env.pageserver.id,
+            pageserver_process.id(),
+        );
+        init_result
+    }
+
+    fn try_init_timeline(
+        &self,
+        new_tenant_id: Option<TenantId>,
+        new_timeline_id: Option<TimelineId>,
+        pg_version: u32,
+    ) -> anyhow::Result<TimelineId> {
+        let initial_tenant_id = self.tenant_create(new_tenant_id, HashMap::new())?;
+        let initial_timeline_info = self.timeline_create(
+            initial_tenant_id,
+            new_timeline_id,
+            None,
+            None,
+            Some(pg_version),
+        )?;
+        Ok(initial_timeline_info.timeline_id)
    }

    pub fn repo_path(&self) -> PathBuf {
@@ -52,7 +52,7 @@ name = "ring"
 version = "*"
 expression = "MIT AND ISC AND OpenSSL"
 license-files = [
-    { path = "LICENSE", hash = 0xbd0eed23 }
+    { path = "LICENSE", hash = 0xbd0eed23 },
 ]

 [licenses.private]
@@ -1,115 +0,0 @@
-### Overview
-Pageserver and proxy periodically collect consumption metrics and push them to a HTTP endpoint.
-
-This doc describes current implementation details.
-For design details see [the RFC](./rfcs/021-metering.md) and [the discussion on Github](https://github.com/neondatabase/neon/pull/2884).
-
- The metrics are collected in a separate thread, and the collection interval and endpoint are configurable.
-
- Metrics are cached, so that we don't send unchanged metrics on every iteration.
-
- Metrics are sent in batches of 1000 (see CHUNK_SIZE const) metrics max with no particular grouping guarantees.
-
-batch format is
-```json
-
-{ "events" : [metric1, metric2, ...]]}
-
-```
-See metric format examples below.
-
- All metrics values are in bytes, unless otherwise specified.
-
- Currently no retries are implemented.
-
-### Pageserver metrics
-
-#### Configuration
-The endpoint and the collection interval are specified in the pageserver config file (or can be passed as command line arguments):
-`metric_collection_endpoint` defaults to None, which means that metric collection is disabled by default.
-`metric_collection_interval` defaults to 10min
-
-#### Metrics
-
-Currently, the following metrics are collected:
-
- `written_size`
-
-Amount of WAL produced , by a timeline, i.e. last_record_lsn
-This is an absolute, per-timeline metric.
-
- `resident_size`
-
-Size of all the layer files in the tenant's directory on disk on the pageserver.
-This is an absolute, per-tenant metric.
-
- `remote_storage_size`
-
-Size of the remote storage (S3) directory.
-This is an absolute, per-tenant metric.
-
- `timeline_logical_size`
-Logical size of the data in the timeline
-This is an absolute, per-timeline metric.
-
- `synthetic_storage_size`
-Size of all tenant's branches including WAL
-This is the same metric that `tenant/{tenant_id}/size` endpoint returns.
-This is an absolute, per-tenant metric.
-
-Synthetic storage size is calculated in a separate thread, so it might be slightly outdated.
-
-#### Format example
-
-```json
-{
-"metric": "remote_storage_size",
-"type": "absolute",
-"time": "2022-12-28T11:07:19.317310284Z",
-"idempotency_key": "2022-12-28 11:07:19.317310324 UTC-1-4019",
-"value": 12345454,
-"tenant_id": "5d07d9ce9237c4cd845ea7918c0afa7d",
-"timeline_id": "a03ebb4f5922a1c56ff7485cc8854143",
-}
-```
-
-`idempotency_key` is a unique key for each metric, so that we can deduplicate metrics.
-It is a combination of the time, node_id and a random number.
-
-### Proxy consumption metrics
-
-#### Configuration
-The endpoint and the collection interval can be passed as command line arguments for proxy:
-`metric_collection_endpoint` no default, which means that metric collection is disabled by default.
-`metric_collection_interval` no default
-
-#### Metrics
-
-Currently, only one proxy metric is collected:
-
- `proxy_io_bytes_per_client`
-Outbound traffic per client.
-This is an incremental, per-endpoint metric.
-
-#### Format example
-
-```json
-{
-"metric": "proxy_io_bytes_per_client",
-"type": "incremental",
-"start_time": "2022-12-28T11:07:19.317310284Z",
-"stop_time": "2022-12-28T11:07:19.317310284Z",
-"idempotency_key": "2022-12-28 11:07:19.317310324 UTC-1-4019",
-"value": 12345454,
-"endpoint_id": "5d07d9ce9237c4cd845ea7918c0afa7d",
-}
-```
-
-The metric is incremental, so the value is the difference between the current and the previous value.
-If there is no previous value, the value, the value is the current value and the `start_time` equals `stop_time`.
-
-### TODO
-
- [ ] Handle errors better: currently if one tenant fails to gather metrics, the whole iteration fails and metrics are not sent for any tenant.
- [ ] Add retries
- [ ] Tune the interval
@@ -1,186 +0,0 @@
-# Consumption tracking
-
-
-# Goals
-
-This proposal is made with two mostly but not entirely overlapping goals:
-
-* Collect info that is needed for consumption-based billing
-* Cross-check AWS bills
-
-
-# Metrics
-
-There are six metrics to collect:
-
-* CPU time. Wall clock seconds * the current number of cores. We have a fixed ratio of memory to cores, so the current memory size is the function of the number of cores. Measured per each `endpoint`.
-
-* Traffic. In/out traffic on the proxy. Measured per each `endpoint`.
-
-* Written size. Amount of data we write. That is different from both traffic and storage size, as only during the writing we
-
-  a) occupy some disk bandwidth on safekeepers
-
-  b) necessarily cross AZ boundaries delivering WAL to all safekeepers
-
-  Each timeline/branch has at most one writer, so the data is collected per branch.
-
-* Synthetic storage size. That is what is exposed now with pageserver's `/v1/tenant/{}/size`. Looks like now it is per-tenant. (Side note: can we make it per branch to show as branch physical size in UI?)
-
-* Real storage size. That is the size of the tenant directory on the pageservers disk. Per-tenant.
-
-* S3 storage size. That is the size of the tenant data on S3. Per-tenant.
-
-That info should be enough to build an internal model that predicts AWS price (hence tracking `written data` and `real storage size`). As for the billing model we probably can get away with mentioning only `CPU time`, `synthetic storage size`, and `traffic` consumption.
-
-# Services participating in metrics collection
-
-## Proxy
-
-For actual implementation details check `/docs/consumption_metrics.md`
-
-Proxy is the only place that knows about traffic flow, so it tracks it and reports it with quite a small interval, let's say 1 minute. A small interval is needed here since the proxy is stateless, and any restart will reset accumulated consumption. Also proxy should report deltas since the last report, not an absolute value of the counter. Such kind of events is easier to integrate over a period of time to get the amount of traffic during some time interval.
-
-Example event:
-
-```json
-{
-"metric": "proxy_io_bytes_per_client",
-"type": "incremental",
-"start_time": "2022-12-28T11:07:19.317310284Z",
-"stop_time": "2022-12-28T11:07:19.317310284Z",
-"idempotency_key": "2022-12-28 11:07:19.317310324 UTC-1-4019",
-"value": 12345454,
-"endpoint_id": "5d07d9ce9237c4cd845ea7918c0afa7d",
-}
-```
-
-Since we report deltas over some period of time, it makes sense to include `event_start_time`/`event_stop_time` where `event_start_time` is the time of the previous report. That will allow us to identify metering gaps better (e.g., failed send/delivery).
-
-When there is no active connection proxy can avoid reporting anything. Also, deltas are additive, so several console instances serving the same user and endpoint can report traffic without coordination.
-
-## Console
-
-The console knows about start/stop events, so it knows the amount of CPU time allocated to each endpoint. It also knows about operation successes and failures and can avoid billing clients after unsuccessful 'suspend' events. The console doesn't know the current compute size within the allowed limits on the endpoint. So with CPU time, we do the following:
-
-* While we don't yet have the autoscaling console can report `cpu time` as the number of seconds since the last `start_compute` event.
-
-* When we have autoscaling, `autoscaler-agent` can report `cpu time`*`compute_units_count` in the same increments as the proxy reports traffic.
-
-Example event:
-
-```json
-{
-    "metric": "effective_compute_seconds",
-    "type": "increment",
-    "endpoint_id": "blazing-warrior-34",
-    "event_start_time": ...,
-    "event_stop_time": ...,
-    "value": 12345454,
-}
-```
-
-I'd also suggest reporting one value, `cpu time`*`compute_units_count`, instead of two separate fields as it makes event schema simpler (it is possible to treat it the same way as traffic) and preserves additivity.
-
-## Pageserver
-
-For actual implementation details check `/docs/consumption_metrics.md`
-
-Pageserver knows / has access to / can calculate the rest of the metrics:
-
-* Written size -- that is basically `last_received_lsn`,
-* Synthetic storage size -- there is a way to calculate it, albeit a costly one,
-* Real storage size -- there is a way to calculate it using a layer map or filesystem,
-* S3 storage size -- can calculate it by S3 API calls
-
-Some of those metrics are expensive to calculate, so the reporting period here is driven mainly by implementation details. We can set it to, for example, once per hour. Not a big deal since the pageserver is stateful, and all metrics can be reported as an absolute value, not increments. At the same time, a smaller reporting period improves UX, so it would be good to have something more real-time.
-
-`written size` is primarily a safekeeper-related metric, but since it is available on both pageserver and safekeeper, we can avoid reporting anything from the safekeeper.
-
-Example event:
-
-```json
-{
-"metric": "remote_storage_size",
-"type": "absolute",
-"time": "2022-12-28T11:07:19.317310284Z",
-"idempotency_key": "2022-12-28 11:07:19.317310324 UTC-1-4019",
-"value": 12345454,
-"tenant_id": "5d07d9ce9237c4cd845ea7918c0afa7d",
-"timeline_id": "a03ebb4f5922a1c56ff7485cc8854143",
-}
-```
-
-# Data collection
-
-## Push vs. pull
-
-We already have pull-based Prometheus metrics, so it is tempting to use them here too. However, in our setup, it is hard to tell when some metric changes. For example, garbage collection will constantly free some disk space over a week, even if the project is down for that week. We could also iterate through all existing tenants/branches/endpoints, but that means some amount of code to do that properly and most likely we will end up with some per-metric hacks in the collector to cut out some of the tenants that are surely not changing that metric.
-
-With the push model, it is easier to publish data only about actively changing metrics -- pageserver knows when it performs s3 offloads, garbage collection and starts/stops consuming data from the safekeeper; proxy knows about connected clients; console / autoscaler-agent knows about active cpu time.
-
-Hence, let's go with a push-based model.
-
-## Common bus vs. proxying through the console
-
-We can implement such push systems in a few ways:
-
-a. Each component pushes its metrics to the "common bus", namely segment, Kafka, or something similar. That approach scales well, but it would be harder to test it locally, will introduce new dependencies, we will have to distribute secrets for that connection to all of the components, etc. We would also have to loop back some of the events and their aggregates to the console, as we want to show some that metrics to the user in real-time.
-
-b. Each component can call HTTP `POST` with its events to the console, and the console can forward it to the segment for later integration with metronome / orb / onebill / etc. With that approach, only the console has to speak with segment. Also since that data passes through the console, the console can save the latest metrics values, so there is no need for constant feedback of that events back from the segment.
-
-# Implementation
-
-Each (proxy|pageserver|autoscaler-agent) sends consumption events to the single endpoint in the console:
-
-```json
-POST /usage_events HTTP/1.1
-Content-Type: application/json
-
-[
-{
-"metric": "remote_storage_size",
-"type": "absolute",
-"time": "2022-12-28T11:07:19.317310284Z",
-"idempotency_key": "2022-12-28 11:07:19.317310324 UTC-1-4019",
-"value": 12345454,
-"tenant_id": "5d07d9ce9237c4cd845ea7918c0afa7d",
-"timeline_id": "a03ebb4f5922a1c56ff7485cc8854143",
-},
-...
-]
-```
-
-![data flow](./images/metering.jpg)
-
-Events could be either:
-* `incremental` -- change in consumption since the previous event or service restart. That is `effective_cpu_seconds`, `traffic_in_bytes`, and `traffic_out_bytes`.
-* `absolute` -- that is the current value of a metric. All of the size-related metrics are absolute.
-
-Each service can post events at its own pace and bundle together data from different tenants/endpoints.
-
-The console algorithm upon receive of events could be the following:
-
-1. Create and send a segment event with the same content (possibly enriching it with tenant/timeline data for endpoint-based events).
-2. Update the latest state of per-tenant and per-endpoint metrics in the database.
-3. Check whether any of that metrics is above the allowed threshold and stop the project if necessary.
-
-Since all the data comes in batches, we can do the batch update to reduce the number of queries in the database. Proxy traffic is probably the most frequent metric, so with batching, we will have extra `number_of_proxies` requests to the database each minute. This is most likely fine for now but will generate many dead tuples in the console database. If that is the case, we can change step 2 to the following:
-
-2.1. Check if there $tenant_$metric / $endpoint_$metric key in Redis
-
-2.2. If no stored value is found and the metric is incremental, then fetch the current value from DWH (which keeps aggregated value for all the events) and publish it.
-
-2.3. Publish a new value (absolute metric) or add an increment to the stored value (incremental metric)
-
-## Consumption watchdog
-
-Since all the data goes through the console, we don't have to run any background thread/coroutines to check whether consumption is within the allowed limits. We only change consumption with `POST /usage_events`, so limit checks could be applied in the same handler.
-
-## Extensibility
-
-If we need to add a new metric (e.g. s3 traffic or something else), the console code should, by default, process it and publish segment event, even if the metric name is unknown to the console.
-
-## Naming & schema
-
-Each metric name should end up with units -- now `_seconds` and `_bytes`, and segment event should always have `tenant_id` and `timeline_id`/`endpoint_id` where applicable.
@@ -18,6 +18,10 @@ Intended to be used in integration tests and in CLI tools for local installation
 Documentation of the Neon features and concepts.
 Now it is mostly dev documentation.

+`/monitoring`:
+
+TODO
+
 `/pageserver`:

 Neon storage service.
@@ -94,13 +98,6 @@ cargo hakari manage-deps

 If you don't have hakari installed (`error: no such subcommand: hakari`), install it by running `cargo install cargo-hakari`.

-### Checking Rust 3rd-parties
-[Cargo deny](https://embarkstudios.github.io/cargo-deny/index.html) is a cargo plugin that lets us lint project's dependency graph to ensure all dependencies conform to requirements. It detects security issues, matches licenses, and ensures crates only come from trusted sources.
-
-```bash
-cargo deny check
-```
-
 ## Using Python
 Note that Debian/Ubuntu Python packages are stale, as it commonly happens,
 so manual installation of dependencies is not recommended.
@@ -1,16 +0,0 @@
-[package]
-name = "consumption_metrics"
-version = "0.1.0"
-edition = "2021"
-license = "Apache-2.0"
-
-# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
-
-[dependencies]
-anyhow = "1.0.68"
-chrono = { version = "0.4", default-features = false, features = ["clock", "serde"] }
-rand = "0.8.3"
-serde = "1.0.152"
-serde_with = "2.1.0"
-utils = { version = "0.1.0", path = "../utils" }
-workspace_hack = { version = "0.1.0", path = "../../workspace_hack" }
@@ -1,50 +0,0 @@
-//!
-//! Shared code for consumption metics collection
-//!
-use chrono::{DateTime, Utc};
-use rand::Rng;
-use serde::Serialize;
-
-#[derive(Serialize, Debug, Clone, Eq, PartialEq, Ord, PartialOrd)]
-#[serde(tag = "type")]
-pub enum EventType {
-    #[serde(rename = "absolute")]
-    Absolute { time: DateTime<Utc> },
-    #[serde(rename = "incremental")]
-    Incremental {
-        start_time: DateTime<Utc>,
-        stop_time: DateTime<Utc>,
-    },
-}
-
-#[derive(Serialize, Debug, Clone, Eq, PartialEq, Ord, PartialOrd)]
-pub struct Event<Extra> {
-    #[serde(flatten)]
-    #[serde(rename = "type")]
-    pub kind: EventType,
-
-    pub metric: &'static str,
-    pub idempotency_key: String,
-    pub value: u64,
-
-    #[serde(flatten)]
-    pub extra: Extra,
-}
-
-pub fn idempotency_key(node_id: String) -> String {
-    format!(
-        "{}-{}-{:04}",
-        Utc::now(),
-        node_id,
-        rand::thread_rng().gen_range(0..=9999)
-    )
-}
-
-pub const CHUNK_SIZE: usize = 1000;
-
-// Just a wrapper around a slice of events
-// to serialize it as `{"events" : [ ] }
-#[derive(serde::Serialize)]
-pub struct EventChunk<'a, T> {
-    pub events: &'a [T],
-}
@@ -1,12 +1,11 @@
 [package]
 name = "metrics"
 version = "0.1.0"
-edition.workspace = true
-license.workspace = true
+edition = "2021"
+license = "Apache-2.0"

 [dependencies]
-prometheus.workspace = true
-libc.workspace = true
-once_cell.workspace = true
-
-workspace_hack.workspace = true
+prometheus = {version = "0.13", default_features=false, features = ["process"]} # removes protobuf dependency
+libc = "0.2"
+once_cell = "1.13.0"
+workspace_hack = { version = "0.1", path = "../../workspace_hack" }
@@ -1,17 +1,17 @@
 [package]
 name = "pageserver_api"
 version = "0.1.0"
-edition.workspace = true
-license.workspace = true
+edition = "2021"
+license = "Apache-2.0"

 [dependencies]
-serde.workspace = true
-serde_with.workspace = true
-const_format.workspace = true
-anyhow.workspace = true
-bytes.workspace = true
-byteorder.workspace = true
-utils.workspace = true
-postgres_ffi.workspace = true
+serde = { version = "1.0", features = ["derive"] }
+serde_with = "2.0"
+const_format = "0.2.21"
+anyhow = { version = "1.0", features = ["backtrace"] }
+bytes = "1.0.1"
+byteorder = "1.4.3"

-workspace_hack.workspace = true
+utils = { path = "../utils" }
+postgres_ffi = { path = "../postgres_ffi" }
+workspace_hack = { version = "0.1", path = "../../workspace_hack" }
@@ -1,4 +1,4 @@
-use std::num::{NonZeroU64, NonZeroUsize};
+use std::num::NonZeroU64;

 use byteorder::{BigEndian, ReadBytesExt};
 use serde::{Deserialize, Serialize};
@@ -44,17 +44,18 @@ impl TenantState {
 /// A state of a timeline in pageserver's memory.
 #[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
 pub enum TimelineState {
-    /// The timeline is recognized by the pageserver but is not yet operational.
-    /// In particular, the walreceiver connection loop is not running for this timeline.
-    /// It will eventually transition to state Active or Broken.
-    Loading,
-    /// The timeline is fully operational.
-    /// It can be queried, and the walreceiver connection loop is running.
+    /// Timeline is fully operational. If the containing Tenant is Active, the timeline's
+    /// background jobs are running otherwise they will be launched when the tenant is activated.
    Active,
-    /// The timeline was previously Loading or Active but is shutting down.
-    /// It cannot transition back into any other state.
+    /// A timeline is recognized by pageserver, but not yet ready to operate.
+    /// The status indicates, that the timeline could eventually go back to Active automatically:
+    /// for example, if the owning tenant goes back to Active again.
+    Suspended,
+    /// A timeline is recognized by pageserver, but not yet ready to operate and not allowed to
+    /// automatically become Active after certain events: only a management call can change this status.
    Stopping,
-    /// The timeline is broken and not operational (previous states: Loading or Active).
+    /// A timeline is recognized by the pageserver, but can no longer be used for
+    /// any operations, because it failed to be activated.
    Broken,
 }

@@ -209,11 +210,6 @@ pub struct TimelineInfo {
    pub state: TimelineState,
 }

-#[derive(Debug, Serialize, Deserialize)]
-pub struct DownloadRemoteLayersTaskSpawnRequest {
-    pub max_concurrent_downloads: NonZeroUsize,
-}
-
 #[derive(Debug, Serialize, Deserialize, Clone)]
 pub struct DownloadRemoteLayersTaskInfo {
    pub task_id: String,
@@ -1,17 +1,18 @@
 [package]
 name = "postgres_connection"
 version = "0.1.0"
-edition.workspace = true
-license.workspace = true
+edition = "2021"
+license = "Apache-2.0"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html

 [dependencies]
-anyhow.workspace = true
-itertools.workspace = true
-postgres.workspace = true
-tokio-postgres.workspace = true
-url.workspace = true
-
-workspace_hack.workspace = true
+anyhow = "1.0"
+itertools = "0.10.3"
+postgres = { git = "https://github.com/neondatabase/rust-postgres.git", rev = "43e6db254a97fdecbce33d8bc0890accfd74495e" }
+tokio-postgres = { git = "https://github.com/neondatabase/rust-postgres.git", rev="43e6db254a97fdecbce33d8bc0890accfd74495e" }
+url = "2.2.2"
+workspace_hack = { version = "0.1", path = "../../workspace_hack" }

 [dev-dependencies]
-once_cell.workspace = true
+once_cell = "1.13.0"
@@ -1,31 +1,30 @@
 [package]
 name = "postgres_ffi"
 version = "0.1.0"
-edition.workspace = true
-license.workspace = true
+edition = "2021"
+license = "Apache-2.0"

 [dependencies]
-rand.workspace = true
-regex.workspace = true
-bytes.workspace = true
-byteorder.workspace = true
-anyhow.workspace = true
-crc32c.workspace = true
-hex.workspace = true
-once_cell.workspace = true
-log.workspace = true
-memoffset.workspace = true
-thiserror.workspace = true
-serde.workspace = true
-utils.workspace = true
-
-workspace_hack.workspace = true
+rand = "0.8.3"
+regex = "1.4.5"
+bytes = "1.0.1"
+byteorder = "1.4.3"
+anyhow = "1.0"
+crc32c = "0.6.0"
+hex = "0.4.3"
+once_cell = "1.13.0"
+log = "0.4.14"
+memoffset = "0.7"
+thiserror = "1.0"
+serde = { version = "1.0", features = ["derive"] }
+utils = { path = "../utils" }
+workspace_hack = { version = "0.1", path = "../../workspace_hack" }

 [dev-dependencies]
-env_logger.workspace = true
-postgres.workspace = true
+env_logger = "0.9"
+postgres = { git = "https://github.com/neondatabase/rust-postgres.git", rev="43e6db254a97fdecbce33d8bc0890accfd74495e" }
 wal_craft = { path = "wal_craft" }

 [build-dependencies]
-anyhow.workspace = true
-bindgen.workspace = true
+anyhow = "1.0"
+bindgen = "0.61"
@@ -1,17 +1,17 @@
 [package]
 name = "wal_craft"
 version = "0.1.0"
-edition.workspace = true
-license.workspace = true
+edition = "2021"
+license = "Apache-2.0"
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html

 [dependencies]
-anyhow.workspace = true
-clap.workspace = true
-env_logger.workspace = true
-log.workspace = true
-once_cell.workspace = true
-postgres.workspace = true
-postgres_ffi.workspace = true
-tempfile.workspace = true
-
-workspace_hack.workspace = true
+anyhow = "1.0"
+clap = "4.0"
+env_logger = "0.9"
+log = "0.4"
+once_cell = "1.13.0"
+postgres = { git = "https://github.com/neondatabase/rust-postgres.git", rev="43e6db254a97fdecbce33d8bc0890accfd74495e" }
+postgres_ffi = { path = "../" }
+tempfile = "3.2"
+workspace_hack = { version = "0.1", path = "../../../workspace_hack" }
@@ -1,18 +1,17 @@
 [package]
 name = "pq_proto"
 version = "0.1.0"
-edition.workspace = true
-license.workspace = true
+edition = "2021"
+license = "Apache-2.0"

 [dependencies]
-anyhow.workspace = true
-bytes.workspace = true
-pin-project-lite.workspace = true
-postgres-protocol.workspace = true
-rand.workspace = true
-serde.workspace = true
-tokio.workspace = true
-tracing.workspace = true
-thiserror.workspace = true
+anyhow = "1.0"
+bytes = "1.0.1"
+pin-project-lite = "0.2.7"
+postgres-protocol = { git = "https://github.com/neondatabase/rust-postgres.git", rev="43e6db254a97fdecbce33d8bc0890accfd74495e" }
+rand = "0.8.3"
+serde = { version = "1.0", features = ["derive"] }
+tokio = { version = "1.17", features = ["macros"] }
+tracing = "0.1"

-workspace_hack.workspace = true
+workspace_hack = { version = "0.1", path = "../../workspace_hack" }
@@ -5,7 +5,7 @@
 // Tools for calling certain async methods in sync contexts.
 pub mod sync;

-use anyhow::{ensure, Context, Result};
+use anyhow::{bail, ensure, Context, Result};
 use bytes::{Buf, BufMut, Bytes, BytesMut};
 use postgres_protocol::PG_EPOCH;
 use serde::{Deserialize, Serialize};
@@ -194,35 +194,6 @@ macro_rules! retry_read {
    };
 }

-/// An error occured during connection being open.
-#[derive(thiserror::Error, Debug)]
-pub enum ConnectionError {
-    /// IO error during writing to or reading from the connection socket.
-    #[error("Socket IO error: {0}")]
-    Socket(std::io::Error),
-    /// Invalid packet was received from client
-    #[error("Protocol error: {0}")]
-    Protocol(String),
-    /// Failed to parse a protocol mesage
-    #[error("Message parse error: {0}")]
-    MessageParse(anyhow::Error),
-}
-
-impl From<anyhow::Error> for ConnectionError {
-    fn from(e: anyhow::Error) -> Self {
-        Self::MessageParse(e)
-    }
-}
-
-impl ConnectionError {
-    pub fn into_io_error(self) -> io::Error {
-        match self {
-            ConnectionError::Socket(io) => io,
-            other => io::Error::new(io::ErrorKind::Other, other.to_string()),
-        }
-    }
-}
-
 impl FeMessage {
    /// Read one message from the stream.
    /// This function returns `Ok(None)` in case of EOF.
@@ -245,9 +216,7 @@ impl FeMessage {
    /// }
    /// ```
    #[inline(never)]
-    pub fn read(
-        stream: &mut (impl io::Read + Unpin),
-    ) -> Result<Option<FeMessage>, ConnectionError> {
+    pub fn read(stream: &mut (impl io::Read + Unpin)) -> anyhow::Result<Option<FeMessage>> {
        Self::read_fut(&mut AsyncishRead(stream)).wait()
    }

@@ -255,7 +224,7 @@ impl FeMessage {
    /// See documentation for `Self::read`.
    pub fn read_fut<Reader>(
        stream: &mut Reader,
-    ) -> SyncFuture<Reader, impl Future<Output = Result<Option<FeMessage>, ConnectionError>> + '_>
+    ) -> SyncFuture<Reader, impl Future<Output = anyhow::Result<Option<FeMessage>>> + '_>
    where
        Reader: tokio::io::AsyncRead + Unpin,
    {
@@ -269,21 +238,17 @@ impl FeMessage {
            let tag = match retry_read!(stream.read_u8().await) {
                Ok(b) => b,
                Err(e) if e.kind() == io::ErrorKind::UnexpectedEof => return Ok(None),
-                Err(e) => return Err(ConnectionError::Socket(e)),
+                Err(e) => return Err(e.into()),
            };

            // The message length includes itself, so it better be at least 4.
-            let len = retry_read!(stream.read_u32().await)
-                .map_err(ConnectionError::Socket)?
+            let len = retry_read!(stream.read_u32().await)?
                .checked_sub(4)
-                .ok_or_else(|| ConnectionError::Protocol("invalid message length".to_string()))?;
+                .context("invalid message length")?;

            let body = {
                let mut buffer = vec![0u8; len as usize];
-                stream
-                    .read_exact(&mut buffer)
-                    .await
-                    .map_err(ConnectionError::Socket)?;
+                stream.read_exact(&mut buffer).await?;
                Bytes::from(buffer)
            };

@@ -300,11 +265,7 @@ impl FeMessage {
                b'c' => Ok(Some(FeMessage::CopyDone)),
                b'f' => Ok(Some(FeMessage::CopyFail)),
                b'p' => Ok(Some(FeMessage::PasswordMessage(body))),
-                tag => {
-                    return Err(ConnectionError::Protocol(format!(
-                        "unknown message tag: {tag},'{body:?}'"
-                    )))
-                }
+                tag => bail!("unknown message tag: {},'{:?}'", tag, body),
            }
        })
    }
@@ -314,9 +275,7 @@ impl FeStartupPacket {
    /// Read startup message from the stream.
    // XXX: It's tempting yet undesirable to accept `stream` by value,
    // since such a change will cause user-supplied &mut references to be consumed
-    pub fn read(
-        stream: &mut (impl io::Read + Unpin),
-    ) -> Result<Option<FeMessage>, ConnectionError> {
+    pub fn read(stream: &mut (impl io::Read + Unpin)) -> anyhow::Result<Option<FeMessage>> {
        Self::read_fut(&mut AsyncishRead(stream)).wait()
    }

@@ -325,7 +284,7 @@ impl FeStartupPacket {
    // since such a change will cause user-supplied &mut references to be consumed
    pub fn read_fut<Reader>(
        stream: &mut Reader,
-    ) -> SyncFuture<Reader, impl Future<Output = Result<Option<FeMessage>, ConnectionError>> + '_>
+    ) -> SyncFuture<Reader, impl Future<Output = anyhow::Result<Option<FeMessage>>> + '_>
    where
        Reader: tokio::io::AsyncRead + Unpin,
    {
@@ -343,41 +302,31 @@ impl FeStartupPacket {
            let len = match retry_read!(stream.read_u32().await) {
                Ok(len) => len as usize,
                Err(e) if e.kind() == io::ErrorKind::UnexpectedEof => return Ok(None),
-                Err(e) => return Err(ConnectionError::Socket(e)),
+                Err(e) => return Err(e.into()),
            };

            #[allow(clippy::manual_range_contains)]
            if len < 4 || len > MAX_STARTUP_PACKET_LENGTH {
-                return Err(ConnectionError::Protocol(format!(
-                    "invalid message length {len}"
-                )));
+                bail!("invalid message length");
            }

-            let request_code =
-                retry_read!(stream.read_u32().await).map_err(ConnectionError::Socket)?;
+            let request_code = retry_read!(stream.read_u32().await)?;

            // the rest of startup packet are params
            let params_len = len - 8;
            let mut params_bytes = vec![0u8; params_len];
-            stream
-                .read_exact(params_bytes.as_mut())
-                .await
-                .map_err(ConnectionError::Socket)?;
+            stream.read_exact(params_bytes.as_mut()).await?;

            // Parse params depending on request code
            let req_hi = request_code >> 16;
            let req_lo = request_code & ((1 << 16) - 1);
            let message = match (req_hi, req_lo) {
                (RESERVED_INVALID_MAJOR_VERSION, CANCEL_REQUEST_CODE) => {
-                    if params_len != 8 {
-                        return Err(ConnectionError::Protocol(
-                            "expected 8 bytes for CancelRequest params".to_string(),
-                        ));
-                    }
+                    ensure!(params_len == 8, "expected 8 bytes for CancelRequest params");
                    let mut cursor = Cursor::new(params_bytes);
                    FeStartupPacket::CancelRequest(CancelKeyData {
-                        backend_pid: cursor.read_i32().await.map_err(ConnectionError::Socket)?,
-                        cancel_key: cursor.read_i32().await.map_err(ConnectionError::Socket)?,
+                        backend_pid: cursor.read_i32().await?,
+                        cancel_key: cursor.read_i32().await?,
                    })
                }
                (RESERVED_INVALID_MAJOR_VERSION, NEGOTIATE_SSL_CODE) => {
@@ -389,9 +338,7 @@ impl FeStartupPacket {
                    FeStartupPacket::GssEncRequest
                }
                (RESERVED_INVALID_MAJOR_VERSION, unrecognized_code) => {
-                    return Err(ConnectionError::Protocol(format!(
-                        "Unrecognized request code {unrecognized_code}"
-                    )));
+                    bail!("Unrecognized request code {}", unrecognized_code)
                }
                // TODO bail if protocol major_version is not 3?
                (major_version, minor_version) => {
@@ -399,21 +346,15 @@ impl FeStartupPacket {
                    // See `postgres: ProcessStartupPacket, build_startup_packet`.
                    let mut tokens = str::from_utf8(&params_bytes)
                        .context("StartupMessage params: invalid utf-8")?
-                        .strip_suffix('\0') // drop packet's own null
-                        .ok_or_else(|| {
-                            ConnectionError::Protocol(
-                                "StartupMessage params: missing null terminator".to_string(),
-                            )
-                        })?
+                        .strip_suffix('\0') // drop packet's own null terminator
+                        .context("StartupMessage params: missing null terminator")?
                        .split_terminator('\0');

                    let mut params = HashMap::new();
                    while let Some(name) = tokens.next() {
-                        let value = tokens.next().ok_or_else(|| {
-                            ConnectionError::Protocol(
-                                "StartupMessage params: key without value".to_string(),
-                            )
-                        })?;
+                        let value = tokens
+                            .next()
+                            .context("StartupMessage params: key without value")?;

                        params.insert(name.to_owned(), value.to_owned());
                    }
@@ -517,7 +458,7 @@ pub enum BeMessage<'a> {
    CloseComplete,
    // None means column is NULL
    DataRow(&'a [Option<&'a [u8]>]),
-    ErrorResponse(&'a str, Option<&'a [u8; 5]>),
+    ErrorResponse(&'a str),
    /// Single byte - used in response to SSLRequest/GSSENCRequest.
    EncryptionResponse(bool),
    NoData,
@@ -665,7 +606,7 @@ fn write_body<R>(buf: &mut BytesMut, f: impl FnOnce(&mut BytesMut) -> R) -> R {
 }

 /// Safe write of s into buf as cstring (String in the protocol).
-fn write_cstr(s: impl AsRef<[u8]>, buf: &mut BytesMut) -> io::Result<()> {
+fn write_cstr(s: impl AsRef<[u8]>, buf: &mut BytesMut) -> Result<(), io::Error> {
    let bytes = s.as_ref();
    if bytes.contains(&0) {
        return Err(io::Error::new(
@@ -685,7 +626,7 @@ fn read_cstr(buf: &mut Bytes) -> anyhow::Result<Bytes> {
    Ok(result)
 }

-pub const SQLSTATE_INTERNAL_ERROR: &[u8; 5] = b"XX000";
+const SQLSTATE_INTERNAL_ERROR: &str = "XX000\0";

 impl<'a> BeMessage<'a> {
    /// Write message to the given buf.
@@ -826,7 +767,10 @@ impl<'a> BeMessage<'a> {
            // First byte of each field represents type of this field. Set just enough fields
            // to satisfy rust-postgres client: 'S' -- severity, 'C' -- error, 'M' -- error
            // message text.
-            BeMessage::ErrorResponse(error_msg, pg_error_code) => {
+            BeMessage::ErrorResponse(error_msg) => {
+                // For all the errors set Severity to Error and error code to
+                // 'internal error'.
+
                // 'E' signalizes ErrorResponse messages
                buf.put_u8(b'E');
                write_body(buf, |buf| {
@@ -834,9 +778,7 @@ impl<'a> BeMessage<'a> {
                    buf.put_slice(b"ERROR\0");

                    buf.put_u8(b'C'); // SQLSTATE error code
-                    buf.put_slice(&terminate_code(
-                        pg_error_code.unwrap_or(SQLSTATE_INTERNAL_ERROR),
-                    ));
+                    buf.put_slice(SQLSTATE_INTERNAL_ERROR.as_bytes());

                    buf.put_u8(b'M'); // the message
                    write_cstr(error_msg, buf)?;
@@ -859,7 +801,7 @@ impl<'a> BeMessage<'a> {
                    buf.put_slice(b"NOTICE\0");

                    buf.put_u8(b'C'); // SQLSTATE error code
-                    buf.put_slice(&terminate_code(SQLSTATE_INTERNAL_ERROR));
+                    buf.put_slice(SQLSTATE_INTERNAL_ERROR.as_bytes());

                    buf.put_u8(b'M'); // the message
                    write_cstr(error_msg.as_bytes(), buf)?;
@@ -1147,12 +1089,3 @@ mod tests {
        let _ = FeStartupPacket::read_fut(stream).await;
    }
 }
-
-fn terminate_code(code: &[u8; 5]) -> [u8; 6] {
-    let mut terminated = [0; 6];
-    for (i, &elem) in code.iter().enumerate() {
-        terminated[i] = elem;
-    }
-
-    terminated
-}
@@ -1,28 +1,28 @@
 [package]
 name = "remote_storage"
 version = "0.1.0"
-edition.workspace = true
-license.workspace = true
+edition = "2021"
+license = "Apache-2.0"

 [dependencies]
-anyhow.workspace = true
-async-trait.workspace = true
-once_cell.workspace = true
-aws-smithy-http.workspace = true
-aws-types.workspace = true
-aws-config.workspace = true
-aws-sdk-s3.workspace = true
-hyper = { workspace = true, features = ["stream"] }
-serde.workspace = true
-serde_json.workspace = true
-tokio = { workspace = true, features = ["sync", "fs", "io-util"] }
-tokio-util.workspace = true
-toml_edit.workspace = true
-tracing.workspace = true
-metrics.workspace = true
-utils.workspace = true
+anyhow = { version = "1.0", features = ["backtrace"] }
+async-trait = "0.1"
+metrics = { version = "0.1", path = "../metrics" }
+utils = { version = "0.1", path = "../utils" }
+once_cell = "1.13.0"
+aws-smithy-http = "0.51.0"
+aws-types = "0.51.0"
+aws-config = { version = "0.51.0", default-features = false, features=["rustls"] }
+aws-sdk-s3 = "0.21.0"
+hyper = { version = "0.14", features = ["stream"] }
+serde = { version = "1.0", features = ["derive"] }
+serde_json = "1"
+tokio = { version = "1.17", features = ["sync", "macros", "fs", "io-util"] }
+tokio-util = { version = "0.7", features = ["io"] }
+toml_edit = { version = "0.14", features = ["easy"] }
+tracing = "0.1.27"

-workspace_hack.workspace = true
+workspace_hack = { version = "0.1", path = "../../workspace_hack" }

 [dev-dependencies]
-tempfile.workspace = true
+tempfile = "3.2"
@@ -1,13 +1,13 @@
 [package]
 name = "safekeeper_api"
 version = "0.1.0"
-edition.workspace = true
-license.workspace = true
+edition = "2021"
+license = "Apache-2.0"

 [dependencies]
-serde.workspace = true
-serde_with.workspace = true
-const_format.workspace = true
-utils.workspace = true
+serde = { version = "1.0", features = ["derive"] }
+serde_with = "2.0"
+const_format = "0.2.21"

-workspace_hack.workspace = true
+utils = { path = "../utils" }
+workspace_hack = { version = "0.1", path = "../../workspace_hack" }
@@ -1,11 +1,9 @@
 [package]
 name = "tenant_size_model"
 version = "0.1.0"
-edition.workspace = true
+edition = "2021"
 publish = false
-license.workspace = true
+license = "Apache-2.0"

 [dependencies]
-anyhow.workspace = true
-
-workspace_hack.workspace = true
+workspace_hack = { version = "0.1", path = "../../workspace_hack" }
@@ -1,8 +1,6 @@
 use std::borrow::Cow;
 use std::collections::HashMap;

-use anyhow::Context;
-
 /// Pricing model or history size builder.
 ///
 /// Maintains knowledge of the branches and their modifications. Generic over the branch name key
@@ -134,25 +132,22 @@ impl<K: std::hash::Hash + Eq + 'static> Storage<K> {
        op: Cow<'static, str>,
        lsn: u64,
        size: Option<u64>,
-    ) -> anyhow::Result<()>
-    where
+    ) where
        K: std::borrow::Borrow<Q>,
-        Q: std::hash::Hash + Eq + std::fmt::Debug,
+        Q: std::hash::Hash + Eq,
    {
-        let Some(lastseg_id) = self.branches.get(branch).copied() else { anyhow::bail!("branch not found: {branch:?}") };
+        let lastseg_id = *self.branches.get(branch).unwrap();
        let newseg_id = self.segments.len();
        let lastseg = &mut self.segments[lastseg_id];

        assert!(lsn > lastseg.end_lsn);

-        let Some(start_size) = lastseg.end_size else { anyhow::bail!("no end_size on latest segment for {branch:?}") };
-
        let newseg = Segment {
            op,
            parent: Some(lastseg_id),
            start_lsn: lastseg.end_lsn,
            end_lsn: lsn,
-            start_size,
+            start_size: lastseg.end_size.unwrap(),
            end_size: size,
            children_after: Vec::new(),
            needed: false,
@@ -161,8 +156,6 @@ impl<K: std::hash::Hash + Eq + 'static> Storage<K> {

        self.segments.push(newseg);
        *self.branches.get_mut(branch).expect("read already") = newseg_id;
-
-        Ok(())
    }

    /// Advances the branch with the named operation, by the relative LSN and logical size bytes.
@@ -172,24 +165,21 @@ impl<K: std::hash::Hash + Eq + 'static> Storage<K> {
        op: Cow<'static, str>,
        lsn_bytes: u64,
        size_bytes: i64,
-    ) -> anyhow::Result<()>
-    where
+    ) where
        K: std::borrow::Borrow<Q>,
-        Q: std::hash::Hash + Eq + std::fmt::Debug,
+        Q: std::hash::Hash + Eq,
    {
-        let Some(lastseg_id) = self.branches.get(branch).copied() else { anyhow::bail!("branch not found: {branch:?}") };
+        let lastseg_id = *self.branches.get(branch).unwrap();
        let newseg_id = self.segments.len();
        let lastseg = &mut self.segments[lastseg_id];

-        let Some(last_end_size) = lastseg.end_size else { anyhow::bail!("no end_size on latest segment for {branch:?}") };
-
        let newseg = Segment {
            op,
            parent: Some(lastseg_id),
            start_lsn: lastseg.end_lsn,
            end_lsn: lastseg.end_lsn + lsn_bytes,
-            start_size: last_end_size,
-            end_size: Some((last_end_size as i64 + size_bytes) as u64),
+            start_size: lastseg.end_size.unwrap(),
+            end_size: Some((lastseg.end_size.unwrap() as i64 + size_bytes) as u64),
            children_after: Vec::new(),
            needed: false,
        };
@@ -197,54 +187,50 @@ impl<K: std::hash::Hash + Eq + 'static> Storage<K> {

        self.segments.push(newseg);
        *self.branches.get_mut(branch).expect("read already") = newseg_id;
-        Ok(())
    }

-    pub fn insert<Q: ?Sized>(&mut self, branch: &Q, bytes: u64) -> anyhow::Result<()>
+    pub fn insert<Q: ?Sized>(&mut self, branch: &Q, bytes: u64)
    where
        K: std::borrow::Borrow<Q>,
-        Q: std::hash::Hash + Eq + std::fmt::Debug,
+        Q: std::hash::Hash + Eq,
    {
-        self.modify_branch(branch, "insert".into(), bytes, bytes as i64)
+        self.modify_branch(branch, "insert".into(), bytes, bytes as i64);
    }

-    pub fn update<Q: ?Sized>(&mut self, branch: &Q, bytes: u64) -> anyhow::Result<()>
+    pub fn update<Q: ?Sized>(&mut self, branch: &Q, bytes: u64)
    where
        K: std::borrow::Borrow<Q>,
-        Q: std::hash::Hash + Eq + std::fmt::Debug,
+        Q: std::hash::Hash + Eq,
    {
-        self.modify_branch(branch, "update".into(), bytes, 0i64)
+        self.modify_branch(branch, "update".into(), bytes, 0i64);
    }

-    pub fn delete<Q: ?Sized>(&mut self, branch: &Q, bytes: u64) -> anyhow::Result<()>
+    pub fn delete<Q: ?Sized>(&mut self, branch: &Q, bytes: u64)
    where
        K: std::borrow::Borrow<Q>,
-        Q: std::hash::Hash + Eq + std::fmt::Debug,
+        Q: std::hash::Hash + Eq,
    {
-        self.modify_branch(branch, "delete".into(), bytes, -(bytes as i64))
+        self.modify_branch(branch, "delete".into(), bytes, -(bytes as i64));
    }

-    pub fn branch<Q: ?Sized>(&mut self, parent: &Q, name: K) -> anyhow::Result<()>
+    /// Panics if the parent branch cannot be found.
+    pub fn branch<Q: ?Sized>(&mut self, parent: &Q, name: K)
    where
-        K: std::borrow::Borrow<Q> + std::fmt::Debug,
-        Q: std::hash::Hash + Eq + std::fmt::Debug,
+        K: std::borrow::Borrow<Q>,
+        Q: std::hash::Hash + Eq,
    {
        // Find the right segment
-        let branchseg_id = *self.branches.get(parent).with_context(|| {
-            format!(
-                "should had found the parent {:?} by key. in branches {:?}",
-                parent, self.branches
-            )
-        })?;
-
+        let branchseg_id = *self
+            .branches
+            .get(parent)
+            .expect("should had found the parent by key");
        let _branchseg = &mut self.segments[branchseg_id];

        // Create branch name for it
        self.branches.insert(name, branchseg_id);
-        Ok(())
    }

-    pub fn calculate(&mut self, retention_period: u64) -> anyhow::Result<SegmentSize> {
+    pub fn calculate(&mut self, retention_period: u64) -> SegmentSize {
        // Phase 1: Mark all the segments that need to be retained
        for (_branch, &last_seg_id) in self.branches.iter() {
            let last_seg = &self.segments[last_seg_id];
@@ -269,7 +255,7 @@ impl<K: std::hash::Hash + Eq + 'static> Storage<K> {
        self.size_from_snapshot_later(0)
    }

-    fn size_from_wal(&self, seg_id: usize) -> anyhow::Result<SegmentSize> {
+    fn size_from_wal(&self, seg_id: usize) -> SegmentSize {
        let seg = &self.segments[seg_id];

        let this_size = seg.end_lsn - seg.start_lsn;
@@ -280,10 +266,10 @@ impl<K: std::hash::Hash + Eq + 'static> Storage<K> {
        for &child_id in seg.children_after.iter() {
            // try each child both ways
            let child = &self.segments[child_id];
-            let p1 = self.size_from_wal(child_id)?;
+            let p1 = self.size_from_wal(child_id);

            let p = if !child.needed {
-                let p2 = self.size_from_snapshot_later(child_id)?;
+                let p2 = self.size_from_snapshot_later(child_id);
                if p1.total() < p2.total() {
                    p1
                } else {
@@ -294,15 +280,15 @@ impl<K: std::hash::Hash + Eq + 'static> Storage<K> {
            };
            children.push(p);
        }
-        Ok(SegmentSize {
+        SegmentSize {
            seg_id,
            method: if seg.needed { WalNeeded } else { Wal },
            this_size,
            children,
-        })
+        }
    }

-    fn size_from_snapshot_later(&self, seg_id: usize) -> anyhow::Result<SegmentSize> {
+    fn size_from_snapshot_later(&self, seg_id: usize) -> SegmentSize {
        // If this is needed, then it's time to do the snapshot and continue
        // with wal method.
        let seg = &self.segments[seg_id];
@@ -313,10 +299,10 @@ impl<K: std::hash::Hash + Eq + 'static> Storage<K> {
            for &child_id in seg.children_after.iter() {
                // try each child both ways
                let child = &self.segments[child_id];
-                let p1 = self.size_from_wal(child_id)?;
+                let p1 = self.size_from_wal(child_id);

                let p = if !child.needed {
-                    let p2 = self.size_from_snapshot_later(child_id)?;
+                    let p2 = self.size_from_snapshot_later(child_id);
                    if p1.total() < p2.total() {
                        p1
                    } else {
@@ -327,12 +313,12 @@ impl<K: std::hash::Hash + Eq + 'static> Storage<K> {
                };
                children.push(p);
            }
-            Ok(SegmentSize {
+            SegmentSize {
                seg_id,
                method: WalNeeded,
                this_size: seg.start_size,
                children,
-            })
+            }
        } else {
            // If any of the direct children are "needed", need to be able to reconstruct here
            let mut children_needed = false;
@@ -347,7 +333,7 @@ impl<K: std::hash::Hash + Eq + 'static> Storage<K> {
            let method1 = if !children_needed {
                let mut children = Vec::new();
                for child in seg.children_after.iter() {
-                    children.push(self.size_from_snapshot_later(*child)?);
+                    children.push(self.size_from_snapshot_later(*child));
                }
                Some(SegmentSize {
                    seg_id,
@@ -363,25 +349,20 @@ impl<K: std::hash::Hash + Eq + 'static> Storage<K> {
            let method2 = if children_needed || seg.children_after.len() >= 2 {
                let mut children = Vec::new();
                for child in seg.children_after.iter() {
-                    children.push(self.size_from_wal(*child)?);
+                    children.push(self.size_from_wal(*child));
                }
-                let Some(this_size) = seg.end_size else { anyhow::bail!("no end_size at junction {seg_id}") };
                Some(SegmentSize {
                    seg_id,
                    method: SnapshotAfter,
-                    this_size,
+                    this_size: seg.end_size.unwrap(),
                    children,
                })
            } else {
                None
            };

-            Ok(match (method1, method2) {
-                (None, None) => anyhow::bail!(
-                    "neither method was applicable: children_after={}, children_needed={}",
-                    seg.children_after.len(),
-                    children_needed
-                ),
+            match (method1, method2) {
+                (None, None) => panic!(),
                (Some(method), None) => method,
                (None, Some(method)) => method,
                (Some(method1), Some(method2)) => {
@@ -391,7 +372,7 @@ impl<K: std::hash::Hash + Eq + 'static> Storage<K> {
                        method2
                    }
                }
-            })
+            }
        }
    }

@@ -7,118 +7,118 @@
 use tenant_size_model::{Segment, SegmentSize, Storage};

 // Main branch only. Some updates on it.
-fn scenario_1() -> anyhow::Result<(Vec<Segment>, SegmentSize)> {
+fn scenario_1() -> (Vec<Segment>, SegmentSize) {
    // Create main branch
    let mut storage = Storage::new("main");

    // Bulk load 5 GB of data to it
-    storage.insert("main", 5_000)?;
+    storage.insert("main", 5_000);

    // Stream of updates
    for _ in 0..5 {
-        storage.update("main", 1_000)?;
+        storage.update("main", 1_000);
    }

-    let size = storage.calculate(1000)?;
+    let size = storage.calculate(1000);

-    Ok((storage.into_segments(), size))
+    (storage.into_segments(), size)
 }

 // Main branch only. Some updates on it.
-fn scenario_2() -> anyhow::Result<(Vec<Segment>, SegmentSize)> {
+fn scenario_2() -> (Vec<Segment>, SegmentSize) {
    // Create main branch
    let mut storage = Storage::new("main");

    // Bulk load 5 GB of data to it
-    storage.insert("main", 5_000)?;
+    storage.insert("main", 5_000);

    // Stream of updates
    for _ in 0..5 {
-        storage.update("main", 1_000)?;
+        storage.update("main", 1_000);
    }

    // Branch
-    storage.branch("main", "child")?;
-    storage.update("child", 1_000)?;
+    storage.branch("main", "child");
+    storage.update("child", 1_000);

    // More updates on parent
-    storage.update("main", 1_000)?;
+    storage.update("main", 1_000);

-    let size = storage.calculate(1000)?;
+    let size = storage.calculate(1000);

-    Ok((storage.into_segments(), size))
+    (storage.into_segments(), size)
 }

 // Like 2, but more updates on main
-fn scenario_3() -> anyhow::Result<(Vec<Segment>, SegmentSize)> {
+fn scenario_3() -> (Vec<Segment>, SegmentSize) {
    // Create main branch
    let mut storage = Storage::new("main");

    // Bulk load 5 GB of data to it
-    storage.insert("main", 5_000)?;
+    storage.insert("main", 5_000);

    // Stream of updates
    for _ in 0..5 {
-        storage.update("main", 1_000)?;
+        storage.update("main", 1_000);
    }

    // Branch
-    storage.branch("main", "child")?;
-    storage.update("child", 1_000)?;
+    storage.branch("main", "child");
+    storage.update("child", 1_000);

    // More updates on parent
    for _ in 0..5 {
-        storage.update("main", 1_000)?;
+        storage.update("main", 1_000);
    }

-    let size = storage.calculate(1000)?;
+    let size = storage.calculate(1000);

-    Ok((storage.into_segments(), size))
+    (storage.into_segments(), size)
 }

 // Diverged branches
-fn scenario_4() -> anyhow::Result<(Vec<Segment>, SegmentSize)> {
+fn scenario_4() -> (Vec<Segment>, SegmentSize) {
    // Create main branch
    let mut storage = Storage::new("main");

    // Bulk load 5 GB of data to it
-    storage.insert("main", 5_000)?;
+    storage.insert("main", 5_000);

    // Stream of updates
    for _ in 0..5 {
-        storage.update("main", 1_000)?;
+        storage.update("main", 1_000);
    }

    // Branch
-    storage.branch("main", "child")?;
-    storage.update("child", 1_000)?;
+    storage.branch("main", "child");
+    storage.update("child", 1_000);

    // More updates on parent
    for _ in 0..8 {
-        storage.update("main", 1_000)?;
+        storage.update("main", 1_000);
    }

-    let size = storage.calculate(1000)?;
+    let size = storage.calculate(1000);

-    Ok((storage.into_segments(), size))
+    (storage.into_segments(), size)
 }

-fn scenario_5() -> anyhow::Result<(Vec<Segment>, SegmentSize)> {
+fn scenario_5() -> (Vec<Segment>, SegmentSize) {
    let mut storage = Storage::new("a");
-    storage.insert("a", 5000)?;
-    storage.branch("a", "b")?;
-    storage.update("b", 4000)?;
-    storage.update("a", 2000)?;
-    storage.branch("a", "c")?;
-    storage.insert("c", 4000)?;
-    storage.insert("a", 2000)?;
+    storage.insert("a", 5000);
+    storage.branch("a", "b");
+    storage.update("b", 4000);
+    storage.update("a", 2000);
+    storage.branch("a", "c");
+    storage.insert("c", 4000);
+    storage.insert("a", 2000);

-    let size = storage.calculate(5000)?;
+    let size = storage.calculate(5000);

-    Ok((storage.into_segments(), size))
+    (storage.into_segments(), size)
 }

-fn scenario_6() -> anyhow::Result<(Vec<Segment>, SegmentSize)> {
+fn scenario_6() -> (Vec<Segment>, SegmentSize) {
    use std::borrow::Cow;

    const NO_OP: Cow<'static, str> = Cow::Borrowed("");
@@ -133,18 +133,18 @@ fn scenario_6() -> anyhow::Result<(Vec<Segment>, SegmentSize)> {

    let mut storage = Storage::new(None);

-    storage.branch(&None, branches[0])?; // at 0
-    storage.modify_branch(&branches[0], NO_OP, 108951064, 43696128)?; // at 108951064
-    storage.branch(&branches[0], branches[1])?; // at 108951064
-    storage.modify_branch(&branches[1], NO_OP, 15560408, -1851392)?; // at 124511472
-    storage.modify_branch(&branches[0], NO_OP, 174464360, -1531904)?; // at 283415424
-    storage.branch(&branches[0], branches[2])?; // at 283415424
-    storage.modify_branch(&branches[2], NO_OP, 15906192, 8192)?; // at 299321616
-    storage.modify_branch(&branches[0], NO_OP, 18909976, 32768)?; // at 302325400
+    storage.branch(&None, branches[0]); // at 0
+    storage.modify_branch(&branches[0], NO_OP, 108951064, 43696128); // at 108951064
+    storage.branch(&branches[0], branches[1]); // at 108951064
+    storage.modify_branch(&branches[1], NO_OP, 15560408, -1851392); // at 124511472
+    storage.modify_branch(&branches[0], NO_OP, 174464360, -1531904); // at 283415424
+    storage.branch(&branches[0], branches[2]); // at 283415424
+    storage.modify_branch(&branches[2], NO_OP, 15906192, 8192); // at 299321616
+    storage.modify_branch(&branches[0], NO_OP, 18909976, 32768); // at 302325400

-    let size = storage.calculate(100_000)?;
+    let size = storage.calculate(100_000);

-    Ok((storage.into_segments(), size))
+    (storage.into_segments(), size)
 }

 fn main() {
@@ -163,8 +163,7 @@ fn main() {
            eprintln!("invalid scenario {}", other);
            std::process::exit(1);
        }
-    }
-    .unwrap();
+    };

    graphviz_tree(&segments, &size);
 }
@@ -252,7 +251,7 @@ fn graphviz_tree(segments: &[Segment], tree: &SegmentSize) {

 #[test]
 fn scenarios_return_same_size() {
-    type ScenarioFn = fn() -> anyhow::Result<(Vec<Segment>, SegmentSize)>;
+    type ScenarioFn = fn() -> (Vec<Segment>, SegmentSize);
    let truths: &[(u32, ScenarioFn, _)] = &[
        (line!(), scenario_1, 8000),
        (line!(), scenario_2, 9000),
@@ -263,7 +262,7 @@ fn scenarios_return_same_size() {
    ];

    for (line, scenario, expected) in truths {
-        let (_, size) = scenario().unwrap();
+        let (_, size) = scenario();
        assert_eq!(*expected, size.total_children(), "scenario on line {line}");
    }
 }
@@ -1,50 +1,48 @@
 [package]
 name = "utils"
 version = "0.1.0"
-edition.workspace = true
-license.workspace = true
+edition = "2021"
+license = "Apache-2.0"

 [dependencies]
-atty.workspace = true
-sentry.workspace = true
-async-trait.workspace = true
-anyhow.workspace = true
-bincode.workspace = true
-bytes.workspace = true
-hyper = { workspace = true, features = ["full"] }
-routerify.workspace = true
-serde.workspace = true
-serde_json.workspace = true
-thiserror.workspace = true
-tokio.workspace = true
-tokio-rustls.workspace = true
-tracing.workspace = true
-tracing-subscriber = { workspace = true, features = ["json"] }
-nix.workspace = true
-signal-hook.workspace = true
-rand.workspace = true
-jsonwebtoken.workspace = true
-hex = { workspace = true, features = ["serde"] }
-rustls.workspace = true
-rustls-split.workspace = true
-git-version.workspace = true
-serde_with.workspace = true
-once_cell.workspace = true
-strum.workspace = true
-strum_macros.workspace = true
+sentry = { version = "0.29.0", default-features = false, features = ["backtrace", "contexts", "panic", "rustls", "reqwest" ] }
+async-trait = "0.1"
+anyhow = "1.0"
+bincode = "1.3"
+bytes = "1.0.1"
+hyper = { version = "0.14.7", features = ["full"] }
+routerify = "3"
+serde = { version = "1.0", features = ["derive"] }
+serde_json = "1"
+thiserror = "1.0"
+tokio = { version = "1.17", features = ["macros"]}
+tokio-rustls = "0.23"
+tracing = "0.1"
+tracing-subscriber = { version = "0.3", features = ["env-filter", "json"] }
+nix = "0.25"
+signal-hook = "0.3.10"
+rand = "0.8.3"
+jsonwebtoken = "8"
+hex = { version = "0.4.3", features = ["serde"] }
+rustls = "0.20.2"
+rustls-split = "0.3.0"
+git-version = "0.3.5"
+serde_with = "2.0"
+once_cell = "1.13.0"
+strum = "0.24"
+strum_macros = "0.24"

-metrics.workspace = true
-pq_proto.workspace = true
-
-workspace_hack.workspace = true
+metrics = { path = "../metrics" }
+pq_proto = { path = "../pq_proto" }
+workspace_hack = { version = "0.1", path = "../../workspace_hack" }

 [dev-dependencies]
-byteorder.workspace = true
-bytes.workspace = true
-hex-literal.workspace = true
-tempfile.workspace = true
-criterion.workspace = true
-rustls-pemfile.workspace = true
+byteorder = "1.4.3"
+bytes = "1.0.1"
+hex-literal = "0.3"
+tempfile = "3.2"
+criterion = "0.4"
+rustls-pemfile = "1"

 [[bench]]
 name = "benchmarks"
@@ -8,7 +8,6 @@ use strum_macros::{EnumString, EnumVariantNames};
 pub enum LogFormat {
    Plain,
    Json,
-    Test,
 }

 impl LogFormat {
@@ -34,13 +33,12 @@ pub fn init(log_format: LogFormat) -> anyhow::Result<()> {
    let base_logger = tracing_subscriber::fmt()
        .with_env_filter(env_filter)
        .with_target(false)
-        .with_ansi(atty::is(atty::Stream::Stdout))
+        .with_ansi(false)
        .with_writer(std::io::stdout);

    match log_format {
        LogFormat::Json => base_logger.json().init(),
        LogFormat::Plain => base_logger.init(),
-        LogFormat::Test => base_logger.with_test_writer().init(),
    }

    Ok(())
@@ -3,9 +3,8 @@
 //! implementation determining how to process the queries. Currently its API
 //! is rather narrow, but we can extend it once required.

-use crate::postgres_backend_async::{log_query_error, short_error, QueryError};
 use crate::sock_split::{BidiStream, ReadStream, WriteStream};
-use anyhow::Context;
+use anyhow::{bail, ensure, Context, Result};
 use bytes::{Bytes, BytesMut};
 use pq_proto::{BeMessage, FeMessage, FeStartupPacket};
 use serde::{Deserialize, Serialize};
@@ -22,32 +21,20 @@ pub trait Handler {
    /// postgres_backend will issue ReadyForQuery after calling this (this
    /// might be not what we want after CopyData streaming, but currently we don't
    /// care).
-    fn process_query(
-        &mut self,
-        pgb: &mut PostgresBackend,
-        query_string: &str,
-    ) -> Result<(), QueryError>;
+    fn process_query(&mut self, pgb: &mut PostgresBackend, query_string: &str) -> Result<()>;

    /// Called on startup packet receival, allows to process params.
    ///
    /// If Ok(false) is returned postgres_backend will skip auth -- that is needed for new users
    /// creation is the proxy code. That is quite hacky and ad-hoc solution, may be we could allow
    /// to override whole init logic in implementations.
-    fn startup(
-        &mut self,
-        _pgb: &mut PostgresBackend,
-        _sm: &FeStartupPacket,
-    ) -> Result<(), QueryError> {
+    fn startup(&mut self, _pgb: &mut PostgresBackend, _sm: &FeStartupPacket) -> Result<()> {
        Ok(())
    }

    /// Check auth jwt
-    fn check_auth_jwt(
-        &mut self,
-        _pgb: &mut PostgresBackend,
-        _jwt_response: &[u8],
-    ) -> Result<(), QueryError> {
-        Err(QueryError::Other(anyhow::anyhow!("JWT auth failed")))
+    fn check_auth_jwt(&mut self, _pgb: &mut PostgresBackend, _jwt_response: &[u8]) -> Result<()> {
+        bail!("JWT auth failed")
    }

    fn is_shutdown_requested(&self) -> bool {
@@ -79,7 +66,7 @@ impl FromStr for AuthType {
        match s {
            "Trust" => Ok(Self::Trust),
            "NeonJWT" => Ok(Self::NeonJWT),
-            _ => anyhow::bail!("invalid value \"{s}\" for auth type"),
+            _ => bail!("invalid value \"{s}\" for auth type"),
        }
    }
 }
@@ -167,7 +154,7 @@ pub fn is_socket_read_timed_out(error: &anyhow::Error) -> bool {
 }

 // Cast a byte slice to a string slice, dropping null terminator if there's one.
-fn cstr_to_str(bytes: &[u8]) -> anyhow::Result<&str> {
+fn cstr_to_str(bytes: &[u8]) -> Result<&str> {
    let without_null = bytes.strip_suffix(&[0]).unwrap_or(bytes);
    std::str::from_utf8(without_null).map_err(|e| e.into())
 }
@@ -201,10 +188,10 @@ impl PostgresBackend {
    }

    /// Get direct reference (into the Option) to the read stream.
-    fn get_stream_in(&mut self) -> anyhow::Result<&mut BidiStream> {
+    fn get_stream_in(&mut self) -> Result<&mut BidiStream> {
        match &mut self.stream {
            Some(Stream::Bidirectional(stream)) => Ok(stream),
-            _ => anyhow::bail!("reader taken"),
+            _ => bail!("reader taken"),
        }
    }

@@ -228,7 +215,7 @@ impl PostgresBackend {
    }

    /// Read full message or return None if connection is closed.
-    pub fn read_message(&mut self) -> Result<Option<FeMessage>, QueryError> {
+    pub fn read_message(&mut self) -> Result<Option<FeMessage>> {
        let (state, stream) = (self.state, self.get_stream_in()?);

        use ProtoState::*;
@@ -236,7 +223,6 @@ impl PostgresBackend {
            Initialization | Encrypted => FeStartupPacket::read(stream),
            Authentication | Established => FeMessage::read(stream),
        }
-        .map_err(QueryError::from)
    }

    /// Write message into internal output buffer.
@@ -260,7 +246,7 @@ impl PostgresBackend {
    }

    // Wrapper for run_message_loop() that shuts down socket when we are done
-    pub fn run(mut self, handler: &mut impl Handler) -> Result<(), QueryError> {
+    pub fn run(mut self, handler: &mut impl Handler) -> Result<()> {
        let ret = self.run_message_loop(handler);
        if let Some(stream) = self.stream.as_mut() {
            let _ = stream.shutdown(Shutdown::Both);
@@ -268,7 +254,7 @@ impl PostgresBackend {
        ret
    }

-    fn run_message_loop(&mut self, handler: &mut impl Handler) -> Result<(), QueryError> {
+    fn run_message_loop(&mut self, handler: &mut impl Handler) -> Result<()> {
        trace!("postgres backend to {:?} started", self.peer_addr);

        let mut unnamed_query_string = Bytes::new();
@@ -277,7 +263,7 @@ impl PostgresBackend {
            match self.read_message() {
                Ok(message) => {
                    if let Some(msg) = message {
-                        trace!("got message {msg:?}");
+                        trace!("got message {:?}", msg);

                        match self.process_message(handler, msg, &mut unnamed_query_string)? {
                            ProcessMsgResult::Continue => continue,
@@ -288,12 +274,10 @@ impl PostgresBackend {
                    }
                }
                Err(e) => {
-                    if let QueryError::Other(e) = &e {
-                        if is_socket_read_timed_out(e) {
-                            continue;
-                        }
+                    // If it is a timeout error, continue the loop
+                    if !is_socket_read_timed_out(&e) {
+                        return Err(e);
                    }
-                    return Err(e);
                }
            }
        }
@@ -311,7 +295,7 @@ impl PostgresBackend {
            }
            stream => {
                self.stream = stream;
-                anyhow::bail!("can't start TLs without bidi stream");
+                bail!("can't start TLs without bidi stream");
            }
        }
    }
@@ -321,16 +305,17 @@ impl PostgresBackend {
        handler: &mut impl Handler,
        msg: FeMessage,
        unnamed_query_string: &mut Bytes,
-    ) -> Result<ProcessMsgResult, QueryError> {
+    ) -> Result<ProcessMsgResult> {
        // Allow only startup and password messages during auth. Otherwise client would be able to bypass auth
        // TODO: change that to proper top-level match of protocol state with separate message handling for each state
-        if self.state < ProtoState::Established
-            && !matches!(
-                msg,
-                FeMessage::PasswordMessage(_) | FeMessage::StartupPacket(_)
-            )
-        {
-            return Err(QueryError::Other(anyhow::anyhow!("protocol violation")));
+        if self.state < ProtoState::Established {
+            ensure!(
+                matches!(
+                    msg,
+                    FeMessage::PasswordMessage(_) | FeMessage::StartupPacket(_)
+                ),
+                "protocol violation"
+            );
        }

        let have_tls = self.tls_config.is_some();
@@ -354,13 +339,8 @@ impl PostgresBackend {
                    }
                    FeStartupPacket::StartupMessage { .. } => {
                        if have_tls && !matches!(self.state, ProtoState::Encrypted) {
-                            self.write_message(&BeMessage::ErrorResponse(
-                                "must connect with TLS",
-                                None,
-                            ))?;
-                            return Err(QueryError::Other(anyhow::anyhow!(
-                                "client did not connect with TLS"
-                            )));
+                            self.write_message(&BeMessage::ErrorResponse("must connect with TLS"))?;
+                            bail!("client did not connect with TLS");
                        }

                        // NB: startup() may change self.auth_type -- we are using that in proxy code
@@ -399,11 +379,8 @@ impl PostgresBackend {
                        let (_, jwt_response) = m.split_last().context("protocol violation")?;

                        if let Err(e) = handler.check_auth_jwt(self, jwt_response) {
-                            self.write_message(&BeMessage::ErrorResponse(
-                                &e.to_string(),
-                                Some(e.pg_error_code()),
-                            ))?;
-                            return Err(e);
+                            self.write_message(&BeMessage::ErrorResponse(&e.to_string()))?;
+                            bail!("auth failed: {}", e);
                        }
                    }
                }
@@ -417,14 +394,33 @@ impl PostgresBackend {
                // remove null terminator
                let query_string = cstr_to_str(&body)?;

-                trace!("got query {query_string:?}");
+                trace!("got query {:?}", query_string);
+                // xxx distinguish fatal and recoverable errors?
                if let Err(e) = handler.process_query(self, query_string) {
-                    log_query_error(query_string, &e);
-                    let short_error = short_error(&e);
-                    self.write_message_noflush(&BeMessage::ErrorResponse(
-                        &short_error,
-                        Some(e.pg_error_code()),
-                    ))?;
+                    // ":?" uses the alternate formatting style, which makes anyhow display the
+                    // full cause of the error, not just the top-level context + its trace.
+                    // We don't want to send that in the ErrorResponse though,
+                    // because it's not relevant to the compute node logs.
+                    //
+                    // We also don't want to log full stacktrace when the error is primitive,
+                    // such as usual connection closed.
+                    let short_error = format!("{:#}", e);
+                    let root_cause = e.root_cause().to_string();
+                    if root_cause.contains("connection closed unexpectedly")
+                        || root_cause.contains("Broken pipe (os error 32)")
+                    {
+                        error!(
+                            "query handler for '{}' failed: {}",
+                            query_string, short_error
+                        );
+                    } else {
+                        error!("query handler for '{}' failed: {:?}", query_string, e);
+                    }
+                    self.write_message_noflush(&BeMessage::ErrorResponse(&short_error))?;
+                    // TODO: untangle convoluted control flow
+                    if e.to_string().contains("failed to run") {
+                        return Ok(ProcessMsgResult::Break);
+                    }
                }
                self.write_message(&BeMessage::ReadyForQuery)?;
            }
@@ -449,13 +445,11 @@ impl PostgresBackend {

            FeMessage::Execute(_) => {
                let query_string = cstr_to_str(unnamed_query_string)?;
-                trace!("got execute {query_string:?}");
+                trace!("got execute {:?}", query_string);
+                // xxx distinguish fatal and recoverable errors?
                if let Err(e) = handler.process_query(self, query_string) {
-                    log_query_error(query_string, &e);
-                    self.write_message(&BeMessage::ErrorResponse(
-                        &e.to_string(),
-                        Some(e.pg_error_code()),
-                    ))?;
+                    error!("query handler for '{}' failed: {:?}", query_string, e);
+                    self.write_message(&BeMessage::ErrorResponse(&e.to_string()))?;
                }
                // NOTE there is no ReadyForQuery message. This handler is used
                // for basebackup and it uses CopyOut which doesn't require
@@ -474,9 +468,7 @@ impl PostgresBackend {
            // We prefer explicit pattern matching to wildcards, because
            // this helps us spot the places where new variants are missing
            FeMessage::CopyData(_) | FeMessage::CopyDone | FeMessage::CopyFail => {
-                return Err(QueryError::Other(anyhow::anyhow!(
-                    "unexpected message type: {msg:?}"
-                )));
+                bail!("unexpected message type: {:?}", msg);
            }
        }

@@ -4,87 +4,39 @@
 //! is rather narrow, but we can extend it once required.

 use crate::postgres_backend::AuthType;
-use anyhow::Context;
+use anyhow::{bail, Context, Result};
 use bytes::{Buf, Bytes, BytesMut};
-use pq_proto::{BeMessage, ConnectionError, FeMessage, FeStartupPacket, SQLSTATE_INTERNAL_ERROR};
-use std::io;
+use pq_proto::{BeMessage, FeMessage, FeStartupPacket};
+use std::future::Future;
 use std::net::SocketAddr;
 use std::pin::Pin;
 use std::sync::Arc;
 use std::task::Poll;
-use std::{future::Future, task::ready};
-use tracing::{debug, error, info, trace};
+use tracing::{debug, error, trace};

 use tokio::io::{AsyncRead, AsyncWrite, AsyncWriteExt, BufReader};
 use tokio_rustls::TlsAcceptor;

-pub fn is_expected_io_error(e: &io::Error) -> bool {
-    use io::ErrorKind::*;
-    matches!(
-        e.kind(),
-        ConnectionRefused | ConnectionAborted | ConnectionReset
-    )
-}
-
-/// An error, occurred during query processing:
-/// either during the connection ([`ConnectionError`]) or before/after it.
-#[derive(thiserror::Error, Debug)]
-pub enum QueryError {
-    /// The connection was lost while processing the query.
-    #[error(transparent)]
-    Disconnected(#[from] ConnectionError),
-    /// Some other error
-    #[error(transparent)]
-    Other(#[from] anyhow::Error),
-}
-
-impl From<io::Error> for QueryError {
-    fn from(e: io::Error) -> Self {
-        Self::Disconnected(ConnectionError::Socket(e))
-    }
-}
-
-impl QueryError {
-    pub fn pg_error_code(&self) -> &'static [u8; 5] {
-        match self {
-            Self::Disconnected(_) => b"08006",         // connection failure
-            Self::Other(_) => SQLSTATE_INTERNAL_ERROR, // internal error
-        }
-    }
-}
-
 #[async_trait::async_trait]
 pub trait Handler {
    /// Handle single query.
    /// postgres_backend will issue ReadyForQuery after calling this (this
    /// might be not what we want after CopyData streaming, but currently we don't
    /// care).
-    async fn process_query(
-        &mut self,
-        pgb: &mut PostgresBackend,
-        query_string: &str,
-    ) -> Result<(), QueryError>;
+    async fn process_query(&mut self, pgb: &mut PostgresBackend, query_string: &str) -> Result<()>;

    /// Called on startup packet receival, allows to process params.
    ///
    /// If Ok(false) is returned postgres_backend will skip auth -- that is needed for new users
    /// creation is the proxy code. That is quite hacky and ad-hoc solution, may be we could allow
    /// to override whole init logic in implementations.
-    fn startup(
-        &mut self,
-        _pgb: &mut PostgresBackend,
-        _sm: &FeStartupPacket,
-    ) -> Result<(), QueryError> {
+    fn startup(&mut self, _pgb: &mut PostgresBackend, _sm: &FeStartupPacket) -> Result<()> {
        Ok(())
    }

    /// Check auth jwt
-    fn check_auth_jwt(
-        &mut self,
-        _pgb: &mut PostgresBackend,
-        _jwt_response: &[u8],
-    ) -> Result<(), QueryError> {
-        Err(QueryError::Other(anyhow::anyhow!("JWT auth failed")))
+    fn check_auth_jwt(&mut self, _pgb: &mut PostgresBackend, _jwt_response: &[u8]) -> Result<()> {
+        bail!("JWT auth failed")
    }
 }

@@ -118,14 +70,17 @@ impl AsyncWrite for Stream {
        self: Pin<&mut Self>,
        cx: &mut std::task::Context<'_>,
        buf: &[u8],
-    ) -> Poll<io::Result<usize>> {
+    ) -> Poll<Result<usize, std::io::Error>> {
        match self.get_mut() {
            Self::Unencrypted(stream) => Pin::new(stream).poll_write(cx, buf),
            Self::Tls(stream) => Pin::new(stream).poll_write(cx, buf),
            Self::Broken => unreachable!(),
        }
    }
-    fn poll_flush(self: Pin<&mut Self>, cx: &mut std::task::Context<'_>) -> Poll<io::Result<()>> {
+    fn poll_flush(
+        self: Pin<&mut Self>,
+        cx: &mut std::task::Context<'_>,
+    ) -> Poll<Result<(), std::io::Error>> {
        match self.get_mut() {
            Self::Unencrypted(stream) => Pin::new(stream).poll_flush(cx),
            Self::Tls(stream) => Pin::new(stream).poll_flush(cx),
@@ -135,7 +90,7 @@ impl AsyncWrite for Stream {
    fn poll_shutdown(
        self: Pin<&mut Self>,
        cx: &mut std::task::Context<'_>,
-    ) -> Poll<io::Result<()>> {
+    ) -> Poll<Result<(), std::io::Error>> {
        match self.get_mut() {
            Self::Unencrypted(stream) => Pin::new(stream).poll_shutdown(cx),
            Self::Tls(stream) => Pin::new(stream).poll_shutdown(cx),
@@ -148,7 +103,7 @@ impl AsyncRead for Stream {
        self: Pin<&mut Self>,
        cx: &mut std::task::Context<'_>,
        buf: &mut tokio::io::ReadBuf<'_>,
-    ) -> Poll<io::Result<()>> {
+    ) -> Poll<Result<(), std::io::Error>> {
        match self.get_mut() {
            Self::Unencrypted(stream) => Pin::new(stream).poll_read(cx, buf),
            Self::Tls(stream) => Pin::new(stream).poll_read(cx, buf),
@@ -184,7 +139,7 @@ pub fn query_from_cstring(query_string: Bytes) -> Vec<u8> {
 }

 // Cast a byte slice to a string slice, dropping null terminator if there's one.
-fn cstr_to_str(bytes: &[u8]) -> anyhow::Result<&str> {
+fn cstr_to_str(bytes: &[u8]) -> Result<&str> {
    let without_null = bytes.strip_suffix(&[0]).unwrap_or(bytes);
    std::str::from_utf8(without_null).map_err(|e| e.into())
 }
@@ -194,7 +149,7 @@ impl PostgresBackend {
        socket: tokio::net::TcpStream,
        auth_type: AuthType,
        tls_config: Option<Arc<rustls::ServerConfig>>,
-    ) -> io::Result<Self> {
+    ) -> std::io::Result<Self> {
        let peer_addr = socket.peer_addr()?;

        Ok(Self {
@@ -212,18 +167,17 @@ impl PostgresBackend {
    }

    /// Read full message or return None if connection is closed.
-    pub async fn read_message(&mut self) -> Result<Option<FeMessage>, QueryError> {
+    pub async fn read_message(&mut self) -> Result<Option<FeMessage>> {
        use ProtoState::*;
        match self.state {
            Initialization | Encrypted => FeStartupPacket::read_fut(&mut self.stream).await,
            Authentication | Established => FeMessage::read_fut(&mut self.stream).await,
            Closed => Ok(None),
        }
-        .map_err(QueryError::from)
    }

    /// Flush output buffer into the socket.
-    pub async fn flush(&mut self) -> io::Result<()> {
+    pub async fn flush(&mut self) -> std::io::Result<()> {
        while self.buf_out.has_remaining() {
            let bytes_written = self.stream.write(self.buf_out.chunk()).await?;
            self.buf_out.advance(bytes_written);
@@ -233,7 +187,7 @@ impl PostgresBackend {
    }

    /// Write message into internal output buffer.
-    pub fn write_message(&mut self, message: &BeMessage<'_>) -> io::Result<&mut Self> {
+    pub fn write_message(&mut self, message: &BeMessage<'_>) -> Result<&mut Self, std::io::Error> {
        BeMessage::write(&mut self.buf_out, message)?;
        Ok(self)
    }
@@ -253,9 +207,12 @@ impl PostgresBackend {
        cx: &mut std::task::Context<'_>,
    ) -> Poll<Result<(), std::io::Error>> {
        while self.buf_out.has_remaining() {
-            match ready!(Pin::new(&mut self.stream).poll_write(cx, self.buf_out.chunk())) {
-                Ok(bytes_written) => self.buf_out.advance(bytes_written),
-                Err(err) => return Poll::Ready(Err(err)),
+            match Pin::new(&mut self.stream).poll_write(cx, self.buf_out.chunk()) {
+                Poll::Ready(Ok(bytes_written)) => {
+                    self.buf_out.advance(bytes_written);
+                }
+                Poll::Ready(Err(err)) => return Poll::Ready(Err(err)),
+                Poll::Pending => return Poll::Pending,
            }
        }
        Poll::Ready(Ok(()))
@@ -266,11 +223,7 @@ impl PostgresBackend {
    }

    // Wrapper for run_message_loop() that shuts down socket when we are done
-    pub async fn run<F, S>(
-        mut self,
-        handler: &mut impl Handler,
-        shutdown_watcher: F,
-    ) -> Result<(), QueryError>
+    pub async fn run<F, S>(mut self, handler: &mut impl Handler, shutdown_watcher: F) -> Result<()>
    where
        F: Fn() -> S,
        S: Future,
@@ -284,7 +237,7 @@ impl PostgresBackend {
        &mut self,
        handler: &mut impl Handler,
        shutdown_watcher: F,
-    ) -> Result<(), QueryError>
+    ) -> Result<()>
    where
        F: Fn() -> S,
        S: Future,
@@ -320,7 +273,7 @@ impl PostgresBackend {
                        return Ok(());
                    }
                }
-                Ok::<(), QueryError>(())
+                Ok::<(), anyhow::Error>(())
            } => {
                // Handshake complete.
                result?;
@@ -365,14 +318,14 @@ impl PostgresBackend {
            self.stream = Stream::Tls(Box::new(tls_stream));
            return Ok(());
        };
-        anyhow::bail!("TLS already started");
+        bail!("TLS already started");
    }

    async fn process_handshake_message(
        &mut self,
        handler: &mut impl Handler,
        msg: FeMessage,
-    ) -> Result<ProcessMsgResult, QueryError> {
+    ) -> Result<ProcessMsgResult> {
        assert!(self.state < ProtoState::Established);
        let have_tls = self.tls_config.is_some();
        match msg {
@@ -395,13 +348,8 @@ impl PostgresBackend {
                    }
                    FeStartupPacket::StartupMessage { .. } => {
                        if have_tls && !matches!(self.state, ProtoState::Encrypted) {
-                            self.write_message(&BeMessage::ErrorResponse(
-                                "must connect with TLS",
-                                None,
-                            ))?;
-                            return Err(QueryError::Other(anyhow::anyhow!(
-                                "client did not connect with TLS"
-                            )));
+                            self.write_message(&BeMessage::ErrorResponse("must connect with TLS"))?;
+                            bail!("client did not connect with TLS");
                        }

                        // NB: startup() may change self.auth_type -- we are using that in proxy code
@@ -441,11 +389,8 @@ impl PostgresBackend {
                        let (_, jwt_response) = m.split_last().context("protocol violation")?;

                        if let Err(e) = handler.check_auth_jwt(self, jwt_response) {
-                            self.write_message(&BeMessage::ErrorResponse(
-                                &e.to_string(),
-                                Some(e.pg_error_code()),
-                            ))?;
-                            return Err(e);
+                            self.write_message(&BeMessage::ErrorResponse(&e.to_string()))?;
+                            bail!("auth failed: {}", e);
                        }
                    }
                }
@@ -468,28 +413,33 @@ impl PostgresBackend {
        handler: &mut impl Handler,
        msg: FeMessage,
        unnamed_query_string: &mut Bytes,
-    ) -> Result<ProcessMsgResult, QueryError> {
+    ) -> Result<ProcessMsgResult> {
        // Allow only startup and password messages during auth. Otherwise client would be able to bypass auth
        // TODO: change that to proper top-level match of protocol state with separate message handling for each state
        assert!(self.state == ProtoState::Established);

        match msg {
            FeMessage::StartupPacket(_) | FeMessage::PasswordMessage(_) => {
-                return Err(QueryError::Other(anyhow::anyhow!("protocol violation")));
+                bail!("protocol violation");
            }

            FeMessage::Query(body) => {
                // remove null terminator
                let query_string = cstr_to_str(&body)?;

-                trace!("got query {query_string:?}");
+                trace!("got query {:?}", query_string);
+                // xxx distinguish fatal and recoverable errors?
                if let Err(e) = handler.process_query(self, query_string).await {
-                    log_query_error(query_string, &e);
-                    let short_error = short_error(&e);
-                    self.write_message(&BeMessage::ErrorResponse(
-                        &short_error,
-                        Some(e.pg_error_code()),
-                    ))?;
+                    // ":?" uses the alternate formatting style, which makes anyhow display the
+                    // full cause of the error, not just the top-level context + its trace.
+                    // We don't want to send that in the ErrorResponse though,
+                    // because it's not relevant to the compute node logs.
+                    error!("query handler for '{}' failed: {:?}", query_string, e);
+                    self.write_message(&BeMessage::ErrorResponse(&e.to_string()))?;
+                    // TODO: untangle convoluted control flow
+                    if e.to_string().contains("failed to run") {
+                        return Ok(ProcessMsgResult::Break);
+                    }
                }
                self.write_message(&BeMessage::ReadyForQuery)?;
            }
@@ -514,13 +464,11 @@ impl PostgresBackend {

            FeMessage::Execute(_) => {
                let query_string = cstr_to_str(unnamed_query_string)?;
-                trace!("got execute {query_string:?}");
+                trace!("got execute {:?}", query_string);
+                // xxx distinguish fatal and recoverable errors?
                if let Err(e) = handler.process_query(self, query_string).await {
-                    log_query_error(query_string, &e);
-                    self.write_message(&BeMessage::ErrorResponse(
-                        &e.to_string(),
-                        Some(e.pg_error_code()),
-                    ))?;
+                    error!("query handler for '{}' failed: {:?}", query_string, e);
+                    self.write_message(&BeMessage::ErrorResponse(&e.to_string()))?;
                }
                // NOTE there is no ReadyForQuery message. This handler is used
                // for basebackup and it uses CopyOut which doesn't require
@@ -539,10 +487,7 @@ impl PostgresBackend {
            // We prefer explicit pattern matching to wildcards, because
            // this helps us spot the places where new variants are missing
            FeMessage::CopyData(_) | FeMessage::CopyDone | FeMessage::CopyFail => {
-                return Err(QueryError::Other(anyhow::anyhow!(
-                    "unexpected message type: {:?}",
-                    msg
-                )));
+                bail!("unexpected message type: {:?}", msg);
            }
        }

@@ -570,9 +515,10 @@ impl<'a> AsyncWrite for CopyDataWriter<'a> {
        // It's not strictly required to flush between each message, but makes it easier
        // to view in wireshark, and usually the messages that the callers write are
        // decently-sized anyway.
-        match ready!(this.pgb.poll_write_buf(cx)) {
-            Ok(()) => {}
-            Err(err) => return Poll::Ready(Err(err)),
+        match this.pgb.poll_write_buf(cx) {
+            Poll::Ready(Ok(())) => {}
+            Poll::Ready(Err(err)) => return Poll::Ready(Err(err)),
+            Poll::Pending => return Poll::Pending,
        }

        // CopyData
@@ -589,9 +535,10 @@ impl<'a> AsyncWrite for CopyDataWriter<'a> {
        cx: &mut std::task::Context<'_>,
    ) -> Poll<Result<(), std::io::Error>> {
        let this = self.get_mut();
-        match ready!(this.pgb.poll_write_buf(cx)) {
-            Ok(()) => {}
-            Err(err) => return Poll::Ready(Err(err)),
+        match this.pgb.poll_write_buf(cx) {
+            Poll::Ready(Ok(())) => {}
+            Poll::Ready(Err(err)) => return Poll::Ready(Err(err)),
+            Poll::Pending => return Poll::Pending,
        }
        this.pgb.poll_flush(cx)
    }
@@ -600,35 +547,11 @@ impl<'a> AsyncWrite for CopyDataWriter<'a> {
        cx: &mut std::task::Context<'_>,
    ) -> Poll<Result<(), std::io::Error>> {
        let this = self.get_mut();
-        match ready!(this.pgb.poll_write_buf(cx)) {
-            Ok(()) => {}
-            Err(err) => return Poll::Ready(Err(err)),
+        match this.pgb.poll_write_buf(cx) {
+            Poll::Ready(Ok(())) => {}
+            Poll::Ready(Err(err)) => return Poll::Ready(Err(err)),
+            Poll::Pending => return Poll::Pending,
        }
        this.pgb.poll_flush(cx)
    }
 }
-
-pub fn short_error(e: &QueryError) -> String {
-    match e {
-        QueryError::Disconnected(connection_error) => connection_error.to_string(),
-        QueryError::Other(e) => format!("{e:#}"),
-    }
-}
-
-pub(super) fn log_query_error(query: &str, e: &QueryError) {
-    match e {
-        QueryError::Disconnected(ConnectionError::Socket(io_error)) => {
-            if is_expected_io_error(io_error) {
-                info!("query handler for '{query}' failed with expected io error: {io_error}");
-            } else {
-                error!("query handler for '{query}' failed with io error: {io_error}");
-            }
-        }
-        QueryError::Disconnected(other_connection_error) => {
-            error!("query handler for '{query}' failed with connection error: {other_connection_error:?}")
-        }
-        QueryError::Other(e) => {
-            error!("query handler for '{query}' failed: {e:?}");
-        }
-    }
-}
@@ -9,10 +9,7 @@ use byteorder::{BigEndian, ReadBytesExt, WriteBytesExt};
 use bytes::{Buf, BufMut, Bytes, BytesMut};
 use once_cell::sync::Lazy;

-use utils::{
-    postgres_backend::{AuthType, Handler, PostgresBackend},
-    postgres_backend_async::QueryError,
-};
+use utils::postgres_backend::{AuthType, Handler, PostgresBackend};

 fn make_tcp_pair() -> (TcpStream, TcpStream) {
    let listener = TcpListener::bind("127.0.0.1:0").unwrap();
@@ -108,7 +105,7 @@ fn ssl() {
            &mut self,
            _pgb: &mut PostgresBackend,
            query_string: &str,
-        ) -> Result<(), QueryError> {
+        ) -> anyhow::Result<()> {
            self.got_query = query_string == QUERY;
            Ok(())
        }
@@ -155,7 +152,7 @@ fn no_ssl() {
            &mut self,
            _pgb: &mut PostgresBackend,
            _query_string: &str,
-        ) -> Result<(), QueryError> {
+        ) -> anyhow::Result<()> {
            panic!()
        }
    }
@@ -215,7 +212,7 @@ fn server_forces_ssl() {
            &mut self,
            _pgb: &mut PostgresBackend,
            _query_string: &str,
-        ) -> Result<(), QueryError> {
+        ) -> anyhow::Result<()> {
            panic!()
        }
    }
@@ -1,8 +1,8 @@
 [package]
 name = "pageserver"
 version = "0.1.0"
-edition.workspace = true
-license.workspace = true
+edition = "2021"
+license = "Apache-2.0"

 [features]
 default = []
@@ -11,68 +11,68 @@ default = []
 testing = ["fail/failpoints"]

 [dependencies]
-anyhow.workspace = true
-async-stream.workspace = true
-async-trait.workspace = true
-byteorder.workspace = true
-bytes.workspace = true
-chrono = { workspace = true, features = ["serde"] }
-clap = { workspace = true, features = ["string"] }
-close_fds.workspace = true
-const_format.workspace = true
-consumption_metrics.workspace = true
-crc32c.workspace = true
-crossbeam-utils.workspace = true
-fail.workspace = true
-futures.workspace = true
-git-version.workspace = true
-hex.workspace = true
-humantime.workspace = true
-humantime-serde.workspace = true
-hyper.workspace = true
-itertools.workspace = true
-nix.workspace = true
-num-traits.workspace = true
-once_cell.workspace = true
-pin-project-lite.workspace = true
-postgres.workspace = true
-postgres-protocol.workspace = true
-postgres-types.workspace = true
-rand.workspace = true
-regex.workspace = true
-scopeguard.workspace = true
-serde.workspace = true
-serde_json = { workspace = true, features = ["raw_value"] }
-serde_with.workspace = true
-signal-hook.workspace = true
-svg_fmt.workspace = true
-tokio-tar.workspace = true
-thiserror.workspace = true
-tokio = { workspace = true, features = ["process", "sync", "fs", "rt", "io-util", "time"] }
-tokio-postgres.workspace = true
-tokio-util.workspace = true
-toml_edit.workspace = true
-tracing.workspace = true
-url.workspace = true
-walkdir.workspace = true
-metrics.workspace = true
-pageserver_api.workspace = true
-postgres_connection.workspace = true
-postgres_ffi.workspace = true
-pq_proto.workspace = true
-remote_storage.workspace = true
-storage_broker.workspace = true
-tenant_size_model.workspace = true
-utils.workspace = true
-workspace_hack.workspace = true
-reqwest.workspace = true
-rpds.workspace = true
-im = "15.1.0"
+amplify_num = { git = "https://github.com/hlinnaka/rust-amplify.git", branch = "unsigned-int-perf" }
+anyhow = { version = "1.0", features = ["backtrace"] }
+async-stream = "0.3"
+async-trait = "0.1"
+byteorder = "1.4.3"
+bytes = "1.0.1"
+chrono = { version = "0.4.23", default-features = false, features = ["clock", "serde"] }
+clap = { version = "4.0", features = ["string"] }
+close_fds = "0.3.2"
+const_format = "0.2.21"
+crc32c = "0.6.0"
+crossbeam-utils = "0.8.5"
+fail = "0.5.0"
+futures = "0.3.13"
+git-version = "0.3.5"
+hex = "0.4.3"
+humantime = "2.1.0"
+humantime-serde = "1.1.1"
+hyper = "0.14"
+itertools = "0.10.3"
+nix = "0.25"
+num-traits = "0.2.15"
+once_cell = "1.13.0"
+pin-project-lite = "0.2.7"
+postgres = { git = "https://github.com/neondatabase/rust-postgres.git", rev="43e6db254a97fdecbce33d8bc0890accfd74495e" }
+postgres-protocol = { git = "https://github.com/neondatabase/rust-postgres.git", rev="43e6db254a97fdecbce33d8bc0890accfd74495e" }
+postgres-types = { git = "https://github.com/neondatabase/rust-postgres.git", rev="43e6db254a97fdecbce33d8bc0890accfd74495e" }
+rand = "0.8.3"
+regex = "1.4.5"
+rstar = "0.9.3"
+scopeguard = "1.1.0"
+serde = { version = "1.0", features = ["derive"] }
+serde_json = { version = "1.0", features = ["raw_value"] }
+serde_with = "2.0"
+signal-hook = "0.3.10"
+svg_fmt = "0.4.1"
+tokio-tar = { git = "https://github.com/neondatabase/tokio-tar.git", rev="404df61437de0feef49ba2ccdbdd94eb8ad6e142" }
+thiserror = "1.0"
+tokio = { version = "1.17", features = ["process", "sync", "macros", "fs", "rt", "io-util", "time"] }
+tokio-postgres = { git = "https://github.com/neondatabase/rust-postgres.git", rev="43e6db254a97fdecbce33d8bc0890accfd74495e" }
+tokio-util = { version = "0.7.3", features = ["io", "io-util"] }
+toml_edit = { version = "0.14", features = ["easy"] }
+tracing = "0.1.36"
+url = "2"
+walkdir = "2.3.2"
+
+metrics = { path = "../libs/metrics" }
+pageserver_api = { path = "../libs/pageserver_api" }
+postgres_connection = { path = "../libs/postgres_connection" }
+postgres_ffi = { path = "../libs/postgres_ffi" }
+pq_proto = { path = "../libs/pq_proto" }
+remote_storage = { path = "../libs/remote_storage" }
+storage_broker = { version = "0.1", path = "../storage_broker" }
+tenant_size_model = { path = "../libs/tenant_size_model" }
+utils = { path = "../libs/utils" }
+workspace_hack = { version = "0.1", path = "../workspace_hack" }
+reqwest = { version = "0.11", default-features = false, features = ["rustls-tls"] }

 [dev-dependencies]
-criterion.workspace = true
-hex-literal.workspace = true
-tempfile.workspace = true
+criterion = "0.4"
+hex-literal = "0.3"
+tempfile = "3.2"

 [[bench]]
 name = "bench_layer_map"
@@ -1,12 +1,13 @@
-use pageserver::keyspace::{KeyPartitioning, KeySpace};
+use anyhow::Result;
 use pageserver::repository::Key;
 use pageserver::tenant::layer_map::LayerMap;
-use pageserver::tenant::storage_layer::Layer;
-use pageserver::tenant::storage_layer::{DeltaFileName, ImageFileName, LayerDescriptor};
+use pageserver::tenant::storage_layer::{DeltaFileName, ImageFileName, ValueReconstructState};
+use pageserver::tenant::storage_layer::{Layer, ValueReconstructResult};
 use rand::prelude::{SeedableRng, SliceRandom, StdRng};
 use std::cmp::{max, min};
 use std::fs::File;
 use std::io::{BufRead, BufReader};
+use std::ops::Range;
 use std::path::PathBuf;
 use std::str::FromStr;
 use std::sync::Arc;
@@ -16,35 +17,102 @@ use utils::lsn::Lsn;

 use criterion::{criterion_group, criterion_main, Criterion};

-fn build_layer_map(filename_dump: PathBuf) -> LayerMap<LayerDescriptor> {
-    let mut layer_map = LayerMap::<LayerDescriptor>::default();
+struct DummyDelta {
+    key_range: Range<Key>,
+    lsn_range: Range<Lsn>,
+}
+
+impl Layer for DummyDelta {
+    fn get_key_range(&self) -> Range<Key> {
+        self.key_range.clone()
+    }
+
+    fn get_lsn_range(&self) -> Range<Lsn> {
+        self.lsn_range.clone()
+    }
+    fn get_value_reconstruct_data(
+        &self,
+        _key: Key,
+        _lsn_range: Range<Lsn>,
+        _reconstruct_data: &mut ValueReconstructState,
+    ) -> Result<ValueReconstructResult> {
+        panic!()
+    }
+
+    fn is_incremental(&self) -> bool {
+        true
+    }
+
+    fn dump(&self, _verbose: bool) -> Result<()> {
+        unimplemented!()
+    }
+
+    fn short_id(&self) -> String {
+        unimplemented!()
+    }
+}
+
+struct DummyImage {
+    key_range: Range<Key>,
+    lsn: Lsn,
+}
+
+impl Layer for DummyImage {
+    fn get_key_range(&self) -> Range<Key> {
+        self.key_range.clone()
+    }
+
+    fn get_lsn_range(&self) -> Range<Lsn> {
+        // End-bound is exclusive
+        self.lsn..(self.lsn + 1)
+    }
+
+    fn get_value_reconstruct_data(
+        &self,
+        _key: Key,
+        _lsn_range: Range<Lsn>,
+        _reconstruct_data: &mut ValueReconstructState,
+    ) -> Result<ValueReconstructResult> {
+        panic!()
+    }
+
+    fn is_incremental(&self) -> bool {
+        false
+    }
+
+    fn dump(&self, _verbose: bool) -> Result<()> {
+        unimplemented!()
+    }
+
+    fn short_id(&self) -> String {
+        unimplemented!()
+    }
+}
+
+fn build_layer_map(filename_dump: PathBuf) -> LayerMap<dyn Layer> {
+    let mut layer_map = LayerMap::<dyn Layer>::default();

    let mut min_lsn = Lsn(u64::MAX);
    let mut max_lsn = Lsn(0);

    let filenames = BufReader::new(File::open(filename_dump).unwrap()).lines();

-    let mut updates = layer_map.batch_update();
    for fname in filenames {
        let fname = &fname.unwrap();
        if let Some(imgfilename) = ImageFileName::parse_str(fname) {
-            let layer = LayerDescriptor {
-                key: imgfilename.key_range,
-                lsn: imgfilename.lsn..(imgfilename.lsn + 1),
-                is_incremental: false,
-                short_id: fname.to_string(),
+            let layer = DummyImage {
+                key_range: imgfilename.key_range,
+                lsn: imgfilename.lsn,
            };
-            updates.insert_historic(Arc::new(layer));
+            layer_map.insert_historic(Arc::new(layer));
            min_lsn = min(min_lsn, imgfilename.lsn);
            max_lsn = max(max_lsn, imgfilename.lsn);
        } else if let Some(deltafilename) = DeltaFileName::parse_str(fname) {
-            let layer = LayerDescriptor {
-                key: deltafilename.key_range.clone(),
-                lsn: deltafilename.lsn_range.clone(),
-                is_incremental: true,
-                short_id: fname.to_string(),
+            let layer = DummyDelta {
+                key_range: deltafilename.key_range,
+                lsn_range: deltafilename.lsn_range.clone(),
            };
-            updates.insert_historic(Arc::new(layer));
+            layer_map.insert_historic(Arc::new(layer));
            min_lsn = min(min_lsn, deltafilename.lsn_range.start);
            max_lsn = max(max_lsn, deltafilename.lsn_range.end);
        } else {
@@ -54,12 +122,11 @@ fn build_layer_map(filename_dump: PathBuf) -> LayerMap<LayerDescriptor> {

    println!("min: {min_lsn}, max: {max_lsn}");

-    updates.flush();
    layer_map
 }

 /// Construct a layer map query pattern for benchmarks
-fn uniform_query_pattern(layer_map: &LayerMap<LayerDescriptor>) -> Vec<(Key, Lsn)> {
+fn uniform_query_pattern(layer_map: &LayerMap<dyn Layer>) -> Vec<(Key, Lsn)> {
    // For each image layer we query one of the pages contained, at LSN right
    // before the image layer was created. This gives us a somewhat uniform
    // coverage of both the lsn and key space because image layers have
@@ -83,41 +150,6 @@ fn uniform_query_pattern(layer_map: &LayerMap<LayerDescriptor>) -> Vec<(Key, Lsn
        .collect()
 }

-// Construct a partitioning for testing get_difficulty map when we
-// don't have an exact result of `collect_keyspace` to work with.
-fn uniform_key_partitioning(layer_map: &LayerMap<LayerDescriptor>, _lsn: Lsn) -> KeyPartitioning {
-    let mut parts = Vec::new();
-
-    // We add a partition boundary at the start of each image layer,
-    // no matter what lsn range it covers. This is just the easiest
-    // thing to do. A better thing to do would be to get a real
-    // partitioning from some database. Even better, remove the need
-    // for key partitions by deciding where to create image layers
-    // directly based on a coverage-based difficulty map.
-    let mut keys: Vec<_> = layer_map
-        .iter_historic_layers()
-        .filter_map(|l| {
-            if l.is_incremental() {
-                None
-            } else {
-                let kr = l.get_key_range();
-                Some(kr.start.next())
-            }
-        })
-        .collect();
-    keys.sort();
-
-    let mut current_key = Key::from_hex("000000000000000000000000000000000000").unwrap();
-    for key in keys {
-        parts.push(KeySpace {
-            ranges: vec![current_key..key],
-        });
-        current_key = key;
-    }
-
-    KeyPartitioning { parts }
-}
-
 // Benchmark using metadata extracted from our performance test environment, from
 // a project where we have run pgbench many timmes. The pgbench database was initialized
 // between each test run.
@@ -151,68 +183,24 @@ fn bench_from_captest_env(c: &mut Criterion) {
 // Benchmark using metadata extracted from a real project that was taknig
 // too long processing layer map queries.
 fn bench_from_real_project(c: &mut Criterion) {
-    // Init layer map
-    let now = Instant::now();
+    // TODO consider compressing this file
    let layer_map = build_layer_map(PathBuf::from("benches/odd-brook-layernames.txt"));
-    println!("Finished layer map init in {:?}", now.elapsed());
-
-    // Choose uniformly distributed queries
    let queries: Vec<(Key, Lsn)> = uniform_query_pattern(&layer_map);

-    // Choose inputs for get_difficulty_map
-    let latest_lsn = layer_map
-        .iter_historic_layers()
-        .map(|l| l.get_lsn_range().end)
-        .max()
-        .unwrap();
-    let partitioning = uniform_key_partitioning(&layer_map, latest_lsn);
-
-    // Check correctness of get_difficulty_map
-    // TODO put this in a dedicated test outside of this mod
-    {
-        println!("running correctness check");
-
-        let now = Instant::now();
-        let result_bruteforce = layer_map.get_difficulty_map_bruteforce(latest_lsn, &partitioning);
-        assert!(result_bruteforce.len() == partitioning.parts.len());
-        println!("Finished bruteforce in {:?}", now.elapsed());
-
-        let now = Instant::now();
-        let result_fast = layer_map.get_difficulty_map(latest_lsn, &partitioning, None);
-        assert!(result_fast.len() == partitioning.parts.len());
-        println!("Finished fast in {:?}", now.elapsed());
-
-        // Assert results are equal. Manually iterate for easier debugging.
-        let zip = std::iter::zip(
-            &partitioning.parts,
-            std::iter::zip(result_bruteforce, result_fast),
-        );
-        for (_part, (bruteforce, fast)) in zip {
-            assert_eq!(bruteforce, fast);
-        }
-
-        println!("No issues found");
-    }
-
-    // Define and name the benchmark function
-    let mut group = c.benchmark_group("real_map");
-    group.bench_function("uniform_queries", |b| {
+    // Test with uniform query pattern
+    c.bench_function("real_map_uniform_queries", |b| {
        b.iter(|| {
            for q in queries.clone().into_iter() {
                layer_map.search(q.0, q.1);
            }
        });
    });
-    group.bench_function("get_difficulty_map", |b| {
-        b.iter(|| {
-            layer_map.get_difficulty_map(latest_lsn, &partitioning, Some(3));
-        });
-    });
-    group.finish();
 }

 // Benchmark using synthetic data. Arrange image layers on stacked diagonal lines.
 fn bench_sequential(c: &mut Criterion) {
+    let mut layer_map: LayerMap<dyn Layer> = LayerMap::default();
+
    // Init layer map. Create 100_000 layers arranged in 1000 diagonal lines.
    //
    // TODO This code is pretty slow and runs even if we're only running other
@@ -220,39 +208,39 @@ fn bench_sequential(c: &mut Criterion) {
    //      Putting it inside the `bench_function` closure is not a solution
    //      because then it runs multiple times during warmup.
    let now = Instant::now();
-    let mut layer_map = LayerMap::default();
-    let mut updates = layer_map.batch_update();
    for i in 0..100_000 {
+        // TODO try inserting a super-wide layer in between every 10 to reflect
+        //      what often happens with L1 layers that include non-rel changes.
+        //      Maybe do that as a separate test.
        let i32 = (i as u32) % 100;
        let zero = Key::from_hex("000000000000000000000000000000000000").unwrap();
-        let layer = LayerDescriptor {
-            key: zero.add(10 * i32)..zero.add(10 * i32 + 1),
-            lsn: Lsn(i)..Lsn(i + 1),
-            is_incremental: false,
-            short_id: format!("Layer {}", i),
+        let layer = DummyImage {
+            key_range: zero.add(10 * i32)..zero.add(10 * i32 + 1),
+            lsn: Lsn(10 * i),
        };
-        updates.insert_historic(Arc::new(layer));
+        layer_map.insert_historic(Arc::new(layer));
    }
-    updates.flush();
-    println!("Finished layer map init in {:?}", now.elapsed());
+
+    // Manually measure runtime without criterion because criterion
+    // has a minimum sample size of 10 and I don't want to run it 10 times.
+    println!("Finished init in {:?}", now.elapsed());

    // Choose 100 uniformly random queries
    let rng = &mut StdRng::seed_from_u64(1);
    let queries: Vec<(Key, Lsn)> = uniform_query_pattern(&layer_map)
-        .choose_multiple(rng, 100)
+        .choose_multiple(rng, 1)
        .copied()
        .collect();

    // Define and name the benchmark function
-    let mut group = c.benchmark_group("sequential");
-    group.bench_function("uniform_queries", |b| {
+    c.bench_function("sequential_uniform_queries", |b| {
+        // Run the search queries
        b.iter(|| {
            for q in queries.clone().into_iter() {
                layer_map.search(q.0, q.1);
            }
        });
    });
-    group.finish();
 }

 criterion_group!(group_1, bench_from_captest_env);
@@ -30,44 +30,33 @@ fn redo_scenarios(c: &mut Criterion) {
    let conf = PageServerConf::dummy_conf(repo_dir.path().to_path_buf());
    let conf = Box::leak(Box::new(conf));
    let tenant_id = TenantId::generate();
-
-    let manager = PostgresRedoManager::new(conf, tenant_id);
+    // std::fs::create_dir_all(conf.tenant_path(&tenant_id)).unwrap();
+    let mut manager = PostgresRedoManager::new(conf, tenant_id);
+    manager.launch_process(14).unwrap();

    let manager = Arc::new(manager);

-    tracing::info!("executing first");
-    short().execute(&manager).unwrap();
-    tracing::info!("first executed");
-
    let thread_counts = [1, 2, 4, 8, 16];

-    let mut group = c.benchmark_group("short");
-    group.sampling_mode(criterion::SamplingMode::Flat);
-
    for thread_count in thread_counts {
-        group.bench_with_input(
-            BenchmarkId::new("short", thread_count),
+        c.bench_with_input(
+            BenchmarkId::new("short-50record", thread_count),
            &thread_count,
            |b, thread_count| {
-                add_multithreaded_walredo_requesters(b, *thread_count, &manager, short);
+                add_multithreaded_walredo_requesters(b, *thread_count, &manager, short, 50);
            },
        );
    }
-    drop(group);
-
-    let mut group = c.benchmark_group("medium");
-    group.sampling_mode(criterion::SamplingMode::Flat);

    for thread_count in thread_counts {
-        group.bench_with_input(
-            BenchmarkId::new("medium", thread_count),
+        c.bench_with_input(
+            BenchmarkId::new("medium-10record", thread_count),
            &thread_count,
            |b, thread_count| {
-                add_multithreaded_walredo_requesters(b, *thread_count, &manager, medium);
+                add_multithreaded_walredo_requesters(b, *thread_count, &manager, medium, 10);
            },
        );
    }
-    drop(group);
 }

 /// Sets up `threads` number of requesters to `request_redo`, with the given input.
@@ -76,66 +65,46 @@ fn add_multithreaded_walredo_requesters(
    threads: u32,
    manager: &Arc<PostgresRedoManager>,
    input_factory: fn() -> Request,
+    request_repeats: usize,
 ) {
-    assert_ne!(threads, 0);
+    b.iter_batched_ref(
+        || {
+            // barrier for all of the threads, and the benchmarked thread
+            let barrier = Arc::new(Barrier::new(threads as usize + 1));

-    if threads == 1 {
-        b.iter_batched_ref(
-            || Some(input_factory()),
-            |input| execute_all(input.take(), manager),
-            criterion::BatchSize::PerIteration,
-        );
-    } else {
-        let (work_tx, work_rx) = std::sync::mpsc::sync_channel(threads as usize);
+            let jhs = (0..threads)
+                .map(|_| {
+                    std::thread::spawn({
+                        let manager = manager.clone();
+                        let barrier = barrier.clone();
+                        move || {
+                            let input = std::iter::repeat(input_factory())
+                                .take(request_repeats)
+                                .collect::<Vec<_>>();

-        let work_rx = std::sync::Arc::new(std::sync::Mutex::new(work_rx));
+                            barrier.wait();

-        let barrier = Arc::new(Barrier::new(threads as usize + 1));
+                            execute_all(input, &manager).unwrap();

-        let jhs = (0..threads)
-            .map(|_| {
-                std::thread::spawn({
-                    let manager = manager.clone();
-                    let barrier = barrier.clone();
-                    let work_rx = work_rx.clone();
-                    move || loop {
-                        // queue up and wait if we want to go another round
-                        if work_rx.lock().unwrap().recv().is_err() {
-                            break;
+                            barrier.wait();
                        }
-
-                        let input = Some(input_factory());
-
-                        barrier.wait();
-
-                        execute_all(input, &manager).unwrap();
-
-                        barrier.wait();
-                    }
+                    })
                })
-            })
-            .collect::<Vec<_>>();
+                .collect::<Vec<_>>();

-        let _jhs = JoinOnDrop(jhs);
+            (barrier, JoinOnDrop(jhs))
+        },
+        |input| {
+            let barrier = &input.0;

-        b.iter_batched(
-            || {
-                for _ in 0..threads {
-                    work_tx.send(()).unwrap()
-                }
-            },
-            |()| {
-                // start the work
-                barrier.wait();
+            // start the work
+            barrier.wait();

-                // wait for work to complete
-                barrier.wait();
-            },
-            criterion::BatchSize::PerIteration,
-        );
-
-        drop(work_tx);
-    }
+            // wait for work to complete
+            barrier.wait();
+        },
+        criterion::BatchSize::PerIteration,
+    );
 }

 struct JoinOnDrop(Vec<std::thread::JoinHandle<()>>);
@@ -152,10 +121,7 @@ impl Drop for JoinOnDrop {
    }
 }

-fn execute_all<I>(input: I, manager: &PostgresRedoManager) -> Result<(), WalRedoError>
-where
-    I: IntoIterator<Item = Request>,
-{
+fn execute_all(input: Vec<Request>, manager: &PostgresRedoManager) -> Result<(), WalRedoError> {
    // just fire all requests as fast as possible
    input.into_iter().try_for_each(|req| {
        let page = req.execute(manager)?;
@@ -177,7 +143,6 @@ macro_rules! lsn {
    }};
 }

-/// Short payload, 1132 bytes.
 // pg_records are copypasted from log, where they are put with Debug impl of Bytes, which uses \0
 // for null bytes.
 #[allow(clippy::octal_escapes)]
@@ -207,7 +172,6 @@ fn short() -> Request {
    }
 }

-/// Medium sized payload, serializes as 26393 bytes.
 // see [`short`]
 #[allow(clippy::octal_escapes)]
 fn medium() -> Request {
@@ -27,7 +27,7 @@ use tracing::*;
 ///
 use tokio_tar::{Builder, EntryType, Header};

-use crate::tenant::Timeline;
+use crate::tenant::{with_ondemand_download, Timeline};
 use pageserver_api::reltag::{RelTag, SlruKind};

 use postgres_ffi::pg_constants::{DEFAULTTABLESPACE_OID, GLOBALTABLESPACE_OID};
@@ -38,14 +38,6 @@ use postgres_ffi::PG_TLI;
 use postgres_ffi::{BLCKSZ, RELSEG_SIZE, WAL_SEGMENT_SIZE};
 use utils::lsn::Lsn;

-/// Create basebackup with non-rel data in it.
-/// Only include relational data if 'full_backup' is true.
-///
-/// Currently we use empty 'req_lsn' in two cases:
-///  * During the basebackup right after timeline creation
-///  * When working without safekeepers. In this situation it is important to match the lsn
-///    we are taking basebackup on with the lsn that is used in pageserver's walreceiver
-///    to start the replication.
 pub async fn send_basebackup_tarball<'a, W>(
    write: &'a mut W,
    timeline: &'a Timeline,
@@ -131,6 +123,14 @@ where
    full_backup: bool,
 }

+// Create basebackup with non-rel data in it.
+// Only include relational data if 'full_backup' is true.
+//
+// Currently we use empty lsn in two cases:
+//  * During the basebackup right after timeline creation
+//  * When working without safekeepers. In this situation it is important to match the lsn
+//    we are taking basebackup on with the lsn that is used in pageserver's walreceiver
+//    to start the replication.
 impl<'a, W> Basebackup<'a, W>
 where
    W: AsyncWrite + Send + Sync + Unpin,
@@ -171,23 +171,30 @@ where
            SlruKind::MultiXactOffsets,
            SlruKind::MultiXactMembers,
        ] {
-            for segno in self.timeline.list_slru_segments(kind, self.lsn).await? {
+            for segno in
+                with_ondemand_download(|| self.timeline.list_slru_segments(kind, self.lsn)).await?
+            {
                self.add_slru_segment(kind, segno).await?;
            }
        }

        // Create tablespace directories
-        for ((spcnode, dbnode), has_relmap_file) in self.timeline.list_dbdirs(self.lsn).await? {
+        for ((spcnode, dbnode), has_relmap_file) in
+            with_ondemand_download(|| self.timeline.list_dbdirs(self.lsn)).await?
+        {
            self.add_dbdir(spcnode, dbnode, has_relmap_file).await?;

            // Gather and send relational files in each database if full backup is requested.
            if self.full_backup {
-                for rel in self.timeline.list_rels(spcnode, dbnode, self.lsn).await? {
+                for rel in
+                    with_ondemand_download(|| self.timeline.list_rels(spcnode, dbnode, self.lsn))
+                        .await?
+                {
                    self.add_rel(rel).await?;
                }
            }
        }
-        for xid in self.timeline.list_twophase_files(self.lsn).await? {
+        for xid in with_ondemand_download(|| self.timeline.list_twophase_files(self.lsn)).await? {
            self.add_twophase_file(xid).await?;
        }

@@ -203,7 +210,8 @@ where
    }

    async fn add_rel(&mut self, tag: RelTag) -> anyhow::Result<()> {
-        let nblocks = self.timeline.get_rel_size(tag, self.lsn, false).await?;
+        let nblocks =
+            with_ondemand_download(|| self.timeline.get_rel_size(tag, self.lsn, false)).await?;

        // If the relation is empty, create an empty file
        if nblocks == 0 {
@@ -221,10 +229,11 @@ where

            let mut segment_data: Vec<u8> = vec![];
            for blknum in startblk..endblk {
-                let img = self
-                    .timeline
-                    .get_rel_page_at_lsn(tag, blknum, self.lsn, false)
-                    .await?;
+                let img = with_ondemand_download(|| {
+                    self.timeline
+                        .get_rel_page_at_lsn(tag, blknum, self.lsn, false)
+                })
+                .await?;
                segment_data.extend_from_slice(&img[..]);
            }

@@ -243,17 +252,17 @@ where
    // Generate SLRU segment files from repository.
    //
    async fn add_slru_segment(&mut self, slru: SlruKind, segno: u32) -> anyhow::Result<()> {
-        let nblocks = self
-            .timeline
-            .get_slru_segment_size(slru, segno, self.lsn)
-            .await?;
+        let nblocks =
+            with_ondemand_download(|| self.timeline.get_slru_segment_size(slru, segno, self.lsn))
+                .await?;

        let mut slru_buf: Vec<u8> = Vec::with_capacity(nblocks as usize * BLCKSZ as usize);
        for blknum in 0..nblocks {
-            let img = self
-                .timeline
-                .get_slru_page_at_lsn(slru, segno, blknum, self.lsn)
-                .await?;
+            let img = with_ondemand_download(|| {
+                self.timeline
+                    .get_slru_page_at_lsn(slru, segno, blknum, self.lsn)
+            })
+            .await?;

            if slru == SlruKind::Clog {
                ensure!(img.len() == BLCKSZ as usize || img.len() == BLCKSZ as usize + 8);
@@ -285,10 +294,9 @@ where
        has_relmap_file: bool,
    ) -> anyhow::Result<()> {
        let relmap_img = if has_relmap_file {
-            let img = self
-                .timeline
-                .get_relmap_file(spcnode, dbnode, self.lsn)
-                .await?;
+            let img =
+                with_ondemand_download(|| self.timeline.get_relmap_file(spcnode, dbnode, self.lsn))
+                    .await?;
            ensure!(img.len() == 512);
            Some(img)
        } else {
@@ -321,9 +329,7 @@ where
            // XLOG_TBLSPC_DROP records. But we probably should just
            // throw an error on CREATE TABLESPACE in the first place.
            if !has_relmap_file
-                && self
-                    .timeline
-                    .list_rels(spcnode, dbnode, self.lsn)
+                && with_ondemand_download(|| self.timeline.list_rels(spcnode, dbnode, self.lsn))
                    .await?
                    .is_empty()
            {
@@ -356,7 +362,7 @@ where
    // Extract twophase state files
    //
    async fn add_twophase_file(&mut self, xid: TransactionId) -> anyhow::Result<()> {
-        let img = self.timeline.get_twophase_file(xid, self.lsn).await?;
+        let img = with_ondemand_download(|| self.timeline.get_twophase_file(xid, self.lsn)).await?;

        let mut buf = BytesMut::new();
        buf.extend_from_slice(&img[..]);
@@ -392,14 +398,10 @@ where
            )
            .await?;

-        let checkpoint_bytes = self
-            .timeline
-            .get_checkpoint(self.lsn)
+        let checkpoint_bytes = with_ondemand_download(|| self.timeline.get_checkpoint(self.lsn))
            .await
            .context("failed to get checkpoint bytes")?;
-        let pg_control_bytes = self
-            .timeline
-            .get_control_file(self.lsn)
+        let pg_control_bytes = with_ondemand_download(|| self.timeline.get_control_file(self.lsn))
            .await
            .context("failed get control bytes")?;

@@ -336,7 +336,6 @@ fn start_pageserver(conf: &'static PageServerConf) -> anyhow::Result<()> {
                    pageserver::consumption_metrics::collect_metrics(
                        metric_collection_endpoint,
                        conf.metric_collection_interval,
-                        conf.synthetic_size_calculation_interval,
                        conf.id,
                    )
                    .instrument(info_span!("metrics_collection"))
@@ -59,8 +59,6 @@ pub mod defaults {

    pub const DEFAULT_METRIC_COLLECTION_INTERVAL: &str = "10 min";
    pub const DEFAULT_METRIC_COLLECTION_ENDPOINT: Option<reqwest::Url> = None;
-    pub const DEFAULT_SYNTHETIC_SIZE_CALCULATION_INTERVAL: &str = "10 min";
-
    ///
    /// Default built-in configuration file.
    ///
@@ -85,7 +83,6 @@ pub mod defaults {
 #concurrent_tenant_size_logical_size_queries = '{DEFAULT_CONCURRENT_TENANT_SIZE_LOGICAL_SIZE_QUERIES}'

 #metric_collection_interval = '{DEFAULT_METRIC_COLLECTION_INTERVAL}'
-#synthetic_size_calculation_interval = '{DEFAULT_SYNTHETIC_SIZE_CALCULATION_INTERVAL}'

 # [tenant_config]
 #checkpoint_distance = {DEFAULT_CHECKPOINT_DISTANCE} # in bytes
@@ -155,7 +152,6 @@ pub struct PageServerConf {
    // How often to collect metrics and send them to the metrics endpoint.
    pub metric_collection_interval: Duration,
    pub metric_collection_endpoint: Option<Url>,
-    pub synthetic_size_calculation_interval: Duration,

    pub test_remote_failures: u64,
 }
@@ -219,7 +215,6 @@ struct PageServerConfigBuilder {

    metric_collection_interval: BuilderValue<Duration>,
    metric_collection_endpoint: BuilderValue<Option<Url>>,
-    synthetic_size_calculation_interval: BuilderValue<Duration>,

    test_remote_failures: BuilderValue<u64>,
 }
@@ -260,10 +255,6 @@ impl Default for PageServerConfigBuilder {
                DEFAULT_METRIC_COLLECTION_INTERVAL,
            )
            .expect("cannot parse default metric collection interval")),
-            synthetic_size_calculation_interval: Set(humantime::parse_duration(
-                DEFAULT_SYNTHETIC_SIZE_CALCULATION_INTERVAL,
-            )
-            .expect("cannot parse default synthetic size calculation interval")),
            metric_collection_endpoint: Set(DEFAULT_METRIC_COLLECTION_ENDPOINT),

            test_remote_failures: Set(0),
@@ -351,14 +342,6 @@ impl PageServerConfigBuilder {
        self.metric_collection_endpoint = BuilderValue::Set(metric_collection_endpoint)
    }

-    pub fn synthetic_size_calculation_interval(
-        &mut self,
-        synthetic_size_calculation_interval: Duration,
-    ) {
-        self.synthetic_size_calculation_interval =
-            BuilderValue::Set(synthetic_size_calculation_interval)
-    }
-
    pub fn test_remote_failures(&mut self, fail_first: u64) {
        self.test_remote_failures = BuilderValue::Set(fail_first);
    }
@@ -416,9 +399,6 @@ impl PageServerConfigBuilder {
            metric_collection_endpoint: self
                .metric_collection_endpoint
                .ok_or(anyhow!("missing metric_collection_endpoint"))?,
-            synthetic_size_calculation_interval: self
-                .synthetic_size_calculation_interval
-                .ok_or(anyhow!("missing synthetic_size_calculation_interval"))?,
            test_remote_failures: self
                .test_remote_failures
                .ok_or(anyhow!("missing test_remote_failuers"))?,
@@ -597,8 +577,7 @@ impl PageServerConf {
                    let endpoint = parse_toml_string(key, item)?.parse().context("failed to parse metric_collection_endpoint")?;
                    builder.metric_collection_endpoint(Some(endpoint));
                },
-                "synthetic_size_calculation_interval" =>
-                    builder.synthetic_size_calculation_interval(parse_toml_duration(key, item)?),
+
                "test_remote_failures" => builder.test_remote_failures(parse_toml_u64(key, item)?),
                _ => bail!("unrecognized pageserver option '{key}'"),
            }
@@ -722,7 +701,6 @@ impl PageServerConf {
            concurrent_tenant_size_logical_size_queries: ConfigurableSemaphore::default(),
            metric_collection_interval: Duration::from_secs(60),
            metric_collection_endpoint: defaults::DEFAULT_METRIC_COLLECTION_ENDPOINT,
-            synthetic_size_calculation_interval: Duration::from_secs(60),
            test_remote_failures: 0,
        }
    }
@@ -856,7 +834,6 @@ id = 10

 metric_collection_interval = '222 s'
 metric_collection_endpoint = 'http://localhost:80/metrics'
-synthetic_size_calculation_interval = '333 s'
 log_format = 'json'

 "#;
@@ -903,9 +880,6 @@ log_format = 'json'
                    defaults::DEFAULT_METRIC_COLLECTION_INTERVAL
                )?,
                metric_collection_endpoint: defaults::DEFAULT_METRIC_COLLECTION_ENDPOINT,
-                synthetic_size_calculation_interval: humantime::parse_duration(
-                    defaults::DEFAULT_SYNTHETIC_SIZE_CALCULATION_INTERVAL
-                )?,
                test_remote_failures: 0,
            },
            "Correct defaults should be used when no config values are provided"
@@ -952,7 +926,6 @@ log_format = 'json'
                concurrent_tenant_size_logical_size_queries: ConfigurableSemaphore::default(),
                metric_collection_interval: Duration::from_secs(222),
                metric_collection_endpoint: Some(Url::parse("http://localhost:80/metrics")?),
-                synthetic_size_calculation_interval: Duration::from_secs(333),
                test_remote_failures: 0,
            },
            "Should be able to parse all basic config values correctly"
@@ -3,74 +3,154 @@
 //! and push them to a HTTP endpoint.
 //! Cache metrics to send only the updated ones.
 //!
-use crate::task_mgr::{self, TaskKind, BACKGROUND_RUNTIME};
-use crate::tenant::mgr;
+
 use anyhow;
-use chrono::Utc;
-use consumption_metrics::{idempotency_key, Event, EventChunk, EventType, CHUNK_SIZE};
+use tracing::*;
+use utils::id::NodeId;
+use utils::id::TimelineId;
+
+use crate::task_mgr;
+use crate::tenant::mgr;
 use pageserver_api::models::TenantState;
-use reqwest::Url;
-use serde::Serialize;
+use utils::id::TenantId;
+
+use serde::{Deserialize, Serialize};
 use serde_with::{serde_as, DisplayFromStr};
 use std::collections::HashMap;
+use std::fmt;
+use std::str::FromStr;
 use std::time::Duration;
-use tracing::*;
-use utils::id::{NodeId, TenantId, TimelineId};

-const WRITTEN_SIZE: &str = "written_size";
-const SYNTHETIC_STORAGE_SIZE: &str = "synthetic_storage_size";
-const RESIDENT_SIZE: &str = "resident_size";
-const REMOTE_STORAGE_SIZE: &str = "remote_storage_size";
-const TIMELINE_LOGICAL_SIZE: &str = "timeline_logical_size";
+use chrono::{DateTime, Utc};
+use rand::Rng;
+use reqwest::Url;

+/// ConsumptionMetric struct that defines the format for one metric entry
+/// i.e.
+///
+/// ```json
+/// {
+/// "metric": "remote_storage_size",
+/// "type": "absolute",
+/// "tenant_id": "5d07d9ce9237c4cd845ea7918c0afa7d",
+/// "timeline_id": "a03ebb4f5922a1c56ff7485cc8854143",
+/// "time": "2022-12-28T11:07:19.317310284Z",
+/// "idempotency_key": "2022-12-28 11:07:19.317310324 UTC-1-4019",
+/// "value": 12345454,
+/// }
+/// ```
 #[serde_as]
-#[derive(Serialize)]
-struct Ids {
+#[derive(Serialize, Deserialize, Debug, Clone, Eq, PartialEq, Ord, PartialOrd)]
+pub struct ConsumptionMetric {
+    pub metric: ConsumptionMetricKind,
+    #[serde(rename = "type")]
+    pub metric_type: &'static str,
    #[serde_as(as = "DisplayFromStr")]
-    tenant_id: TenantId,
+    pub tenant_id: TenantId,
    #[serde_as(as = "Option<DisplayFromStr>")]
    #[serde(skip_serializing_if = "Option::is_none")]
-    timeline_id: Option<TimelineId>,
+    pub timeline_id: Option<TimelineId>,
+    pub time: DateTime<Utc>,
+    pub idempotency_key: String,
+    pub value: u64,
+}
+
+impl ConsumptionMetric {
+    pub fn new_absolute<R: Rng + ?Sized>(
+        metric: ConsumptionMetricKind,
+        tenant_id: TenantId,
+        timeline_id: Option<TimelineId>,
+        value: u64,
+        node_id: NodeId,
+        rng: &mut R,
+    ) -> Self {
+        Self {
+            metric,
+            metric_type: "absolute",
+            tenant_id,
+            timeline_id,
+            time: Utc::now(),
+            // key that allows metric collector to distinguish unique events
+            idempotency_key: format!("{}-{}-{:04}", Utc::now(), node_id, rng.gen_range(0..=9999)),
+            value,
+        }
+    }
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Ord, PartialOrd, Serialize, Deserialize)]
+#[serde(rename_all = "snake_case")]
+pub enum ConsumptionMetricKind {
+    /// Amount of WAL produced , by a timeline, i.e. last_record_lsn
+    /// This is an absolute, per-timeline metric.
+    WrittenSize,
+    /// Size of all tenant branches including WAL
+    /// This is an absolute, per-tenant metric.
+    /// This is the same metric that tenant/tenant_id/size endpoint returns.
+    SyntheticStorageSize,
+    /// Size of all the layer files in the tenant's directory on disk on the pageserver.
+    /// This is an absolute, per-tenant metric.
+    /// See also prometheus metric RESIDENT_PHYSICAL_SIZE.
+    ResidentSize,
+    /// Size of the remote storage (S3) directory.
+    /// This is an absolute, per-tenant metric.
+    RemoteStorageSize,
+    /// Logical size of the data in the timeline
+    /// This is an absolute, per-timeline metric
+    TimelineLogicalSize,
+}
+
+impl FromStr for ConsumptionMetricKind {
+    type Err = anyhow::Error;
+
+    fn from_str(s: &str) -> Result<Self, Self::Err> {
+        match s {
+            "written_size" => Ok(Self::WrittenSize),
+            "synthetic_storage_size" => Ok(Self::SyntheticStorageSize),
+            "resident_size" => Ok(Self::ResidentSize),
+            "remote_storage_size" => Ok(Self::RemoteStorageSize),
+            "timeline_logical_size" => Ok(Self::TimelineLogicalSize),
+            _ => anyhow::bail!("invalid value \"{s}\" for metric type"),
+        }
+    }
+}
+
+impl fmt::Display for ConsumptionMetricKind {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        f.write_str(match self {
+            ConsumptionMetricKind::WrittenSize => "written_size",
+            ConsumptionMetricKind::SyntheticStorageSize => "synthetic_storage_size",
+            ConsumptionMetricKind::ResidentSize => "resident_size",
+            ConsumptionMetricKind::RemoteStorageSize => "remote_storage_size",
+            ConsumptionMetricKind::TimelineLogicalSize => "timeline_logical_size",
+        })
+    }
 }

-/// Key that uniquely identifies the object, this metric describes.
 #[derive(Debug, Clone, PartialEq, Eq, Hash)]
-pub struct PageserverConsumptionMetricsKey {
-    pub tenant_id: TenantId,
-    pub timeline_id: Option<TimelineId>,
-    pub metric: &'static str,
+pub struct ConsumptionMetricsKey {
+    tenant_id: TenantId,
+    timeline_id: Option<TimelineId>,
+    metric: ConsumptionMetricKind,
+}
+
+#[derive(serde::Serialize)]
+struct EventChunk<'a> {
+    events: &'a [ConsumptionMetric],
 }

 /// Main thread that serves metrics collection
 pub async fn collect_metrics(
    metric_collection_endpoint: &Url,
    metric_collection_interval: Duration,
-    synthetic_size_calculation_interval: Duration,
    node_id: NodeId,
 ) -> anyhow::Result<()> {
    let mut ticker = tokio::time::interval(metric_collection_interval);

    info!("starting collect_metrics");

-    // spin up background worker that caclulates tenant sizes
-    task_mgr::spawn(
-        BACKGROUND_RUNTIME.handle(),
-        TaskKind::CalculateSyntheticSize,
-        None,
-        None,
-        "synthetic size calculation",
-        false,
-        async move {
-            calculate_synthetic_size_worker(synthetic_size_calculation_interval)
-                .instrument(info_span!("synthetic_size_worker"))
-                .await?;
-            Ok(())
-        },
-    );
-
    // define client here to reuse it for all requests
    let client = reqwest::Client::new();
-    let mut cached_metrics: HashMap<PageserverConsumptionMetricsKey, u64> = HashMap::new();
+    let mut cached_metrics: HashMap<ConsumptionMetricsKey, u64> = HashMap::new();

    loop {
        tokio::select! {
@@ -79,10 +159,7 @@ pub async fn collect_metrics(
                return Ok(());
            },
            _ = ticker.tick() => {
-                if let Err(err) = collect_metrics_iteration(&client, &mut cached_metrics, metric_collection_endpoint, node_id).await
-                {
-                    error!("metrics collection failed: {err:?}");
-                }
+                collect_metrics_task(&client, &mut cached_metrics, metric_collection_endpoint, node_id).await?;
            }
        }
    }
@@ -92,20 +169,15 @@ pub async fn collect_metrics(
 ///
 /// Gather per-tenant and per-timeline metrics and send them to the `metric_collection_endpoint`.
 /// Cache metrics to avoid sending the same metrics multiple times.
-///
-/// TODO
-/// - refactor this function (chunking+sending part) to reuse it in proxy module;
-/// - improve error handling. Now if one tenant fails to collect metrics,
-/// the whole iteration fails and metrics for other tenants are not collected.
-pub async fn collect_metrics_iteration(
+pub async fn collect_metrics_task(
    client: &reqwest::Client,
-    cached_metrics: &mut HashMap<PageserverConsumptionMetricsKey, u64>,
+    cached_metrics: &mut HashMap<ConsumptionMetricsKey, u64>,
    metric_collection_endpoint: &reqwest::Url,
    node_id: NodeId,
 ) -> anyhow::Result<()> {
-    let mut current_metrics: Vec<(PageserverConsumptionMetricsKey, u64)> = Vec::new();
+    let mut current_metrics: Vec<(ConsumptionMetricsKey, u64)> = Vec::new();
    trace!(
-        "starting collect_metrics_iteration. metric_collection_endpoint: {}",
+        "starting collect_metrics_task. metric_collection_endpoint: {}",
        metric_collection_endpoint
    );

@@ -129,10 +201,10 @@ pub async fn collect_metrics_iteration(
                let timeline_written_size = u64::from(timeline.get_last_record_lsn());

                current_metrics.push((
-                    PageserverConsumptionMetricsKey {
+                    ConsumptionMetricsKey {
                        tenant_id,
                        timeline_id: Some(timeline.timeline_id),
-                        metric: WRITTEN_SIZE,
+                        metric: ConsumptionMetricKind::WrittenSize,
                    },
                    timeline_written_size,
                ));
@@ -141,10 +213,10 @@ pub async fn collect_metrics_iteration(
                // Only send timeline logical size when it is fully calculated.
                if is_exact {
                    current_metrics.push((
-                        PageserverConsumptionMetricsKey {
+                        ConsumptionMetricsKey {
                            tenant_id,
                            timeline_id: Some(timeline.timeline_id),
-                            metric: TIMELINE_LOGICAL_SIZE,
+                            metric: ConsumptionMetricKind::TimelineLogicalSize,
                        },
                        timeline_logical_size,
                    ));
@@ -162,34 +234,24 @@ pub async fn collect_metrics_iteration(
        );

        current_metrics.push((
-            PageserverConsumptionMetricsKey {
+            ConsumptionMetricsKey {
                tenant_id,
                timeline_id: None,
-                metric: RESIDENT_SIZE,
+                metric: ConsumptionMetricKind::ResidentSize,
            },
            tenant_resident_size,
        ));

        current_metrics.push((
-            PageserverConsumptionMetricsKey {
+            ConsumptionMetricsKey {
                tenant_id,
                timeline_id: None,
-                metric: REMOTE_STORAGE_SIZE,
+                metric: ConsumptionMetricKind::RemoteStorageSize,
            },
            tenant_remote_size,
        ));

-        // Note that this metric is calculated in a separate bgworker
-        // Here we only use cached value, which may lag behind the real latest one
-        let tenant_synthetic_size = tenant.get_cached_synthetic_size();
-        current_metrics.push((
-            PageserverConsumptionMetricsKey {
-                tenant_id,
-                timeline_id: None,
-                metric: SYNTHETIC_STORAGE_SIZE,
-            },
-            tenant_synthetic_size,
-        ));
+        // TODO add SyntheticStorageSize metric
    }

    // Filter metrics
@@ -205,29 +267,35 @@ pub async fn collect_metrics_iteration(

    // Send metrics.
    // Split into chunks of 1000 metrics to avoid exceeding the max request size
+    const CHUNK_SIZE: usize = 1000;
    let chunks = current_metrics.chunks(CHUNK_SIZE);

-    let mut chunk_to_send: Vec<Event<Ids>> = Vec::with_capacity(CHUNK_SIZE);
+    let mut chunk_to_send: Vec<ConsumptionMetric> = Vec::with_capacity(1000);

    for chunk in chunks {
        chunk_to_send.clear();

-        // enrich metrics with type,timestamp and idempotency key before sending
-        chunk_to_send.extend(chunk.iter().map(|(curr_key, curr_val)| Event {
-            kind: EventType::Absolute { time: Utc::now() },
-            metric: curr_key.metric,
-            idempotency_key: idempotency_key(node_id.to_string()),
-            value: *curr_val,
-            extra: Ids {
-                tenant_id: curr_key.tenant_id,
-                timeline_id: curr_key.timeline_id,
-            },
-        }));
+        // this code block is needed to convince compiler
+        // that rng is not reused aroung await point
+        {
+            // enrich metrics with timestamp and metric_kind before sending
+            let mut rng = rand::thread_rng();
+            chunk_to_send.extend(chunk.iter().map(|(curr_key, curr_val)| {
+                ConsumptionMetric::new_absolute(
+                    curr_key.metric,
+                    curr_key.tenant_id,
+                    curr_key.timeline_id,
+                    *curr_val,
+                    node_id,
+                    &mut rng,
+                )
+            }));
+        }

        let chunk_json = serde_json::value::to_raw_value(&EventChunk {
            events: &chunk_to_send,
        })
-        .expect("PageserverConsumptionMetric should not fail serialization");
+        .expect("ConsumptionMetric should not fail serialization");

        let res = client
            .post(metric_collection_endpoint.clone())
@@ -254,39 +322,3 @@ pub async fn collect_metrics_iteration(

    Ok(())
 }
-
-/// Caclculate synthetic size for each active tenant
-pub async fn calculate_synthetic_size_worker(
-    synthetic_size_calculation_interval: Duration,
-) -> anyhow::Result<()> {
-    info!("starting calculate_synthetic_size_worker");
-
-    let mut ticker = tokio::time::interval(synthetic_size_calculation_interval);
-
-    loop {
-        tokio::select! {
-            _ = task_mgr::shutdown_watcher() => {
-                return Ok(());
-            },
-        _ = ticker.tick() => {
-
-                let tenants = mgr::list_tenants().await;
-                // iterate through list of Active tenants and collect metrics
-                for (tenant_id, tenant_state) in tenants {
-
-                    if tenant_state != TenantState::Active {
-                        continue;
-                    }
-
-                    if let Ok(tenant) = mgr::get_tenant(tenant_id, true).await
-                    {
-                        if let Err(e) = tenant.calculate_synthetic_size().await {
-                            error!("failed to calculate synthetic size for tenant {}: {}", tenant_id, e);
-                        }
-                    }
-
-                }
-            }
-        }
-    }
-}
@@ -430,13 +430,6 @@ paths:
        schema:
          type: string
          format: hex
-      - name: inputs_only
-        in: query
-        required: false
-        schema:
-          type: boolean
-        description: |
-          When true, skip calculation and only provide the model inputs (for debugging). Defaults to false.
    get:
      description: |
        Calculate tenant's size, which is a mixture of WAL (bytes) and logical_size (bytes).
@@ -456,9 +449,8 @@ paths:
                    format: hex
                  size:
                    type: integer
-                    nullable: true
                    description: |
-                      Size metric in bytes or null if inputs_only=true was given.
+                      Size metric in bytes.
        "401":
          description: Unauthorized Error
          content:
@@ -3,7 +3,6 @@ use std::sync::Arc;
 use anyhow::{anyhow, Context, Result};
 use hyper::StatusCode;
 use hyper::{Body, Request, Response, Uri};
-use pageserver_api::models::DownloadRemoteLayersTaskSpawnRequest;
 use remote_storage::GenericRemoteStorage;
 use tokio_util::sync::CancellationToken;
 use tracing::*;
@@ -14,7 +13,7 @@ use super::models::{
 };
 use crate::pgdatadir_mapping::LsnForTimestamp;
 use crate::tenant::config::TenantConfOpt;
-use crate::tenant::{PageReconstructError, Timeline};
+use crate::tenant::{with_ondemand_download, Timeline};
 use crate::{config::PageServerConf, tenant::mgr};
 use utils::{
    auth::JwtAuth,
@@ -78,15 +77,6 @@ fn check_permission(request: &Request<Body>, tenant_id: Option<TenantId>) -> Res
    })
 }

-fn apierror_from_prerror(err: PageReconstructError) -> ApiError {
-    match err {
-        PageReconstructError::Other(err) => ApiError::InternalServerError(err),
-        PageReconstructError::WalRedo(err) => {
-            ApiError::InternalServerError(anyhow::Error::new(err))
-        }
-    }
-}
-
 // Helper function to construct a TimelineInfo struct for a timeline
 async fn build_timeline_info(
    timeline: &Arc<Timeline>,
@@ -239,7 +229,11 @@ fn query_param_present(request: &Request<Body>, param: &str) -> bool {
    request
        .uri()
        .query()
-        .map(|v| url::form_urlencoded::parse(v.as_bytes()).any(|(p, _)| p == param))
+        .map(|v| {
+            url::form_urlencoded::parse(v.as_bytes())
+                .into_owned()
+                .any(|(p, _)| p == param)
+        })
        .unwrap_or(false)
 }

@@ -248,12 +242,13 @@ fn get_query_param(request: &Request<Body>, param_name: &str) -> Result<String,
        Err(ApiError::BadRequest(anyhow!("empty query in request"))),
        |v| {
            url::form_urlencoded::parse(v.as_bytes())
+                .into_owned()
                .find(|(k, _)| k == param_name)
                .map_or(
                    Err(ApiError::BadRequest(anyhow!(
                        "no {param_name} specified in query parameters"
                    ))),
-                    |(_, v)| Ok(v.into_owned()),
+                    |(_, v)| Ok(v),
                )
        },
    )
@@ -277,7 +272,7 @@ async fn timeline_detail_handler(request: Request<Body>) -> Result<Response<Body

        let timeline_info = build_timeline_info(&timeline, include_non_incremental_logical_size)
            .await
-            .context("get local timeline info")
+            .context("Failed to get local timeline info: {e:#}")
            .map_err(ApiError::InternalServerError)?;

        Ok::<_, ApiError>(timeline_info)
@@ -303,10 +298,9 @@ async fn get_lsn_by_timestamp_handler(request: Request<Body>) -> Result<Response
        .await
        .and_then(|tenant| tenant.get_timeline(timeline_id, true))
        .map_err(ApiError::NotFound)?;
-    let result = timeline
-        .find_lsn_for_timestamp(timestamp_pg)
+    let result = with_ondemand_download(|| timeline.find_lsn_for_timestamp(timestamp_pg))
        .await
-        .map_err(apierror_from_prerror)?;
+        .map_err(ApiError::InternalServerError)?;

    let result = match result {
        LsnForTimestamp::Present(lsn) => format!("{lsn}"),
@@ -448,39 +442,21 @@ async fn tenant_status(request: Request<Body>) -> Result<Response<Body>, ApiErro
    json_response(StatusCode::OK, tenant_info)
 }

-/// HTTP endpoint to query the current tenant_size of a tenant.
-///
-/// This is not used by consumption metrics under [`crate::consumption_metrics`], but can be used
-/// to debug any of the calculations. Requires `tenant_id` request parameter, supports
-/// `inputs_only=true|false` (default false) which supports debugging failure to calculate model
-/// values.
 async fn tenant_size_handler(request: Request<Body>) -> Result<Response<Body>, ApiError> {
    let tenant_id: TenantId = parse_request_param(&request, "tenant_id")?;
    check_permission(&request, Some(tenant_id))?;

-    let inputs_only = if query_param_present(&request, "inputs_only") {
-        get_query_param(&request, "inputs_only")?
-            .parse()
-            .map_err(|_| ApiError::BadRequest(anyhow!("failed to parse inputs_only")))?
-    } else {
-        false
-    };
-
    let tenant = mgr::get_tenant(tenant_id, true)
        .await
        .map_err(ApiError::InternalServerError)?;

-    // this can be long operation
+    // this can be long operation, it currently is not backed by any request coalescing or similar
    let inputs = tenant
        .gather_size_inputs()
        .await
        .map_err(ApiError::InternalServerError)?;

-    let size = if !inputs_only {
-        Some(inputs.calculate().map_err(ApiError::InternalServerError)?)
-    } else {
-        None
-    };
+    let size = inputs.calculate().map_err(ApiError::InternalServerError)?;

    /// Private response type with the additional "unstable" `inputs` field.
    ///
@@ -492,9 +468,7 @@ async fn tenant_size_handler(request: Request<Body>) -> Result<Response<Body>, A
        #[serde_as(as = "serde_with::DisplayFromStr")]
        id: TenantId,
        /// Size is a mixture of WAL and logical size, so the unit is bytes.
-        ///
-        /// Will be none if `?inputs_only=true` was given.
-        size: Option<u64>,
+        size: u64,
        inputs: crate::tenant::size::ModelInputs,
    }

@@ -611,7 +585,7 @@ async fn tenant_create_handler(mut request: Request<Body>) -> Result<Response<Bo
            // is Active when this function returns.
            if let res @ Err(_) = tenant.wait_to_become_active().await {
                // This shouldn't happen because we just created the tenant directory
-                // in tenant::mgr::create_tenant, and there aren't any remote timelines
+                // in tenant_mgr::create_tenant, and there aren't any remote timelines
                // to load, so, nothing can really fail during load.
                // Don't do cleanup because we don't know how we got here.
                // The tenant will likely be in `Broken` state and subsequent
@@ -764,16 +738,16 @@ async fn timeline_compact_handler(request: Request<Body>) -> Result<Response<Bod
    let timeline_id: TimelineId = parse_request_param(&request, "timeline_id")?;
    check_permission(&request, Some(tenant_id))?;

-    let result_receiver = mgr::immediate_compact(tenant_id, timeline_id)
+    let tenant = mgr::get_tenant(tenant_id, true)
        .await
-        .context("spawn compaction task")
-        .map_err(ApiError::InternalServerError)?;
-
-    let result: anyhow::Result<()> = result_receiver
+        .map_err(ApiError::NotFound)?;
+    let timeline = tenant
+        .get_timeline(timeline_id, true)
+        .map_err(ApiError::NotFound)?;
+    timeline
+        .compact()
        .await
-        .context("receive compaction result")
        .map_err(ApiError::InternalServerError)?;
-    result.map_err(ApiError::InternalServerError)?;

    json_response(StatusCode::OK, ())
 }
@@ -804,11 +778,10 @@ async fn timeline_checkpoint_handler(request: Request<Body>) -> Result<Response<
 }

 async fn timeline_download_remote_layers_handler_post(
-    mut request: Request<Body>,
+    request: Request<Body>,
 ) -> Result<Response<Body>, ApiError> {
    let tenant_id: TenantId = parse_request_param(&request, "tenant_id")?;
    let timeline_id: TimelineId = parse_request_param(&request, "timeline_id")?;
-    let body: DownloadRemoteLayersTaskSpawnRequest = json_request(&mut request).await?;
    check_permission(&request, Some(tenant_id))?;

    let tenant = mgr::get_tenant(tenant_id, true)
@@ -817,7 +790,7 @@ async fn timeline_download_remote_layers_handler_post(
    let timeline = tenant
        .get_timeline(timeline_id, true)
        .map_err(ApiError::NotFound)?;
-    match timeline.spawn_download_all_remote_layers(body).await {
+    match timeline.spawn_download_all_remote_layers().await {
        Ok(st) => json_response(StatusCode::ACCEPTED, st),
        Err(st) => json_response(StatusCode::CONFLICT, st),
    }
@@ -143,11 +143,7 @@ async fn import_rel(
    // Call put_rel_creation for every segment of the relation,
    // because there is no guarantee about the order in which we are processing segments.
    // ignore "relation already exists" error
-    //
-    // FIXME: use proper error type for this, instead of parsing the error message.
-    // Or better yet, keep track of which relations we've already created
-    // https://github.com/neondatabase/neon/issues/3309
-    if let Err(e) = modification.put_rel_creation(rel, nblocks as u32).await {
+    if let Err(e) = modification.put_rel_creation(rel, nblocks as u32) {
        if e.to_string().contains("already exists") {
            debug!("relation {} already exists. we must be extending it", rel);
        } else {
@@ -182,7 +178,7 @@ async fn import_rel(
    //
    // If we process rel segments out of order,
    // put_rel_extend will skip the update.
-    modification.put_rel_extend(rel, blknum).await?;
+    modification.put_rel_extend(rel, blknum)?;

    Ok(())
 }
@@ -210,9 +206,7 @@ async fn import_slru(

    ensure!(nblocks <= pg_constants::SLRU_PAGES_PER_SEGMENT as usize);

-    modification
-        .put_slru_segment_creation(slru, segno, nblocks as u32)
-        .await?;
+    modification.put_slru_segment_creation(slru, segno, nblocks as u32)?;

    let mut rpageno = 0;
    loop {
@@ -498,7 +492,7 @@ async fn import_file(
            }
            "pg_filenode.map" => {
                let bytes = read_all_bytes(reader).await?;
-                modification.put_relmap_file(spcnode, dbnode, bytes).await?;
+                modification.put_relmap_file(spcnode, dbnode, bytes)?;
                debug!("imported relmap file")
            }
            "PG_VERSION" => {
@@ -521,7 +515,7 @@ async fn import_file(
        match file_name.as_ref() {
            "pg_filenode.map" => {
                let bytes = read_all_bytes(reader).await?;
-                modification.put_relmap_file(spcnode, dbnode, bytes).await?;
+                modification.put_relmap_file(spcnode, dbnode, bytes)?;
                debug!("imported relmap file")
            }
            "PG_VERSION" => {
@@ -551,9 +545,7 @@ async fn import_file(
        let xid = u32::from_str_radix(file_name.as_ref(), 16)?;

        let bytes = read_all_bytes(reader).await?;
-        modification
-            .put_twophase_file(xid, Bytes::copy_from_slice(&bytes[..]))
-            .await?;
+        modification.put_twophase_file(xid, Bytes::copy_from_slice(&bytes[..]))?;
        debug!("imported twophase file");
    } else if file_path.starts_with("pg_wal") {
        debug!("found wal file in base section. ignore it");
@@ -209,34 +209,15 @@ pub static NUM_ONDISK_LAYERS: Lazy<IntGauge> = Lazy::new(|| {

 // remote storage metrics

-/// NB: increment _after_ recording the current value into [`REMOTE_TIMELINE_CLIENT_CALLS_STARTED_HIST`].
-static REMOTE_TIMELINE_CLIENT_CALLS_UNFINISHED_GAUGE: Lazy<IntGaugeVec> = Lazy::new(|| {
+static REMOTE_UPLOAD_QUEUE_UNFINISHED_TASKS: Lazy<IntGaugeVec> = Lazy::new(|| {
    register_int_gauge_vec!(
-        "pageserver_remote_timeline_client_calls_unfinished",
-        "Number of ongoing calls to remote timeline client. \
-         Used to populate pageserver_remote_timeline_client_calls_started. \
-         This metric is not useful for sampling from Prometheus, but useful in tests.",
+        "pageserver_remote_upload_queue_unfinished_tasks",
+        "Number of tasks in the upload queue that are not finished yet.",
        &["tenant_id", "timeline_id", "file_kind", "op_kind"],
    )
    .expect("failed to define a metric")
 });

-static REMOTE_TIMELINE_CLIENT_CALLS_STARTED_HIST: Lazy<HistogramVec> = Lazy::new(|| {
-    register_histogram_vec!(
-        "pageserver_remote_timeline_client_calls_started",
-        "When calling a remote timeline client method, we record the current value \
-         of the calls_unfinished gauge in this histogram. Plot the histogram \
-         over time in a heatmap to visualize how many operations were ongoing \
-         at a given instant. It gives you a better idea of the queue depth \
-         than plotting the gauge directly, since operations may complete faster \
-         than the sampling interval.",
-        &["tenant_id", "timeline_id", "file_kind", "op_kind"],
-        // The calls_unfinished gauge is an integer gauge, hence we have integer buckets.
-        vec![0.0, 1.0, 2.0, 4.0, 6.0, 8.0, 10.0, 15.0, 20.0, 40.0, 60.0, 80.0, 100.0, 500.0],
-    )
-    .expect("failed to define a metric")
-});
-
 #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
 pub enum RemoteOpKind {
    Upload,
@@ -267,12 +248,15 @@ impl RemoteOpFileKind {
    }
 }

+pub static REMOTE_OPERATION_KINDS: &[&str] = &["upload", "download", "delete"];
+pub static REMOTE_OPERATION_FILE_KINDS: &[&str] = &["layer", "index"];
+pub static REMOTE_OPERATION_STATUSES: &[&str] = &["success", "failure"];
+
 pub static REMOTE_OPERATION_TIME: Lazy<HistogramVec> = Lazy::new(|| {
    register_histogram_vec!(
        "pageserver_remote_operation_seconds",
        "Time spent on remote storage operations. \
-        Grouped by tenant, timeline, operation_kind and status. \
-        Does not account for time spent waiting in remote timeline client's queues.",
+        Grouped by tenant, timeline, operation_kind and status",
        &["tenant_id", "timeline_id", "file_kind", "op_kind", "status"]
    )
    .expect("failed to define a metric")
@@ -491,6 +475,21 @@ impl Drop for TimelineMetrics {
        for op in SMGR_QUERY_TIME_OPERATIONS {
            let _ = SMGR_QUERY_TIME.remove_label_values(&[op, tenant_id, timeline_id]);
        }
+
+        let _ = REMOTE_UPLOAD_QUEUE_UNFINISHED_TASKS.remove_label_values(&[tenant_id, timeline_id]);
+        for file_kind in REMOTE_OPERATION_FILE_KINDS {
+            for op in REMOTE_OPERATION_KINDS {
+                for status in REMOTE_OPERATION_STATUSES {
+                    let _ = REMOTE_OPERATION_TIME.remove_label_values(&[
+                        tenant_id,
+                        timeline_id,
+                        file_kind,
+                        op,
+                        status,
+                    ]);
+                }
+            }
+        }
    }
 }

@@ -511,8 +510,7 @@ pub struct RemoteTimelineClientMetrics {
    timeline_id: String,
    remote_physical_size_gauge: Mutex<Option<UIntGauge>>,
    remote_operation_time: Mutex<HashMap<(&'static str, &'static str, &'static str), Histogram>>,
-    calls_unfinished_gauge: Mutex<HashMap<(&'static str, &'static str), IntGauge>>,
-    calls_started_hist: Mutex<HashMap<(&'static str, &'static str), Histogram>>,
+    unfinished_tasks: Mutex<HashMap<(&'static str, &'static str), IntGauge>>,
 }

 impl RemoteTimelineClientMetrics {
@@ -521,8 +519,7 @@ impl RemoteTimelineClientMetrics {
            tenant_id: tenant_id.to_string(),
            timeline_id: timeline_id.to_string(),
            remote_operation_time: Mutex::new(HashMap::default()),
-            calls_unfinished_gauge: Mutex::new(HashMap::default()),
-            calls_started_hist: Mutex::new(HashMap::default()),
+            unfinished_tasks: Mutex::new(HashMap::default()),
            remote_physical_size_gauge: Mutex::new(None),
        }
    }
@@ -561,16 +558,16 @@ impl RemoteTimelineClientMetrics {
        });
        metric.clone()
    }
-    fn calls_unfinished_gauge(
+    pub fn unfinished_tasks(
        &self,
        file_kind: &RemoteOpFileKind,
        op_kind: &RemoteOpKind,
    ) -> IntGauge {
        // XXX would be nice to have an upgradable RwLock
-        let mut guard = self.calls_unfinished_gauge.lock().unwrap();
+        let mut guard = self.unfinished_tasks.lock().unwrap();
        let key = (file_kind.as_str(), op_kind.as_str());
        let metric = guard.entry(key).or_insert_with(move || {
-            REMOTE_TIMELINE_CLIENT_CALLS_UNFINISHED_GAUGE
+            REMOTE_UPLOAD_QUEUE_UNFINISHED_TASKS
                .get_metric_with_label_values(&[
                    &self.tenant_id.to_string(),
                    &self.timeline_id.to_string(),
@@ -581,79 +578,6 @@ impl RemoteTimelineClientMetrics {
        });
        metric.clone()
    }
-
-    fn calls_started_hist(
-        &self,
-        file_kind: &RemoteOpFileKind,
-        op_kind: &RemoteOpKind,
-    ) -> Histogram {
-        // XXX would be nice to have an upgradable RwLock
-        let mut guard = self.calls_started_hist.lock().unwrap();
-        let key = (file_kind.as_str(), op_kind.as_str());
-        let metric = guard.entry(key).or_insert_with(move || {
-            REMOTE_TIMELINE_CLIENT_CALLS_STARTED_HIST
-                .get_metric_with_label_values(&[
-                    &self.tenant_id.to_string(),
-                    &self.timeline_id.to_string(),
-                    key.0,
-                    key.1,
-                ])
-                .unwrap()
-        });
-        metric.clone()
-    }
-}
-
-/// See [`RemoteTimelineClientMetrics::call_begin`].
-#[must_use]
-pub(crate) struct RemoteTimelineClientCallMetricGuard(Option<IntGauge>);
-
-impl RemoteTimelineClientCallMetricGuard {
-    /// Consume this guard object without decrementing the metric.
-    /// The caller vouches to do this manually, so that the prior increment of the gauge will cancel out.
-    pub fn will_decrement_manually(mut self) {
-        self.0 = None; // prevent drop() from decrementing
-    }
-}
-
-impl Drop for RemoteTimelineClientCallMetricGuard {
-    fn drop(&mut self) {
-        if let RemoteTimelineClientCallMetricGuard(Some(guard)) = self {
-            guard.dec();
-        }
-    }
-}
-
-impl RemoteTimelineClientMetrics {
-    /// Increment the metrics that track ongoing calls to the remote timeline client instance.
-    ///
-    /// Drop the returned guard object once the operation is finished to decrement the values.
-    /// Or, use [`RemoteTimelineClientCallMetricGuard::will_decrement_manually`] and [`call_end`] if that
-    /// is more suitable.
-    /// Never do both.
-    pub(crate) fn call_begin(
-        &self,
-        file_kind: &RemoteOpFileKind,
-        op_kind: &RemoteOpKind,
-    ) -> RemoteTimelineClientCallMetricGuard {
-        let unfinished_metric = self.calls_unfinished_gauge(file_kind, op_kind);
-        self.calls_started_hist(file_kind, op_kind)
-            .observe(unfinished_metric.get() as f64);
-        unfinished_metric.inc();
-        RemoteTimelineClientCallMetricGuard(Some(unfinished_metric))
-    }
-
-    /// Manually decrement the metric instead of using the guard object.
-    /// Using the guard object is generally preferable.
-    /// See [`call_begin`] for more context.
-    pub(crate) fn call_end(&self, file_kind: &RemoteOpFileKind, op_kind: &RemoteOpKind) {
-        let unfinished_metric = self.calls_unfinished_gauge(file_kind, op_kind);
-        debug_assert!(
-            unfinished_metric.get() > 0,
-            "begin and end should cancel out"
-        );
-        unfinished_metric.dec();
-    }
 }

 impl Drop for RemoteTimelineClientMetrics {
@@ -663,22 +587,13 @@ impl Drop for RemoteTimelineClientMetrics {
            timeline_id,
            remote_physical_size_gauge,
            remote_operation_time,
-            calls_unfinished_gauge,
-            calls_started_hist,
+            unfinished_tasks,
        } = self;
        for ((a, b, c), _) in remote_operation_time.get_mut().unwrap().drain() {
            let _ = REMOTE_OPERATION_TIME.remove_label_values(&[tenant_id, timeline_id, a, b, c]);
        }
-        for ((a, b), _) in calls_unfinished_gauge.get_mut().unwrap().drain() {
-            let _ = REMOTE_TIMELINE_CLIENT_CALLS_UNFINISHED_GAUGE.remove_label_values(&[
-                tenant_id,
-                timeline_id,
-                a,
-                b,
-            ]);
-        }
-        for ((a, b), _) in calls_started_hist.get_mut().unwrap().drain() {
-            let _ = REMOTE_TIMELINE_CLIENT_CALLS_STARTED_HIST.remove_label_values(&[
+        for ((a, b), _) in unfinished_tasks.get_mut().unwrap().drain() {
+            let _ = REMOTE_UPLOAD_QUEUE_UNFINISHED_TASKS.remove_label_values(&[
                tenant_id,
                timeline_id,
                a,
@@ -9,7 +9,7 @@
 //  custom protocol.
 //

-use anyhow::Context;
+use anyhow::{bail, ensure, Context, Result};
 use bytes::Buf;
 use bytes::Bytes;
 use futures::{Stream, StreamExt};
@@ -19,8 +19,6 @@ use pageserver_api::models::{
    PagestreamFeMessage, PagestreamGetPageRequest, PagestreamGetPageResponse,
    PagestreamNblocksRequest, PagestreamNblocksResponse,
 };
-use pq_proto::ConnectionError;
-use pq_proto::FeStartupPacket;
 use pq_proto::{BeMessage, FeMessage, RowDescriptor};
 use std::io;
 use std::net::TcpListener;
@@ -30,7 +28,6 @@ use std::sync::Arc;
 use std::time::Duration;
 use tracing::*;
 use utils::id::ConnectionId;
-use utils::postgres_backend_async::QueryError;
 use utils::{
    auth::{Claims, JwtAuth, Scope},
    id::{TenantId, TimelineId},
@@ -63,8 +60,8 @@ fn copyin_stream(pgb: &mut PostgresBackend) -> impl Stream<Item = io::Result<Byt
                _ = task_mgr::shutdown_watcher() => {
                    // We were requested to shut down.
                    let msg = format!("pageserver is shutting down");
-                    let _ = pgb.write_message(&BeMessage::ErrorResponse(&msg, None));
-                    Err(QueryError::Other(anyhow::anyhow!(msg)))
+                    let _ = pgb.write_message(&BeMessage::ErrorResponse(&msg));
+                    Err(anyhow::anyhow!(msg))
                }

                msg = pgb.read_message() => { msg }
@@ -77,15 +74,14 @@ fn copyin_stream(pgb: &mut PostgresBackend) -> impl Stream<Item = io::Result<Byt
                        FeMessage::CopyDone => { break },
                        FeMessage::Sync => continue,
                        FeMessage::Terminate => {
-                            let msg = "client terminated connection with Terminate message during COPY";
-                            let query_error_error = QueryError::Disconnected(ConnectionError::Socket(io::Error::new(io::ErrorKind::ConnectionReset, msg)));
-                            pgb.write_message(&BeMessage::ErrorResponse(msg, Some(query_error_error.pg_error_code())))?;
+                            let msg = format!("client terminated connection with Terminate message during COPY");
+                            pgb.write_message(&BeMessage::ErrorResponse(&msg))?;
                            Err(io::Error::new(io::ErrorKind::ConnectionReset, msg))?;
                            break;
                        }
                        m => {
-                            let msg = format!("unexpected message {m:?}");
-                            pgb.write_message(&BeMessage::ErrorResponse(&msg, None))?;
+                            let msg = format!("unexpected message {:?}", m);
+                            pgb.write_message(&BeMessage::ErrorResponse(&msg))?;
                            Err(io::Error::new(io::ErrorKind::Other, msg))?;
                            break;
                        }
@@ -95,16 +91,12 @@ fn copyin_stream(pgb: &mut PostgresBackend) -> impl Stream<Item = io::Result<Byt
                }
                Ok(None) => {
                    let msg = "client closed connection during COPY";
-                    let query_error_error = QueryError::Disconnected(ConnectionError::Socket(io::Error::new(io::ErrorKind::ConnectionReset, msg)));
-                    pgb.write_message(&BeMessage::ErrorResponse(msg, Some(query_error_error.pg_error_code())))?;
+                    pgb.write_message(&BeMessage::ErrorResponse(msg))?;
                    pgb.flush().await?;
                    Err(io::Error::new(io::ErrorKind::ConnectionReset, msg))?;
                }
-                Err(QueryError::Disconnected(ConnectionError::Socket(io_error))) => {
-                    Err(io_error)?;
-                }
-                Err(other) => {
-                    Err(io::Error::new(io::ErrorKind::Other, other))?;
+                Err(e) => {
+                    Err(io::Error::new(io::ErrorKind::Other, e))?;
                }
            };
        }
@@ -202,19 +194,23 @@ async fn page_service_conn_main(
            // we've been requested to shut down
            Ok(())
        }
-        Err(QueryError::Disconnected(ConnectionError::Socket(io_error))) => {
+        Err(err) => {
+            let root_cause_io_err_kind = err
+                .root_cause()
+                .downcast_ref::<io::Error>()
+                .map(|e| e.kind());
+
            // `ConnectionReset` error happens when the Postgres client closes the connection.
            // As this disconnection happens quite often and is expected,
            // we decided to downgrade the logging level to `INFO`.
            // See: https://github.com/neondatabase/neon/issues/1683.
-            if io_error.kind() == io::ErrorKind::ConnectionReset {
+            if root_cause_io_err_kind == Some(io::ErrorKind::ConnectionReset) {
                info!("Postgres client disconnected");
                Ok(())
            } else {
-                Err(io_error).context("Postgres connection error")
+                Err(err)
            }
        }
-        other => other.context("Postgres query error"),
    }
 }

@@ -251,6 +247,7 @@ impl PageRequestMetrics {
    }
 }

+#[derive(Debug)]
 struct PageServerHandler {
    _conf: &'static PageServerConf,
    auth: Option<Arc<JwtAuth>>,
@@ -315,7 +312,7 @@ impl PageServerHandler {
                Some(FeMessage::CopyData(bytes)) => bytes,
                Some(FeMessage::Terminate) => break,
                Some(m) => {
-                    anyhow::bail!("unexpected message: {m:?} during COPY");
+                    bail!("unexpected message: {m:?} during COPY");
                }
                None => break, // client disconnected
            };
@@ -372,7 +369,7 @@ impl PageServerHandler {
        base_lsn: Lsn,
        _end_lsn: Lsn,
        pg_version: u32,
-    ) -> Result<(), QueryError> {
+    ) -> anyhow::Result<()> {
        task_mgr::associate_with(Some(tenant_id), Some(timeline_id));
        // Create empty timeline
        info!("creating new timeline");
@@ -426,16 +423,11 @@ impl PageServerHandler {
        timeline_id: TimelineId,
        start_lsn: Lsn,
        end_lsn: Lsn,
-    ) -> Result<(), QueryError> {
+    ) -> anyhow::Result<()> {
        task_mgr::associate_with(Some(tenant_id), Some(timeline_id));

        let timeline = get_active_timeline_with_timeout(tenant_id, timeline_id).await?;
-        let last_record_lsn = timeline.get_last_record_lsn();
-        if last_record_lsn != start_lsn {
-            return Err(QueryError::Other(
-                anyhow::anyhow!("Cannot import WAL from Lsn {start_lsn} because timeline does not start from the same lsn: {last_record_lsn}"))
-            );
-        }
+        ensure!(timeline.get_last_record_lsn() == start_lsn);

        // TODO leave clean state on error. For now you can use detach to clean
        // up broken state from a failed import.
@@ -459,11 +451,7 @@ impl PageServerHandler {
        }

        // TODO Does it make sense to overshoot?
-        if timeline.get_last_record_lsn() < end_lsn {
-            return Err(QueryError::Other(
-                anyhow::anyhow!("Cannot import WAL from Lsn {start_lsn} because timeline does not start from the same lsn: {last_record_lsn}"))
-            );
-        }
+        ensure!(timeline.get_last_record_lsn() >= end_lsn);

        // Flush data to disk, then upload to s3. No need for a forced checkpoint.
        // We only want to persist the data, and it doesn't matter if it's in the
@@ -492,7 +480,7 @@ impl PageServerHandler {
        mut lsn: Lsn,
        latest: bool,
        latest_gc_cutoff_lsn: &RcuReadGuard<Lsn>,
-    ) -> anyhow::Result<Lsn> {
+    ) -> Result<Lsn> {
        if latest {
            // Latest page version was requested. If LSN is given, it is a hint
            // to the page server that there have been no modifications to the
@@ -523,11 +511,11 @@ impl PageServerHandler {
            }
        } else {
            if lsn == Lsn(0) {
-                anyhow::bail!("invalid LSN(0) in request");
+                bail!("invalid LSN(0) in request");
            }
            timeline.wait_lsn(lsn).await?;
        }
-        anyhow::ensure!(
+        ensure!(
            lsn >= **latest_gc_cutoff_lsn,
            "tried to request a page version that was garbage collected. requested at {} gc cutoff {}",
            lsn, **latest_gc_cutoff_lsn
@@ -540,12 +528,15 @@ impl PageServerHandler {
        &self,
        timeline: &Timeline,
        req: &PagestreamExistsRequest,
-    ) -> anyhow::Result<PagestreamBeMessage> {
+    ) -> Result<PagestreamBeMessage> {
        let latest_gc_cutoff_lsn = timeline.get_latest_gc_cutoff_lsn();
        let lsn = Self::wait_or_get_last_lsn(timeline, req.lsn, req.latest, &latest_gc_cutoff_lsn)
            .await?;

-        let exists = timeline.get_rel_exists(req.rel, lsn, req.latest).await?;
+        let exists = crate::tenant::with_ondemand_download(|| {
+            timeline.get_rel_exists(req.rel, lsn, req.latest)
+        })
+        .await?;

        Ok(PagestreamBeMessage::Exists(PagestreamExistsResponse {
            exists,
@@ -557,12 +548,15 @@ impl PageServerHandler {
        &self,
        timeline: &Timeline,
        req: &PagestreamNblocksRequest,
-    ) -> anyhow::Result<PagestreamBeMessage> {
+    ) -> Result<PagestreamBeMessage> {
        let latest_gc_cutoff_lsn = timeline.get_latest_gc_cutoff_lsn();
        let lsn = Self::wait_or_get_last_lsn(timeline, req.lsn, req.latest, &latest_gc_cutoff_lsn)
            .await?;

-        let n_blocks = timeline.get_rel_size(req.rel, lsn, req.latest).await?;
+        let n_blocks = crate::tenant::with_ondemand_download(|| {
+            timeline.get_rel_size(req.rel, lsn, req.latest)
+        })
+        .await?;

        Ok(PagestreamBeMessage::Nblocks(PagestreamNblocksResponse {
            n_blocks,
@@ -574,14 +568,15 @@ impl PageServerHandler {
        &self,
        timeline: &Timeline,
        req: &PagestreamDbSizeRequest,
-    ) -> anyhow::Result<PagestreamBeMessage> {
+    ) -> Result<PagestreamBeMessage> {
        let latest_gc_cutoff_lsn = timeline.get_latest_gc_cutoff_lsn();
        let lsn = Self::wait_or_get_last_lsn(timeline, req.lsn, req.latest, &latest_gc_cutoff_lsn)
            .await?;

-        let total_blocks = timeline
-            .get_db_size(DEFAULTTABLESPACE_OID, req.dbnode, lsn, req.latest)
-            .await?;
+        let total_blocks = crate::tenant::with_ondemand_download(|| {
+            timeline.get_db_size(DEFAULTTABLESPACE_OID, req.dbnode, lsn, req.latest)
+        })
+        .await?;
        let db_size = total_blocks as i64 * BLCKSZ as i64;

        Ok(PagestreamBeMessage::DbSize(PagestreamDbSizeResponse {
@@ -594,7 +589,7 @@ impl PageServerHandler {
        &self,
        timeline: &Timeline,
        req: &PagestreamGetPageRequest,
-    ) -> anyhow::Result<PagestreamBeMessage> {
+    ) -> Result<PagestreamBeMessage> {
        let latest_gc_cutoff_lsn = timeline.get_latest_gc_cutoff_lsn();
        let lsn = Self::wait_or_get_last_lsn(timeline, req.lsn, req.latest, &latest_gc_cutoff_lsn)
            .await?;
@@ -607,9 +602,10 @@ impl PageServerHandler {
        }
        */

-        let page = timeline
-            .get_rel_page_at_lsn(req.rel, req.blkno, lsn, req.latest)
-            .await?;
+        let page = crate::tenant::with_ondemand_download(|| {
+            timeline.get_rel_page_at_lsn(req.rel, req.blkno, lsn, req.latest)
+        })
+        .await?;

        Ok(PagestreamBeMessage::GetPage(PagestreamGetPageResponse {
            page,
@@ -642,7 +638,7 @@ impl PageServerHandler {
        pgb.write_message(&BeMessage::CopyOutResponse)?;
        pgb.flush().await?;

-        // Send a tarball of the latest layer on the timeline
+        /* Send a tarball of the latest layer on the timeline */
        {
            let mut writer = pgb.copyout_writer();
            basebackup::send_basebackup_tarball(&mut writer, &timeline, lsn, prev_lsn, full_backup)
@@ -658,7 +654,7 @@ impl PageServerHandler {

    // when accessing management api supply None as an argument
    // when using to authorize tenant pass corresponding tenant id
-    fn check_permission(&self, tenant_id: Option<TenantId>) -> anyhow::Result<()> {
+    fn check_permission(&self, tenant_id: Option<TenantId>) -> Result<()> {
        if self.auth.is_none() {
            // auth is set to Trust, nothing to check so just return ok
            return Ok(());
@@ -680,19 +676,20 @@ impl postgres_backend_async::Handler for PageServerHandler {
        &mut self,
        _pgb: &mut PostgresBackend,
        jwt_response: &[u8],
-    ) -> Result<(), QueryError> {
+    ) -> anyhow::Result<()> {
        // this unwrap is never triggered, because check_auth_jwt only called when auth_type is NeonJWT
        // which requires auth to be present
        let data = self
            .auth
            .as_ref()
            .unwrap()
-            .decode(str::from_utf8(jwt_response).context("jwt response is not UTF-8")?)?;
+            .decode(str::from_utf8(jwt_response)?)?;

-        if matches!(data.claims.scope, Scope::Tenant) && data.claims.tenant_id.is_none() {
-            return Err(QueryError::Other(anyhow::anyhow!(
+        if matches!(data.claims.scope, Scope::Tenant) {
+            ensure!(
+                data.claims.tenant_id.is_some(),
                "jwt token scope is Tenant, but tenant id is missing"
-            )));
+            )
        }

        info!(
@@ -704,33 +701,22 @@ impl postgres_backend_async::Handler for PageServerHandler {
        Ok(())
    }

-    fn startup(
-        &mut self,
-        _pgb: &mut PostgresBackend,
-        _sm: &FeStartupPacket,
-    ) -> Result<(), QueryError> {
-        Ok(())
-    }
-
    async fn process_query(
        &mut self,
        pgb: &mut PostgresBackend,
        query_string: &str,
-    ) -> Result<(), QueryError> {
-        debug!("process query {query_string:?}");
+    ) -> anyhow::Result<()> {
+        debug!("process query {:?}", query_string);

        if query_string.starts_with("pagestream ") {
            let (_, params_raw) = query_string.split_at("pagestream ".len());
            let params = params_raw.split(' ').collect::<Vec<_>>();
-            if params.len() != 2 {
-                return Err(QueryError::Other(anyhow::anyhow!(
-                    "invalid param number for pagestream command"
-                )));
-            }
-            let tenant_id = TenantId::from_str(params[0])
-                .with_context(|| format!("Failed to parse tenant id from {}", params[0]))?;
-            let timeline_id = TimelineId::from_str(params[1])
-                .with_context(|| format!("Failed to parse timeline id from {}", params[1]))?;
+            ensure!(
+                params.len() == 2,
+                "invalid param number for pagestream command"
+            );
+            let tenant_id = TenantId::from_str(params[0])?;
+            let timeline_id = TimelineId::from_str(params[1])?;

            self.check_permission(Some(tenant_id))?;

@@ -740,24 +726,18 @@ impl postgres_backend_async::Handler for PageServerHandler {
            let (_, params_raw) = query_string.split_at("basebackup ".len());
            let params = params_raw.split_whitespace().collect::<Vec<_>>();

-            if params.len() < 2 {
-                return Err(QueryError::Other(anyhow::anyhow!(
-                    "invalid param number for basebackup command"
-                )));
-            }
+            ensure!(
+                params.len() >= 2,
+                "invalid param number for basebackup command"
+            );

-            let tenant_id = TenantId::from_str(params[0])
-                .with_context(|| format!("Failed to parse tenant id from {}", params[0]))?;
-            let timeline_id = TimelineId::from_str(params[1])
-                .with_context(|| format!("Failed to parse timeline id from {}", params[1]))?;
+            let tenant_id = TenantId::from_str(params[0])?;
+            let timeline_id = TimelineId::from_str(params[1])?;

            self.check_permission(Some(tenant_id))?;

            let lsn = if params.len() == 3 {
-                Some(
-                    Lsn::from_str(params[2])
-                        .with_context(|| format!("Failed to parse Lsn from {}", params[2]))?,
-                )
+                Some(Lsn::from_str(params[2])?)
            } else {
                None
            };
@@ -772,16 +752,13 @@ impl postgres_backend_async::Handler for PageServerHandler {
            let (_, params_raw) = query_string.split_at("get_last_record_rlsn ".len());
            let params = params_raw.split_whitespace().collect::<Vec<_>>();

-            if params.len() != 2 {
-                return Err(QueryError::Other(anyhow::anyhow!(
-                    "invalid param number for get_last_record_rlsn command"
-                )));
-            }
+            ensure!(
+                params.len() == 2,
+                "invalid param number for get_last_record_rlsn command"
+            );

-            let tenant_id = TenantId::from_str(params[0])
-                .with_context(|| format!("Failed to parse tenant id from {}", params[0]))?;
-            let timeline_id = TimelineId::from_str(params[1])
-                .with_context(|| format!("Failed to parse timeline id from {}", params[1]))?;
+            let tenant_id = TenantId::from_str(params[0])?;
+            let timeline_id = TimelineId::from_str(params[1])?;

            self.check_permission(Some(tenant_id))?;
            let timeline = get_active_timeline_with_timeout(tenant_id, timeline_id).await?;
@@ -803,31 +780,22 @@ impl postgres_backend_async::Handler for PageServerHandler {
            let (_, params_raw) = query_string.split_at("fullbackup ".len());
            let params = params_raw.split_whitespace().collect::<Vec<_>>();

-            if params.len() < 2 {
-                return Err(QueryError::Other(anyhow::anyhow!(
-                    "invalid param number for fullbackup command"
-                )));
-            }
+            ensure!(
+                params.len() >= 2,
+                "invalid param number for fullbackup command"
+            );

-            let tenant_id = TenantId::from_str(params[0])
-                .with_context(|| format!("Failed to parse tenant id from {}", params[0]))?;
-            let timeline_id = TimelineId::from_str(params[1])
-                .with_context(|| format!("Failed to parse timeline id from {}", params[1]))?;
+            let tenant_id = TenantId::from_str(params[0])?;
+            let timeline_id = TimelineId::from_str(params[1])?;

            // The caller is responsible for providing correct lsn and prev_lsn.
            let lsn = if params.len() > 2 {
-                Some(
-                    Lsn::from_str(params[2])
-                        .with_context(|| format!("Failed to parse Lsn from {}", params[2]))?,
-                )
+                Some(Lsn::from_str(params[2])?)
            } else {
                None
            };
            let prev_lsn = if params.len() > 3 {
-                Some(
-                    Lsn::from_str(params[3])
-                        .with_context(|| format!("Failed to parse Lsn from {}", params[3]))?,
-                )
+                Some(Lsn::from_str(params[3])?)
            } else {
                None
            };
@@ -852,21 +820,12 @@ impl postgres_backend_async::Handler for PageServerHandler {
            //     -c "import basebackup $TENANT $TIMELINE $START_LSN $END_LSN $PG_VERSION"
            let (_, params_raw) = query_string.split_at("import basebackup ".len());
            let params = params_raw.split_whitespace().collect::<Vec<_>>();
-            if params.len() != 5 {
-                return Err(QueryError::Other(anyhow::anyhow!(
-                    "invalid param number for import basebackup command"
-                )));
-            }
-            let tenant_id = TenantId::from_str(params[0])
-                .with_context(|| format!("Failed to parse tenant id from {}", params[0]))?;
-            let timeline_id = TimelineId::from_str(params[1])
-                .with_context(|| format!("Failed to parse timeline id from {}", params[1]))?;
-            let base_lsn = Lsn::from_str(params[2])
-                .with_context(|| format!("Failed to parse Lsn from {}", params[2]))?;
-            let end_lsn = Lsn::from_str(params[3])
-                .with_context(|| format!("Failed to parse Lsn from {}", params[3]))?;
-            let pg_version = u32::from_str(params[4])
-                .with_context(|| format!("Failed to parse pg_version from {}", params[4]))?;
+            ensure!(params.len() == 5);
+            let tenant_id = TenantId::from_str(params[0])?;
+            let timeline_id = TimelineId::from_str(params[1])?;
+            let base_lsn = Lsn::from_str(params[2])?;
+            let end_lsn = Lsn::from_str(params[3])?;
+            let pg_version = u32::from_str(params[4])?;

            self.check_permission(Some(tenant_id))?;

@@ -884,10 +843,7 @@ impl postgres_backend_async::Handler for PageServerHandler {
                Ok(()) => pgb.write_message(&BeMessage::CommandComplete(b"SELECT 1"))?,
                Err(e) => {
                    error!("error importing base backup between {base_lsn} and {end_lsn}: {e:?}");
-                    pgb.write_message(&BeMessage::ErrorResponse(
-                        &e.to_string(),
-                        Some(e.pg_error_code()),
-                    ))?
+                    pgb.write_message(&BeMessage::ErrorResponse(&e.to_string()))?
                }
            };
        } else if query_string.starts_with("import wal ") {
@@ -897,19 +853,11 @@ impl postgres_backend_async::Handler for PageServerHandler {
            // caller should poll the http api to check when that is done.
            let (_, params_raw) = query_string.split_at("import wal ".len());
            let params = params_raw.split_whitespace().collect::<Vec<_>>();
-            if params.len() != 4 {
-                return Err(QueryError::Other(anyhow::anyhow!(
-                    "invalid param number for import wal command"
-                )));
-            }
-            let tenant_id = TenantId::from_str(params[0])
-                .with_context(|| format!("Failed to parse tenant id from {}", params[0]))?;
-            let timeline_id = TimelineId::from_str(params[1])
-                .with_context(|| format!("Failed to parse timeline id from {}", params[1]))?;
-            let start_lsn = Lsn::from_str(params[2])
-                .with_context(|| format!("Failed to parse Lsn from {}", params[2]))?;
-            let end_lsn = Lsn::from_str(params[3])
-                .with_context(|| format!("Failed to parse Lsn from {}", params[3]))?;
+            ensure!(params.len() == 4);
+            let tenant_id = TenantId::from_str(params[0])?;
+            let timeline_id = TimelineId::from_str(params[1])?;
+            let start_lsn = Lsn::from_str(params[2])?;
+            let end_lsn = Lsn::from_str(params[3])?;

            self.check_permission(Some(tenant_id))?;

@@ -920,10 +868,7 @@ impl postgres_backend_async::Handler for PageServerHandler {
                Ok(()) => pgb.write_message(&BeMessage::CommandComplete(b"SELECT 1"))?,
                Err(e) => {
                    error!("error importing WAL between {start_lsn} and {end_lsn}: {e:?}");
-                    pgb.write_message(&BeMessage::ErrorResponse(
-                        &e.to_string(),
-                        Some(e.pg_error_code()),
-                    ))?
+                    pgb.write_message(&BeMessage::ErrorResponse(&e.to_string()))?
                }
            };
        } else if query_string.to_ascii_lowercase().starts_with("set ") {
@@ -934,13 +879,8 @@ impl postgres_backend_async::Handler for PageServerHandler {
            // show <tenant_id>
            let (_, params_raw) = query_string.split_at("show ".len());
            let params = params_raw.split(' ').collect::<Vec<_>>();
-            if params.len() != 1 {
-                return Err(QueryError::Other(anyhow::anyhow!(
-                    "invalid param number for config command"
-                )));
-            }
-            let tenant_id = TenantId::from_str(params[0])
-                .with_context(|| format!("Failed to parse tenant id from {}", params[0]))?;
+            ensure!(params.len() == 1, "invalid param number for config command");
+            let tenant_id = TenantId::from_str(params[0])?;

            self.check_permission(Some(tenant_id))?;

@@ -981,9 +921,7 @@ impl postgres_backend_async::Handler for PageServerHandler {
            ]))?
            .write_message(&BeMessage::CommandComplete(b"SELECT 1"))?;
        } else {
-            return Err(QueryError::Other(anyhow::anyhow!(
-                "unknown command {query_string}"
-            )));
+            bail!("unknown command");
        }

        Ok(())
@@ -995,7 +933,7 @@ impl postgres_backend_async::Handler for PageServerHandler {
 /// If the tenant is Loading, waits for it to become Active, for up to 30 s. That
 /// ensures that queries don't fail immediately after pageserver startup, because
 /// all tenants are still loading.
-async fn get_active_tenant_with_timeout(tenant_id: TenantId) -> anyhow::Result<Arc<Tenant>> {
+async fn get_active_tenant_with_timeout(tenant_id: TenantId) -> Result<Arc<Tenant>> {
    let tenant = mgr::get_tenant(tenant_id, false).await?;
    match tokio::time::timeout(Duration::from_secs(30), tenant.wait_to_become_active()).await {
        Ok(wait_result) => wait_result
@@ -1009,7 +947,7 @@ async fn get_active_tenant_with_timeout(tenant_id: TenantId) -> anyhow::Result<A
 async fn get_active_timeline_with_timeout(
    tenant_id: TenantId,
    timeline_id: TimelineId,
-) -> anyhow::Result<Arc<Timeline>> {
+) -> Result<Arc<Timeline>> {
    get_active_tenant_with_timeout(tenant_id)
        .await
        .and_then(|tenant| tenant.get_timeline(timeline_id, true))
@@ -6,10 +6,11 @@
 //! walingest.rs handles a few things like implicit relation creation and extension.
 //! Clarify that)
 //!
-use super::tenant::{PageReconstructError, Timeline};
+use super::tenant::PageReconstructResult;
 use crate::keyspace::{KeySpace, KeySpaceAccum};
-use crate::repository::*;
+use crate::tenant::{with_ondemand_download, Timeline};
 use crate::walrecord::NeonWalRecord;
+use crate::{repository::*, try_no_ondemand_download};
 use anyhow::Context;
 use bytes::{Buf, Bytes};
 use pageserver_api::reltag::{RelTag, SlruKind};
@@ -91,80 +92,76 @@ impl Timeline {
    //------------------------------------------------------------------------------

    /// Look up given page version.
-    pub async fn get_rel_page_at_lsn(
+    pub fn get_rel_page_at_lsn(
        &self,
        tag: RelTag,
        blknum: BlockNumber,
        lsn: Lsn,
        latest: bool,
-    ) -> Result<Bytes, PageReconstructError> {
+    ) -> PageReconstructResult<Bytes> {
        if tag.relnode == 0 {
-            return Err(PageReconstructError::Other(anyhow::anyhow!(
-                "invalid relnode"
-            )));
+            return PageReconstructResult::from(anyhow::anyhow!("invalid relnode"));
        }

-        let nblocks = self.get_rel_size(tag, lsn, latest).await?;
+        let nblocks = try_no_ondemand_download!(self.get_rel_size(tag, lsn, latest));
        if blknum >= nblocks {
            debug!(
                "read beyond EOF at {} blk {} at {}, size is {}: returning all-zeros page",
                tag, blknum, lsn, nblocks
            );
-            return Ok(ZERO_PAGE.clone());
+            return PageReconstructResult::Success(ZERO_PAGE.clone());
        }

        let key = rel_block_to_key(tag, blknum);
-        self.get(key, lsn).await
+        self.get(key, lsn)
    }

    // Get size of a database in blocks
-    pub async fn get_db_size(
+    pub fn get_db_size(
        &self,
        spcnode: Oid,
        dbnode: Oid,
        lsn: Lsn,
        latest: bool,
-    ) -> Result<usize, PageReconstructError> {
+    ) -> PageReconstructResult<usize> {
        let mut total_blocks = 0;

-        let rels = self.list_rels(spcnode, dbnode, lsn).await?;
+        let rels = try_no_ondemand_download!(self.list_rels(spcnode, dbnode, lsn));

        for rel in rels {
-            let n_blocks = self.get_rel_size(rel, lsn, latest).await?;
+            let n_blocks = try_no_ondemand_download!(self.get_rel_size(rel, lsn, latest));
            total_blocks += n_blocks as usize;
        }
-        Ok(total_blocks)
+        PageReconstructResult::Success(total_blocks)
    }

    /// Get size of a relation file
-    pub async fn get_rel_size(
+    pub fn get_rel_size(
        &self,
        tag: RelTag,
        lsn: Lsn,
        latest: bool,
-    ) -> Result<BlockNumber, PageReconstructError> {
+    ) -> PageReconstructResult<BlockNumber> {
        if tag.relnode == 0 {
-            return Err(PageReconstructError::Other(anyhow::anyhow!(
-                "invalid relnode"
-            )));
+            return PageReconstructResult::from(anyhow::anyhow!("invalid relnode"));
        }

        if let Some(nblocks) = self.get_cached_rel_size(&tag, lsn) {
-            return Ok(nblocks);
+            return PageReconstructResult::Success(nblocks);
        }

        if (tag.forknum == FSM_FORKNUM || tag.forknum == VISIBILITYMAP_FORKNUM)
-            && !self.get_rel_exists(tag, lsn, latest).await?
+            && !try_no_ondemand_download!(self.get_rel_exists(tag, lsn, latest))
        {
            // FIXME: Postgres sometimes calls smgrcreate() to create
            // FSM, and smgrnblocks() on it immediately afterwards,
            // without extending it.  Tolerate that by claiming that
            // any non-existent FSM fork has size 0.
-            return Ok(0);
+            return PageReconstructResult::Success(0);
        }

        let key = rel_size_to_key(tag);
-        let mut buf = self.get(key, lsn).await?;
+        let mut buf = try_no_ondemand_download!(self.get(key, lsn));
        let nblocks = buf.get_u32_le();

        if latest {
@@ -177,49 +174,47 @@ impl Timeline {
            // associated with most recent value of LSN.
            self.update_cached_rel_size(tag, lsn, nblocks);
        }
-        Ok(nblocks)
+        PageReconstructResult::Success(nblocks)
    }

    /// Does relation exist?
-    pub async fn get_rel_exists(
+    pub fn get_rel_exists(
        &self,
        tag: RelTag,
        lsn: Lsn,
        _latest: bool,
-    ) -> Result<bool, PageReconstructError> {
+    ) -> PageReconstructResult<bool> {
        if tag.relnode == 0 {
-            return Err(PageReconstructError::Other(anyhow::anyhow!(
-                "invalid relnode"
-            )));
+            return PageReconstructResult::from(anyhow::anyhow!("invalid relnode"));
        }

        // first try to lookup relation in cache
        if let Some(_nblocks) = self.get_cached_rel_size(&tag, lsn) {
-            return Ok(true);
+            return PageReconstructResult::Success(true);
        }
        // fetch directory listing
        let key = rel_dir_to_key(tag.spcnode, tag.dbnode);
-        let buf = self.get(key, lsn).await?;
+        let buf = try_no_ondemand_download!(self.get(key, lsn));

        match RelDirectory::des(&buf).context("deserialization failure") {
            Ok(dir) => {
                let exists = dir.rels.get(&(tag.relnode, tag.forknum)).is_some();
-                Ok(exists)
+                PageReconstructResult::Success(exists)
            }
-            Err(e) => Err(PageReconstructError::from(e)),
+            Err(e) => PageReconstructResult::from(e),
        }
    }

    /// Get a list of all existing relations in given tablespace and database.
-    pub async fn list_rels(
+    pub fn list_rels(
        &self,
        spcnode: Oid,
        dbnode: Oid,
        lsn: Lsn,
-    ) -> Result<HashSet<RelTag>, PageReconstructError> {
+    ) -> PageReconstructResult<HashSet<RelTag>> {
        // fetch directory listing
        let key = rel_dir_to_key(spcnode, dbnode);
-        let buf = self.get(key, lsn).await?;
+        let buf = try_no_ondemand_download!(self.get(key, lsn));

        match RelDirectory::des(&buf).context("deserialization failure") {
            Ok(dir) => {
@@ -231,53 +226,53 @@ impl Timeline {
                        forknum: *forknum,
                    }));

-                Ok(rels)
+                PageReconstructResult::Success(rels)
            }
-            Err(e) => Err(PageReconstructError::from(e)),
+            Err(e) => PageReconstructResult::from(e),
        }
    }

    /// Look up given SLRU page version.
-    pub async fn get_slru_page_at_lsn(
+    pub fn get_slru_page_at_lsn(
        &self,
        kind: SlruKind,
        segno: u32,
        blknum: BlockNumber,
        lsn: Lsn,
-    ) -> Result<Bytes, PageReconstructError> {
+    ) -> PageReconstructResult<Bytes> {
        let key = slru_block_to_key(kind, segno, blknum);
-        self.get(key, lsn).await
+        self.get(key, lsn)
    }

    /// Get size of an SLRU segment
-    pub async fn get_slru_segment_size(
+    pub fn get_slru_segment_size(
        &self,
        kind: SlruKind,
        segno: u32,
        lsn: Lsn,
-    ) -> Result<BlockNumber, PageReconstructError> {
+    ) -> PageReconstructResult<BlockNumber> {
        let key = slru_segment_size_to_key(kind, segno);
-        let mut buf = self.get(key, lsn).await?;
-        Ok(buf.get_u32_le())
+        let mut buf = try_no_ondemand_download!(self.get(key, lsn));
+        PageReconstructResult::Success(buf.get_u32_le())
    }

    /// Get size of an SLRU segment
-    pub async fn get_slru_segment_exists(
+    pub fn get_slru_segment_exists(
        &self,
        kind: SlruKind,
        segno: u32,
        lsn: Lsn,
-    ) -> Result<bool, PageReconstructError> {
+    ) -> PageReconstructResult<bool> {
        // fetch directory listing
        let key = slru_dir_to_key(kind);
-        let buf = self.get(key, lsn).await?;
+        let buf = try_no_ondemand_download!(self.get(key, lsn));

        match SlruSegmentDirectory::des(&buf).context("deserialization failure") {
            Ok(dir) => {
                let exists = dir.segments.get(&segno).is_some();
-                Ok(exists)
+                PageReconstructResult::Success(exists)
            }
-            Err(e) => Err(PageReconstructError::from(e)),
+            Err(e) => PageReconstructResult::from(e),
        }
    }

@@ -288,10 +283,10 @@ impl Timeline {
    /// so it's not well defined which LSN you get if there were multiple commits
    /// "in flight" at that point in time.
    ///
-    pub async fn find_lsn_for_timestamp(
+    pub fn find_lsn_for_timestamp(
        &self,
        search_timestamp: TimestampTz,
-    ) -> Result<LsnForTimestamp, PageReconstructError> {
+    ) -> PageReconstructResult<LsnForTimestamp> {
        let gc_cutoff_lsn_guard = self.get_latest_gc_cutoff_lsn();
        let min_lsn = *gc_cutoff_lsn_guard;
        let max_lsn = self.get_last_record_lsn();
@@ -307,14 +302,12 @@ impl Timeline {
            // cannot overflow, high and low are both smaller than u64::MAX / 2
            let mid = (high + low) / 2;

-            let cmp = self
-                .is_latest_commit_timestamp_ge_than(
-                    search_timestamp,
-                    Lsn(mid * 8),
-                    &mut found_smaller,
-                    &mut found_larger,
-                )
-                .await?;
+            let cmp = try_no_ondemand_download!(self.is_latest_commit_timestamp_ge_than(
+                search_timestamp,
+                Lsn(mid * 8),
+                &mut found_smaller,
+                &mut found_larger,
+            ));

            if cmp {
                high = mid;
@@ -326,15 +319,15 @@ impl Timeline {
            (false, false) => {
                // This can happen if no commit records have been processed yet, e.g.
                // just after importing a cluster.
-                Ok(LsnForTimestamp::NoData(max_lsn))
+                PageReconstructResult::Success(LsnForTimestamp::NoData(max_lsn))
            }
            (true, false) => {
                // Didn't find any commit timestamps larger than the request
-                Ok(LsnForTimestamp::Future(max_lsn))
+                PageReconstructResult::Success(LsnForTimestamp::Future(max_lsn))
            }
            (false, true) => {
                // Didn't find any commit timestamps smaller than the request
-                Ok(LsnForTimestamp::Past(max_lsn))
+                PageReconstructResult::Success(LsnForTimestamp::Past(max_lsn))
            }
            (true, true) => {
                // low is the LSN of the first commit record *after* the search_timestamp,
@@ -344,7 +337,7 @@ impl Timeline {
                // Otherwise, if you restore to the returned LSN, the database will
                // include physical changes from later commits that will be marked
                // as aborted, and will need to be vacuumed away.
-                Ok(LsnForTimestamp::Present(Lsn((low - 1) * 8)))
+                PageReconstructResult::Success(LsnForTimestamp::Present(Lsn((low - 1) * 8)))
            }
        }
    }
@@ -356,21 +349,26 @@ impl Timeline {
    /// Additionally, sets 'found_smaller'/'found_Larger, if encounters any commits
    /// with a smaller/larger timestamp.
    ///
-    pub async fn is_latest_commit_timestamp_ge_than(
+    pub fn is_latest_commit_timestamp_ge_than(
        &self,
        search_timestamp: TimestampTz,
        probe_lsn: Lsn,
        found_smaller: &mut bool,
        found_larger: &mut bool,
-    ) -> Result<bool, PageReconstructError> {
-        for segno in self.list_slru_segments(SlruKind::Clog, probe_lsn).await? {
-            let nblocks = self
-                .get_slru_segment_size(SlruKind::Clog, segno, probe_lsn)
-                .await?;
+    ) -> PageReconstructResult<bool> {
+        for segno in try_no_ondemand_download!(self.list_slru_segments(SlruKind::Clog, probe_lsn)) {
+            let nblocks = try_no_ondemand_download!(self.get_slru_segment_size(
+                SlruKind::Clog,
+                segno,
+                probe_lsn
+            ));
            for blknum in (0..nblocks).rev() {
-                let clog_page = self
-                    .get_slru_page_at_lsn(SlruKind::Clog, segno, blknum, probe_lsn)
-                    .await?;
+                let clog_page = try_no_ondemand_download!(self.get_slru_page_at_lsn(
+                    SlruKind::Clog,
+                    segno,
+                    blknum,
+                    probe_lsn
+                ));

                if clog_page.len() == BLCKSZ as usize + 8 {
                    let mut timestamp_bytes = [0u8; 8];
@@ -379,85 +377,76 @@ impl Timeline {

                    if timestamp >= search_timestamp {
                        *found_larger = true;
-                        return Ok(true);
+                        return PageReconstructResult::Success(true);
                    } else {
                        *found_smaller = true;
                    }
                }
            }
        }
-        Ok(false)
+        PageReconstructResult::Success(false)
    }

    /// Get a list of SLRU segments
-    pub async fn list_slru_segments(
+    pub fn list_slru_segments(
        &self,
        kind: SlruKind,
        lsn: Lsn,
-    ) -> Result<HashSet<u32>, PageReconstructError> {
+    ) -> PageReconstructResult<HashSet<u32>> {
        // fetch directory entry
        let key = slru_dir_to_key(kind);

-        let buf = self.get(key, lsn).await?;
+        let buf = try_no_ondemand_download!(self.get(key, lsn));
        match SlruSegmentDirectory::des(&buf).context("deserialization failure") {
-            Ok(dir) => Ok(dir.segments),
-            Err(e) => Err(PageReconstructError::from(e)),
+            Ok(dir) => PageReconstructResult::Success(dir.segments),
+            Err(e) => PageReconstructResult::from(e),
        }
    }

-    pub async fn get_relmap_file(
+    pub fn get_relmap_file(
        &self,
        spcnode: Oid,
        dbnode: Oid,
        lsn: Lsn,
-    ) -> Result<Bytes, PageReconstructError> {
+    ) -> PageReconstructResult<Bytes> {
        let key = relmap_file_key(spcnode, dbnode);

-        self.get(key, lsn).await
+        let buf = try_no_ondemand_download!(self.get(key, lsn));
+        PageReconstructResult::Success(buf)
    }

-    pub async fn list_dbdirs(
-        &self,
-        lsn: Lsn,
-    ) -> Result<HashMap<(Oid, Oid), bool>, PageReconstructError> {
+    pub fn list_dbdirs(&self, lsn: Lsn) -> PageReconstructResult<HashMap<(Oid, Oid), bool>> {
        // fetch directory entry
-        let buf = self.get(DBDIR_KEY, lsn).await?;
+        let buf = try_no_ondemand_download!(self.get(DBDIR_KEY, lsn));

        match DbDirectory::des(&buf).context("deserialization failure") {
-            Ok(dir) => Ok(dir.dbdirs),
-            Err(e) => Err(PageReconstructError::from(e)),
+            Ok(dir) => PageReconstructResult::Success(dir.dbdirs),
+            Err(e) => PageReconstructResult::from(e),
        }
    }

-    pub async fn get_twophase_file(
-        &self,
-        xid: TransactionId,
-        lsn: Lsn,
-    ) -> Result<Bytes, PageReconstructError> {
+    pub fn get_twophase_file(&self, xid: TransactionId, lsn: Lsn) -> PageReconstructResult<Bytes> {
        let key = twophase_file_key(xid);
-        let buf = self.get(key, lsn).await?;
-        Ok(buf)
+        let buf = try_no_ondemand_download!(self.get(key, lsn));
+        PageReconstructResult::Success(buf)
    }

-    pub async fn list_twophase_files(
-        &self,
-        lsn: Lsn,
-    ) -> Result<HashSet<TransactionId>, PageReconstructError> {
+    pub fn list_twophase_files(&self, lsn: Lsn) -> PageReconstructResult<HashSet<TransactionId>> {
        // fetch directory entry
-        let buf = self.get(TWOPHASEDIR_KEY, lsn).await?;
+        let buf = try_no_ondemand_download!(self.get(TWOPHASEDIR_KEY, lsn));

        match TwoPhaseDirectory::des(&buf).context("deserialization failure") {
-            Ok(dir) => Ok(dir.xids),
-            Err(e) => Err(PageReconstructError::from(e)),
+            Ok(dir) => PageReconstructResult::Success(dir.xids),
+            Err(e) => PageReconstructResult::from(e),
        }
    }

-    pub async fn get_control_file(&self, lsn: Lsn) -> Result<Bytes, PageReconstructError> {
-        self.get(CONTROLFILE_KEY, lsn).await
+    pub fn get_control_file(&self, lsn: Lsn) -> PageReconstructResult<Bytes> {
+        self.get(CONTROLFILE_KEY, lsn)
    }

-    pub async fn get_checkpoint(&self, lsn: Lsn) -> Result<Bytes, PageReconstructError> {
-        self.get(CHECKPOINT_KEY, lsn).await
+    pub fn get_checkpoint(&self, lsn: Lsn) -> PageReconstructResult<Bytes> {
+        self.get(CHECKPOINT_KEY, lsn)
    }

    /// Does the same as get_current_logical_size but counted on demand.
@@ -471,24 +460,20 @@ impl Timeline {
        cancel: CancellationToken,
    ) -> Result<u64, CalculateLogicalSizeError> {
        // Fetch list of database dirs and iterate them
-        let buf = self.get(DBDIR_KEY, lsn).await.context("read dbdir")?;
+        let buf = self.get_download(DBDIR_KEY, lsn).await?;
        let dbdir = DbDirectory::des(&buf).context("deserialize db directory")?;

        let mut total_size: u64 = 0;
        for (spcnode, dbnode) in dbdir.dbdirs.keys() {
-            for rel in self
-                .list_rels(*spcnode, *dbnode, lsn)
-                .await
-                .context("list rels")?
+            for rel in
+                crate::tenant::with_ondemand_download(|| self.list_rels(*spcnode, *dbnode, lsn))
+                    .await?
            {
                if cancel.is_cancelled() {
                    return Err(CalculateLogicalSizeError::Cancelled);
                }
                let relsize_key = rel_size_to_key(rel);
-                let mut buf = self
-                    .get(relsize_key, lsn)
-                    .await
-                    .with_context(|| format!("read relation size of {rel:?}"))?;
+                let mut buf = self.get_download(relsize_key, lsn).await?;
                let relsize = buf.get_u32_le();

                total_size += relsize as u64;
@@ -509,7 +494,7 @@ impl Timeline {
        result.add_key(DBDIR_KEY);

        // Fetch list of database dirs and iterate them
-        let buf = self.get(DBDIR_KEY, lsn).await?;
+        let buf = self.get_download(DBDIR_KEY, lsn).await?;
        let dbdir = DbDirectory::des(&buf).context("deserialization failure")?;

        let mut dbs: Vec<(Oid, Oid)> = dbdir.dbdirs.keys().cloned().collect();
@@ -518,15 +503,15 @@ impl Timeline {
            result.add_key(relmap_file_key(spcnode, dbnode));
            result.add_key(rel_dir_to_key(spcnode, dbnode));

-            let mut rels: Vec<RelTag> = self
-                .list_rels(spcnode, dbnode, lsn)
-                .await?
-                .into_iter()
-                .collect();
+            let mut rels: Vec<RelTag> =
+                with_ondemand_download(|| self.list_rels(spcnode, dbnode, lsn))
+                    .await?
+                    .into_iter()
+                    .collect();
            rels.sort_unstable();
            for rel in rels {
                let relsize_key = rel_size_to_key(rel);
-                let mut buf = self.get(relsize_key, lsn).await?;
+                let mut buf = self.get_download(relsize_key, lsn).await?;
                let relsize = buf.get_u32_le();

                result.add_range(rel_block_to_key(rel, 0)..rel_block_to_key(rel, relsize));
@@ -542,13 +527,13 @@ impl Timeline {
        ] {
            let slrudir_key = slru_dir_to_key(kind);
            result.add_key(slrudir_key);
-            let buf = self.get(slrudir_key, lsn).await?;
+            let buf = self.get_download(slrudir_key, lsn).await?;
            let dir = SlruSegmentDirectory::des(&buf).context("deserialization failure")?;
            let mut segments: Vec<u32> = dir.segments.iter().cloned().collect();
            segments.sort_unstable();
            for segno in segments {
                let segsize_key = slru_segment_size_to_key(kind, segno);
-                let mut buf = self.get(segsize_key, lsn).await?;
+                let mut buf = self.get_download(segsize_key, lsn).await?;
                let segsize = buf.get_u32_le();

                result.add_range(
@@ -560,7 +545,7 @@ impl Timeline {

        // Then pg_twophase
        result.add_key(TWOPHASEDIR_KEY);
-        let buf = self.get(TWOPHASEDIR_KEY, lsn).await?;
+        let buf = self.get_download(TWOPHASEDIR_KEY, lsn).await?;
        let twophase_dir = TwoPhaseDirectory::des(&buf).context("deserialization failure")?;
        let mut xids: Vec<TransactionId> = twophase_dir.xids.iter().cloned().collect();
        xids.sort_unstable();
@@ -718,14 +703,9 @@ impl<'a> DatadirModification<'a> {
    }

    /// Store a relmapper file (pg_filenode.map) in the repository
-    pub async fn put_relmap_file(
-        &mut self,
-        spcnode: Oid,
-        dbnode: Oid,
-        img: Bytes,
-    ) -> anyhow::Result<()> {
+    pub fn put_relmap_file(&mut self, spcnode: Oid, dbnode: Oid, img: Bytes) -> anyhow::Result<()> {
        // Add it to the directory (if it doesn't exist already)
-        let buf = self.get(DBDIR_KEY).await?;
+        let buf = self.get(DBDIR_KEY).no_ondemand_download()?;
        let mut dbdir = DbDirectory::des(&buf)?;

        let r = dbdir.dbdirs.insert((spcnode, dbnode), true);
@@ -751,13 +731,9 @@ impl<'a> DatadirModification<'a> {
        Ok(())
    }

-    pub async fn put_twophase_file(
-        &mut self,
-        xid: TransactionId,
-        img: Bytes,
-    ) -> anyhow::Result<()> {
+    pub fn put_twophase_file(&mut self, xid: TransactionId, img: Bytes) -> anyhow::Result<()> {
        // Add it to the directory entry
-        let buf = self.get(TWOPHASEDIR_KEY).await?;
+        let buf = self.get(TWOPHASEDIR_KEY).no_ondemand_download()?;
        let mut dir = TwoPhaseDirectory::des(&buf)?;
        if !dir.xids.insert(xid) {
            anyhow::bail!("twophase file for xid {} already exists", xid);
@@ -781,16 +757,16 @@ impl<'a> DatadirModification<'a> {
        Ok(())
    }

-    pub async fn drop_dbdir(&mut self, spcnode: Oid, dbnode: Oid) -> anyhow::Result<()> {
+    pub fn drop_dbdir(&mut self, spcnode: Oid, dbnode: Oid) -> anyhow::Result<()> {
        let req_lsn = self.tline.get_last_record_lsn();

        let total_blocks = self
            .tline
            .get_db_size(spcnode, dbnode, req_lsn, true)
-            .await?;
+            .no_ondemand_download()?;

        // Remove entry from dbdir
-        let buf = self.get(DBDIR_KEY).await?;
+        let buf = self.get(DBDIR_KEY).no_ondemand_download()?;
        let mut dir = DbDirectory::des(&buf)?;
        if dir.dbdirs.remove(&(spcnode, dbnode)).is_some() {
            let buf = DbDirectory::ser(&dir)?;
@@ -813,15 +789,11 @@ impl<'a> DatadirModification<'a> {
    /// Create a relation fork.
    ///
    /// 'nblocks' is the initial size.
-    pub async fn put_rel_creation(
-        &mut self,
-        rel: RelTag,
-        nblocks: BlockNumber,
-    ) -> anyhow::Result<()> {
+    pub fn put_rel_creation(&mut self, rel: RelTag, nblocks: BlockNumber) -> anyhow::Result<()> {
        anyhow::ensure!(rel.relnode != 0, "invalid relnode");
        // It's possible that this is the first rel for this db in this
        // tablespace.  Create the reldir entry for it if so.
-        let mut dbdir = DbDirectory::des(&self.get(DBDIR_KEY).await?)?;
+        let mut dbdir = DbDirectory::des(&self.get(DBDIR_KEY).no_ondemand_download()?)?;
        let rel_dir_key = rel_dir_to_key(rel.spcnode, rel.dbnode);
        let mut rel_dir = if dbdir.dbdirs.get(&(rel.spcnode, rel.dbnode)).is_none() {
            // Didn't exist. Update dbdir
@@ -833,7 +805,7 @@ impl<'a> DatadirModification<'a> {
            RelDirectory::default()
        } else {
            // reldir already exists, fetch it
-            RelDirectory::des(&self.get(rel_dir_key).await?)?
+            RelDirectory::des(&self.get(rel_dir_key).no_ondemand_download()?)?
        };

        // Add the new relation to the rel directory entry, and write it back
@@ -861,17 +833,17 @@ impl<'a> DatadirModification<'a> {
    }

    /// Truncate relation
-    pub async fn put_rel_truncation(
-        &mut self,
-        rel: RelTag,
-        nblocks: BlockNumber,
-    ) -> anyhow::Result<()> {
+    pub fn put_rel_truncation(&mut self, rel: RelTag, nblocks: BlockNumber) -> anyhow::Result<()> {
        anyhow::ensure!(rel.relnode != 0, "invalid relnode");
        let last_lsn = self.tline.get_last_record_lsn();
-        if self.tline.get_rel_exists(rel, last_lsn, true).await? {
+        if self
+            .tline
+            .get_rel_exists(rel, last_lsn, true)
+            .no_ondemand_download()?
+        {
            let size_key = rel_size_to_key(rel);
            // Fetch the old size first
-            let old_size = self.get(size_key).await?.get_u32_le();
+            let old_size = self.get(size_key).no_ondemand_download()?.get_u32_le();

            // Update the entry with the new size.
            let buf = nblocks.to_le_bytes();
@@ -891,16 +863,12 @@ impl<'a> DatadirModification<'a> {

    /// Extend relation
    /// If new size is smaller, do nothing.
-    pub async fn put_rel_extend(
-        &mut self,
-        rel: RelTag,
-        nblocks: BlockNumber,
-    ) -> anyhow::Result<()> {
+    pub fn put_rel_extend(&mut self, rel: RelTag, nblocks: BlockNumber) -> anyhow::Result<()> {
        anyhow::ensure!(rel.relnode != 0, "invalid relnode");

        // Put size
        let size_key = rel_size_to_key(rel);
-        let old_size = self.get(size_key).await?.get_u32_le();
+        let old_size = self.get(size_key).no_ondemand_download()?.get_u32_le();

        // only extend relation here. never decrease the size
        if nblocks > old_size {
@@ -916,12 +884,12 @@ impl<'a> DatadirModification<'a> {
    }

    /// Drop a relation.
-    pub async fn put_rel_drop(&mut self, rel: RelTag) -> anyhow::Result<()> {
+    pub fn put_rel_drop(&mut self, rel: RelTag) -> anyhow::Result<()> {
        anyhow::ensure!(rel.relnode != 0, "invalid relnode");

        // Remove it from the directory entry
        let dir_key = rel_dir_to_key(rel.spcnode, rel.dbnode);
-        let buf = self.get(dir_key).await?;
+        let buf = self.get(dir_key).no_ondemand_download()?;
        let mut dir = RelDirectory::des(&buf)?;

        if dir.rels.remove(&(rel.relnode, rel.forknum)) {
@@ -932,7 +900,7 @@ impl<'a> DatadirModification<'a> {

        // update logical size
        let size_key = rel_size_to_key(rel);
-        let old_size = self.get(size_key).await?.get_u32_le();
+        let old_size = self.get(size_key).no_ondemand_download()?.get_u32_le();
        self.pending_nblocks -= old_size as i64;

        // Remove enty from relation size cache
@@ -944,7 +912,7 @@ impl<'a> DatadirModification<'a> {
        Ok(())
    }

-    pub async fn put_slru_segment_creation(
+    pub fn put_slru_segment_creation(
        &mut self,
        kind: SlruKind,
        segno: u32,
@@ -952,7 +920,7 @@ impl<'a> DatadirModification<'a> {
    ) -> anyhow::Result<()> {
        // Add it to the directory entry
        let dir_key = slru_dir_to_key(kind);
-        let buf = self.get(dir_key).await?;
+        let buf = self.get(dir_key).no_ondemand_download()?;
        let mut dir = SlruSegmentDirectory::des(&buf)?;

        if !dir.segments.insert(segno) {
@@ -988,10 +956,10 @@ impl<'a> DatadirModification<'a> {
    }

    /// This method is used for marking truncated SLRU files
-    pub async fn drop_slru_segment(&mut self, kind: SlruKind, segno: u32) -> anyhow::Result<()> {
+    pub fn drop_slru_segment(&mut self, kind: SlruKind, segno: u32) -> anyhow::Result<()> {
        // Remove it from the directory entry
        let dir_key = slru_dir_to_key(kind);
-        let buf = self.get(dir_key).await?;
+        let buf = self.get(dir_key).no_ondemand_download()?;
        let mut dir = SlruSegmentDirectory::des(&buf)?;

        if !dir.segments.remove(&segno) {
@@ -1015,9 +983,9 @@ impl<'a> DatadirModification<'a> {
    }

    /// This method is used for marking truncated SLRU files
-    pub async fn drop_twophase_file(&mut self, xid: TransactionId) -> anyhow::Result<()> {
+    pub fn drop_twophase_file(&mut self, xid: TransactionId) -> anyhow::Result<()> {
        // Remove it from the directory entry
-        let buf = self.get(TWOPHASEDIR_KEY).await?;
+        let buf = self.get(TWOPHASEDIR_KEY).no_ondemand_download()?;
        let mut dir = TwoPhaseDirectory::des(&buf)?;

        if !dir.xids.remove(&xid) {
@@ -1111,7 +1079,7 @@ impl<'a> DatadirModification<'a> {

    // Internal helper functions to batch the modifications

-    async fn get(&self, key: Key) -> Result<Bytes, PageReconstructError> {
+    fn get(&self, key: Key) -> PageReconstructResult<Bytes> {
        // Have we already updated the same key? Read the pending updated
        // version in that case.
        //
@@ -1119,20 +1087,18 @@ impl<'a> DatadirModification<'a> {
        // value that has been removed, deletion only avoids leaking storage.
        if let Some(value) = self.pending_updates.get(&key) {
            if let Value::Image(img) = value {
-                Ok(img.clone())
+                PageReconstructResult::Success(img.clone())
            } else {
                // Currently, we never need to read back a WAL record that we
                // inserted in the same "transaction". All the metadata updates
                // work directly with Images, and we never need to read actual
                // data pages. We could handle this if we had to, by calling
                // the walredo manager, but let's keep it simple for now.
-                Err(PageReconstructError::from(anyhow::anyhow!(
-                    "unexpected pending WAL record"
-                )))
+                PageReconstructResult::from(anyhow::anyhow!("unexpected pending WAL record"))
            }
        } else {
            let lsn = Lsn::max(self.tline.get_last_record_lsn(), self.lsn);
-            self.tline.get(key, lsn).await
+            self.tline.get(key, lsn)
        }
    }

@@ -1405,15 +1371,15 @@ fn slru_segment_key_range(kind: SlruKind, segno: u32) -> Range<Key> {
    Key {
        field1: 0x01,
        field2,
-        field3: 1,
-        field4: segno,
+        field3: segno,
+        field4: 0,
        field5: 0,
        field6: 0,
    }..Key {
        field1: 0x01,
        field2,
-        field3: 1,
-        field4: segno,
+        field3: segno,
+        field4: 0,
        field5: 1,
        field6: 0,
    }
@@ -37,17 +37,6 @@ impl Key {
            | self.field6 as i128
    }

-    pub fn from_i128(x: i128) -> Self {
-        Key {
-            field1: ((x >> 120) & 0xf) as u8,
-            field2: ((x >> 104) & 0xFFFF) as u32,
-            field3: (x >> 72) as u32,
-            field4: (x >> 40) as u32,
-            field5: (x >> 32) as u8,
-            field6: x as u32,
-        }
-    }
-
    pub fn next(&self) -> Key {
        self.add(1)
    }
@@ -183,29 +183,12 @@ pub enum TaskKind {
    // associated with one later, after receiving a command from the client.
    PageRequestHandler,

-    /// Manages the WAL receiver connection for one timeline.
-    /// It subscribes to events from storage_broker and decides which safekeeper to connect to.
-    /// Once the decision has been made, it establishes the connection using the `tokio-postgres` library.
-    /// There is at most one connection at any given time.
-    ///
-    /// That `tokio-postgres` library represents a connection as two objects: a `Client` and a `Connection`.
-    /// The `Client` object is what library users use to make requests & get responses.
-    /// Internally, `Client` hands over requests to the `Connection` object.
-    /// The `Connection` object is responsible for speaking the wire protocol.
-    ///
-    /// Walreceiver uses its own abstraction called `TaskHandle` to represent the activity of establishing and handling a connection.
-    /// That abstraction doesn't use `task_mgr` and hence, has no `TaskKind`.
-    /// The [`WalReceiverManager`] task ensures that this `TaskHandle` task does not outlive the [`WalReceiverManager`] task.
-    ///
-    /// Once the connection is established, the `TaskHandle` task creates a
-    /// [`WalReceiverConnection`] task_mgr task that is responsible for polling
-    /// the `Connection` object.
-    /// A `CancellationToken` created by the `TaskHandle` task ensures
-    /// that the [`WalReceiverConnection`] task will cancel soon after as the `TaskHandle` is dropped.
+    // Manages the WAL receiver connection for one timeline. It subscribes to
+    // events from storage_broker, decides which safekeeper to connect to. It spawns a
+    // separate WalReceiverConnection task to handle each connection.
    WalReceiverManager,

-    /// The task that polls the `tokio-postgres::Connection` object.
-    /// See the comment on [`WalReceiverManager`].
+    // Handles a connection to a safekeeper, to stream WAL to a timeline.
    WalReceiverConnection,

    // Garbage collection worker. One per tenant
@@ -237,8 +220,6 @@ pub enum TaskKind {

    // task that drives downloading layers
    DownloadAllRemoteLayers,
-    // Task that calculates synthetis size for all active tenants
-    CalculateSyntheticSize,
 }

 #[derive(Default)]
@@ -13,13 +13,11 @@

 use anyhow::{bail, Context};
 use bytes::Bytes;
-use futures::FutureExt;
 use futures::Stream;
 use pageserver_api::models::TimelineState;
 use remote_storage::DownloadError;
 use remote_storage::GenericRemoteStorage;
 use tokio::sync::watch;
-use tokio::task::JoinSet;
 use tracing::*;
 use utils::crashsafe::path_with_suffix_extension;

@@ -38,8 +36,6 @@ use std::path::Path;
 use std::path::PathBuf;
 use std::process::Command;
 use std::process::Stdio;
-use std::sync::atomic::AtomicU64;
-use std::sync::atomic::Ordering;
 use std::sync::Arc;
 use std::sync::MutexGuard;
 use std::sync::{Mutex, RwLock};
@@ -94,7 +90,7 @@ mod timeline;

 pub mod size;

-pub use timeline::{PageReconstructError, Timeline};
+pub use timeline::{with_ondemand_download, PageReconstructError, PageReconstructResult, Timeline};

 // re-export this function so that page_cache.rs can use it.
 pub use crate::tenant::ephemeral_file::writeback as writeback_ephemeral_file;
@@ -141,7 +137,6 @@ pub struct Tenant {

    /// Cached logical sizes updated updated on each [`Tenant::gather_size_inputs`].
    cached_logical_sizes: tokio::sync::Mutex<HashMap<(TimelineId, Lsn), u64>>,
-    cached_synthetic_tenant_size: Arc<AtomicU64>,
 }

 /// A timeline with some of its files on disk, being initialized.
@@ -188,7 +183,7 @@ impl UninitializedTimeline<'_> {
        mut self,
        timelines: &mut HashMap<TimelineId, Arc<Timeline>>,
        load_layer_map: bool,
-        activate: bool,
+        launch_wal_receiver: bool,
    ) -> anyhow::Result<Arc<Timeline>> {
        let timeline_id = self.timeline_id;
        let tenant_id = self.owning_tenant.tenant_id;
@@ -221,12 +216,13 @@ impl UninitializedTimeline<'_> {
                        "Failed to remove uninit mark file for timeline {tenant_id}/{timeline_id}"
                    )
                })?;
+                new_timeline.set_state(TimelineState::Active);
                v.insert(Arc::clone(&new_timeline));

                new_timeline.maybe_spawn_flush_loop();

-                if activate {
-                    new_timeline.activate();
+                if launch_wal_receiver {
+                    new_timeline.launch_wal_receiver();
                }
            }
        }
@@ -440,16 +436,8 @@ struct RemoteStartupData {

 impl Tenant {
    /// Yet another helper for timeline initialization.
-    /// Contains the common part of `load_local_timeline` and `load_remote_timeline`.
-    ///
-    /// - Initializes the Timeline struct and inserts it into the tenant's hash map
-    /// - Scans the local timeline directory for layer files and builds the layer map
-    /// - Downloads remote index file and adds remote files to the layer map
-    /// - Schedules remote upload tasks for any files that are present locally but missing from remote storage.
-    ///
-    /// If the operation fails, the timeline is left in the tenant's hash map in Broken state. On success,
-    /// it is marked as Active.
-    async fn timeline_init_and_sync(
+    /// Contains common part for `load_local_timeline` and `load_remote_timeline`
+    async fn setup_timeline(
        &self,
        timeline_id: TimelineId,
        remote_client: Option<RemoteTimelineClient>,
@@ -492,7 +480,10 @@ impl Tenant {
            // But we shouldnt start walreceiver before we have all the data locally, because working walreceiver
            // will ingest data which may require looking at the layers which are not yet available locally
            match timeline.initialize_with_lock(&mut timelines_accessor, true, false) {
-                Ok(new_timeline) => new_timeline,
+                Ok(initialized_timeline) => {
+                    timelines_accessor.insert(timeline_id, initialized_timeline.clone());
+                    Ok(initialized_timeline)
+                }
                Err(e) => {
                    error!("Failed to initialize timeline {tenant_id}/{timeline_id}: {e:?}");
                    // FIXME using None is a hack, it wont hurt, just ugly.
@@ -508,14 +499,16 @@ impl Tenant {
                            None,
                        )
                        .with_context(|| {
-                            format!("creating broken timeline data for {tenant_id}/{timeline_id}")
+                            format!(
+                            "Failed to crate broken timeline data for {tenant_id}/{timeline_id}"
+                        )
                        })?;
                    broken_timeline.set_state(TimelineState::Broken);
                    timelines_accessor.insert(timeline_id, broken_timeline);
-                    return Err(e);
+                    Err(e)
                }
            }
-        };
+        }?;

        if self.remote_storage.is_some() {
            // Reconcile local state with remote storage, downloading anything that's
@@ -601,7 +594,7 @@ impl Tenant {
                match tenant_clone.attach().await {
                    Ok(_) => {}
                    Err(e) => {
-                        tenant_clone.set_broken(&e.to_string());
+                        tenant_clone.set_broken();
                        error!("error attaching tenant: {:?}", e);
                    }
                }
@@ -617,7 +610,7 @@ impl Tenant {
    #[instrument(skip(self), fields(tenant_id=%self.tenant_id))]
    async fn attach(self: &Arc<Tenant>) -> anyhow::Result<()> {
        // Create directory with marker file to indicate attaching state.
-        // The load_local_tenants() function in tenant::mgr relies on the marker file
+        // The load_local_tenants() function in tenant_mgr relies on the marker file
        // to determine whether a tenant has finished attaching.
        let tenant_dir = self.conf.tenant_path(&self.tenant_id);
        let marker_file = self.conf.tenant_attaching_mark_file_path(&self.tenant_id);
@@ -646,62 +639,26 @@ impl Tenant {
            .as_ref()
            .ok_or_else(|| anyhow::anyhow!("cannot attach without remote storage"))?;

-        let remote_timeline_ids = remote_timeline_client::list_remote_timelines(
+        let remote_timelines = remote_timeline_client::list_remote_timelines(
            remote_storage,
            self.conf,
            self.tenant_id,
        )
        .await?;

-        info!("found {} timelines", remote_timeline_ids.len());
+        info!("found {} timelines", remote_timelines.len());

-        // Download & parse index parts
-        let mut part_downloads = JoinSet::new();
-        for timeline_id in remote_timeline_ids {
-            let client = RemoteTimelineClient::new(
-                remote_storage.clone(),
-                self.conf,
-                self.tenant_id,
-                timeline_id,
-            );
-            part_downloads.spawn(
-                async move {
-                    debug!("starting index part download");
-
-                    let index_part = client
-                        .download_index_file()
-                        .await
-                        .context("download index file")?;
-
-                    let remote_metadata = index_part.parse_metadata().context("parse metadata")?;
-
-                    debug!("finished index part download");
-
-                    Result::<_, anyhow::Error>::Ok((
-                        timeline_id,
-                        client,
-                        index_part,
-                        remote_metadata,
-                    ))
-                }
-                .map(move |res| {
-                    res.with_context(|| format!("download index part for timeline {timeline_id}"))
-                })
-                .instrument(info_span!("download_index_part", timeline=%timeline_id)),
-            );
-        }
-        // Wait for all the download tasks to complete & collect results.
-        let mut remote_clients = HashMap::new();
-        let mut index_parts = HashMap::new();
-        let mut timeline_ancestors = HashMap::new();
-        while let Some(result) = part_downloads.join_next().await {
-            // NB: we already added timeline_id as context to the error
-            let result: Result<_, anyhow::Error> = result.context("joinset task join")?;
-            let (timeline_id, client, index_part, remote_metadata) = result?;
-            debug!("successfully downloaded index part for timeline {timeline_id}");
+        let mut timeline_ancestors: HashMap<TimelineId, TimelineMetadata> = HashMap::new();
+        let mut index_parts: HashMap<TimelineId, IndexPart> = HashMap::new();
+        for (timeline_id, index_part) in remote_timelines {
+            let remote_metadata = index_part.parse_metadata().with_context(|| {
+                format!(
+                    "Failed to parse metadata file from remote storage for tenant {} timeline {}",
+                    self.tenant_id, timeline_id
+                )
+            })?;
            timeline_ancestors.insert(timeline_id, remote_metadata);
            index_parts.insert(timeline_id, index_part);
-            remote_clients.insert(timeline_id, client);
        }

        // For every timeline, download the metadata file, scan the local directory,
@@ -714,7 +671,7 @@ impl Tenant {
                timeline_id,
                index_parts.remove(&timeline_id).unwrap(),
                remote_metadata,
-                remote_clients.remove(&timeline_id).unwrap(),
+                remote_storage.clone(),
            )
            .await
            .with_context(|| {
@@ -757,19 +714,22 @@ impl Tenant {
        Ok(size)
    }

-    #[instrument(skip_all, fields(timeline_id=%timeline_id))]
+    #[instrument(skip(self, index_part, remote_metadata, remote_storage), fields(timeline_id=%timeline_id))]
    async fn load_remote_timeline(
        &self,
        timeline_id: TimelineId,
        index_part: IndexPart,
        remote_metadata: TimelineMetadata,
-        remote_client: RemoteTimelineClient,
+        remote_storage: GenericRemoteStorage,
    ) -> anyhow::Result<()> {
        info!("downloading index file for timeline {}", timeline_id);
        tokio::fs::create_dir_all(self.conf.timeline_path(&timeline_id, &self.tenant_id))
            .await
            .context("Failed to create new timeline directory")?;

+        let remote_client =
+            RemoteTimelineClient::new(remote_storage, self.conf, self.tenant_id, timeline_id)?;
+
        let ancestor = if let Some(ancestor_id) = remote_metadata.ancestor_timeline() {
            let timelines = self.timelines.lock().unwrap();
            Some(Arc::clone(timelines.get(&ancestor_id).ok_or_else(
@@ -788,7 +748,7 @@ impl Tenant {
        // cannot be older than the local one
        let local_metadata = None;

-        self.timeline_init_and_sync(
+        self.setup_timeline(
            timeline_id,
            Some(remote_client),
            Some(RemoteStartupData {
@@ -865,7 +825,7 @@ impl Tenant {
                match tenant_clone.load().await {
                    Ok(()) => {}
                    Err(err) => {
-                        tenant_clone.set_broken(&err.to_string());
+                        tenant_clone.set_broken();
                        error!("could not load tenant {tenant_id}: {err:?}");
                    }
                }
@@ -1026,14 +986,18 @@ impl Tenant {
            None
        };

-        let remote_client = self.remote_storage.as_ref().map(|remote_storage| {
-            RemoteTimelineClient::new(
-                remote_storage.clone(),
-                self.conf,
-                self.tenant_id,
-                timeline_id,
-            )
-        });
+        let remote_client = self
+            .remote_storage
+            .as_ref()
+            .map(|remote_storage| {
+                RemoteTimelineClient::new(
+                    remote_storage.clone(),
+                    self.conf,
+                    self.tenant_id,
+                    timeline_id,
+                )
+            })
+            .transpose()?;

        let remote_startup_data = match &remote_client {
            Some(remote_client) => match remote_client.download_index_file().await {
@@ -1053,7 +1017,7 @@ impl Tenant {
            None => None,
        };

-        self.timeline_init_and_sync(
+        self.setup_timeline(
            timeline_id,
            remote_client,
            remote_startup_data,
@@ -1461,7 +1425,8 @@ impl Tenant {
                    tasks::start_background_loops(self.tenant_id);

                    for timeline in not_broken_timelines {
-                        timeline.activate();
+                        timeline.set_state(TimelineState::Active);
+                        timeline.launch_wal_receiver();
                    }
                }
            }
@@ -1485,7 +1450,7 @@ impl Tenant {
                        .values()
                        .filter(|timeline| timeline.current_state() != TimelineState::Broken);
                    for timeline in not_broken_timelines {
-                        timeline.set_state(TimelineState::Stopping);
+                        timeline.set_state(TimelineState::Suspended);
                    }
                }
                TenantState::Broken => {
@@ -1500,7 +1465,7 @@ impl Tenant {
        });
    }

-    pub fn set_broken(&self, reason: &str) {
+    pub fn set_broken(&self) {
        self.state.send_modify(|current_state| {
            match *current_state {
                TenantState::Active => {
@@ -1509,22 +1474,18 @@ impl Tenant {
                    // activated should never be marked as broken. We cope with it the best
                    // we can, but it shouldn't happen.
                    *current_state = TenantState::Broken;
-                    warn!("Changing Active tenant to Broken state, reason: {}", reason);
+                    warn!("Changing Active tenant to Broken state");
                }
                TenantState::Broken => {
                    // This shouldn't happen either
-                    warn!("Tenant is already in Broken state");
+                    warn!("Tenant is already broken");
                }
                TenantState::Stopping => {
                    // This shouldn't happen either
                    *current_state = TenantState::Broken;
-                    warn!(
-                        "Marking Stopping tenant as Broken state, reason: {}",
-                        reason
-                    );
+                    warn!("Marking Stopping tenant as Broken");
                }
                TenantState::Loading | TenantState::Attaching => {
-                    info!("Setting tenant as Broken state, reason: {}", reason);
                    *current_state = TenantState::Broken;
                }
            }
@@ -1726,7 +1687,6 @@ impl Tenant {
            remote_storage,
            state,
            cached_logical_sizes: tokio::sync::Mutex::new(HashMap::new()),
-            cached_synthetic_tenant_size: Arc::new(AtomicU64::new(0)),
        }
    }

@@ -1879,12 +1839,7 @@ impl Tenant {

        utils::failpoint_sleep_millis_async!("gc_iteration_internal_after_getting_gc_timelines");

-        // If there is nothing to GC, we don't want any messages in the INFO log.
-        if !gc_timelines.is_empty() {
-            info!("{} timelines need GC", gc_timelines.len());
-        } else {
-            debug!("{} timelines need GC", gc_timelines.len());
-        }
+        info!("starting on {} timelines", gc_timelines.len());

        // Perform GC for each timeline.
        //
@@ -2236,7 +2191,7 @@ impl Tenant {
                self.conf,
                tenant_id,
                new_timeline_id,
-            );
+            )?;
            remote_client.init_upload_queue_for_empty_remote(&new_metadata)?;
            Some(remote_client)
        } else {
@@ -2364,24 +2319,6 @@ impl Tenant {

        size::gather_inputs(self, logical_sizes_at_once, &mut shared_cache).await
    }
-
-    /// Calculate synthetic tenant size
-    /// This is periodically called by background worker.
-    /// result is cached in tenant struct
-    #[instrument(skip_all, fields(tenant_id=%self.tenant_id))]
-    pub async fn calculate_synthetic_size(&self) -> anyhow::Result<u64> {
-        let inputs = self.gather_size_inputs().await?;
-
-        let size = inputs.calculate()?;
-
-        self.cached_synthetic_tenant_size
-            .store(size, Ordering::Relaxed);
-
-        Ok(size)
-    }
-    pub fn get_cached_synthetic_size(&self) -> u64 {
-        self.cached_synthetic_tenant_size.load(Ordering::Relaxed)
-    }
 }

 fn remove_timeline_and_uninit_mark(timeline_dir: &Path, uninit_mark: &Path) -> anyhow::Result<()> {
@@ -2625,10 +2562,8 @@ where
 pub mod harness {
    use bytes::{Bytes, BytesMut};
    use once_cell::sync::Lazy;
-    use once_cell::sync::OnceCell;
    use std::sync::{Arc, RwLock, RwLockReadGuard, RwLockWriteGuard};
    use std::{fs, path::PathBuf};
-    use utils::logging;
    use utils::lsn::Lsn;

    use crate::{
@@ -2692,8 +2627,6 @@ pub mod harness {
        ),
    }

-    static LOG_HANDLE: OnceCell<()> = OnceCell::new();
-
    impl<'a> TenantHarness<'a> {
        pub fn create(test_name: &'static str) -> anyhow::Result<Self> {
            Self::create_internal(test_name, false)
@@ -2708,10 +2641,6 @@ pub mod harness {
                (Some(LOCK.read().unwrap()), None)
            };

-            LOG_HANDLE.get_or_init(|| {
-                logging::init(logging::LogFormat::Test).expect("Failed to init test logging")
-            });
-
            let repo_dir = PageServerConf::test_repo_dir(test_name);
            let _ = fs::remove_dir_all(&repo_dir);
            fs::create_dir_all(&repo_dir)?;
@@ -2847,15 +2776,15 @@ mod tests {
        drop(writer);

        assert_eq!(
-            tline.get(*TEST_KEY, Lsn(0x10)).await?,
+            tline.get(*TEST_KEY, Lsn(0x10)).no_ondemand_download()?,
            TEST_IMG("foo at 0x10")
        );
        assert_eq!(
-            tline.get(*TEST_KEY, Lsn(0x1f)).await?,
+            tline.get(*TEST_KEY, Lsn(0x1f)).no_ondemand_download()?,
            TEST_IMG("foo at 0x10")
        );
        assert_eq!(
-            tline.get(*TEST_KEY, Lsn(0x20)).await?,
+            tline.get(*TEST_KEY, Lsn(0x20)).no_ondemand_download()?,
            TEST_IMG("foo at 0x20")
        );

@@ -2934,15 +2863,15 @@ mod tests {

        // Check page contents on both branches
        assert_eq!(
-            from_utf8(&tline.get(TEST_KEY_A, Lsn(0x40)).await?)?,
+            from_utf8(&tline.get(TEST_KEY_A, Lsn(0x40)).no_ondemand_download()?)?,
            "foo at 0x40"
        );
        assert_eq!(
-            from_utf8(&newtline.get(TEST_KEY_A, Lsn(0x40)).await?)?,
+            from_utf8(&newtline.get(TEST_KEY_A, Lsn(0x40)).no_ondemand_download()?)?,
            "bar at 0x40"
        );
        assert_eq!(
-            from_utf8(&newtline.get(TEST_KEY_B, Lsn(0x40)).await?)?,
+            from_utf8(&newtline.get(TEST_KEY_B, Lsn(0x40)).no_ondemand_download()?)?,
            "foobar at 0x20"
        );

@@ -3101,7 +3030,10 @@ mod tests {
        tenant
            .gc_iteration(Some(TIMELINE_ID), 0x10, Duration::ZERO)
            .await?;
-        assert!(newtline.get(*TEST_KEY, Lsn(0x25)).await.is_ok());
+        assert!(newtline
+            .get(*TEST_KEY, Lsn(0x25))
+            .no_ondemand_download()
+            .is_ok());

        Ok(())
    }
@@ -3131,7 +3063,7 @@ mod tests {

        // Check that the data is still accessible on the branch.
        assert_eq!(
-            newtline.get(*TEST_KEY, Lsn(0x50)).await?,
+            newtline.get(*TEST_KEY, Lsn(0x50)).no_ondemand_download()?,
            TEST_IMG(&format!("foo at {}", Lsn(0x40)))
        );

@@ -3279,23 +3211,23 @@ mod tests {
        tline.compact().await?;

        assert_eq!(
-            tline.get(*TEST_KEY, Lsn(0x10)).await?,
+            tline.get(*TEST_KEY, Lsn(0x10)).no_ondemand_download()?,
            TEST_IMG("foo at 0x10")
        );
        assert_eq!(
-            tline.get(*TEST_KEY, Lsn(0x1f)).await?,
+            tline.get(*TEST_KEY, Lsn(0x1f)).no_ondemand_download()?,
            TEST_IMG("foo at 0x10")
        );
        assert_eq!(
-            tline.get(*TEST_KEY, Lsn(0x20)).await?,
+            tline.get(*TEST_KEY, Lsn(0x20)).no_ondemand_download()?,
            TEST_IMG("foo at 0x20")
        );
        assert_eq!(
-            tline.get(*TEST_KEY, Lsn(0x30)).await?,
+            tline.get(*TEST_KEY, Lsn(0x30)).no_ondemand_download()?,
            TEST_IMG("foo at 0x30")
        );
        assert_eq!(
-            tline.get(*TEST_KEY, Lsn(0x40)).await?,
+            tline.get(*TEST_KEY, Lsn(0x40)).no_ondemand_download()?,
            TEST_IMG("foo at 0x40")
        );

@@ -3405,7 +3337,7 @@ mod tests {
            for (blknum, last_lsn) in updated.iter().enumerate() {
                test_key.field6 = blknum as u32;
                assert_eq!(
-                    tline.get(test_key, lsn).await?,
+                    tline.get(test_key, lsn).no_ondemand_download()?,
                    TEST_IMG(&format!("{} at {}", blknum, last_lsn))
                );
            }
@@ -3491,7 +3423,7 @@ mod tests {
            for (blknum, last_lsn) in updated.iter().enumerate() {
                test_key.field6 = blknum as u32;
                assert_eq!(
-                    tline.get(test_key, lsn).await?,
+                    tline.get(test_key, lsn).no_ondemand_download()?,
                    TEST_IMG(&format!("{} at {}", blknum, last_lsn))
                );
            }
@@ -3566,7 +3498,7 @@ mod tests {
                println!("checking [{idx}][{blknum}] at {lsn}");
                test_key.field6 = blknum as u32;
                assert_eq!(
-                    tline.get(test_key, *lsn).await?,
+                    tline.get(test_key, *lsn).no_ondemand_download()?,
                    TEST_IMG(&format!("{idx} {blknum} at {lsn}"))
                );
            }
@@ -1,583 +0,0 @@
-use std::collections::BTreeMap;
-use std::ops::Range;
-
-use tracing::info;
-
-use super::layer_coverage::LayerCoverageTuple;
-
-/// Layers in this module are identified and indexed by this data.
-///
-/// This is a helper struct to enable sorting layers by lsn.start.
-///
-/// These three values are enough to uniquely identify a layer, since
-/// a layer is obligated to contain all contents within range, so two
-/// deltas (or images) with the same range have identical content.
-#[derive(Debug, PartialEq, Eq, Clone)]
-pub struct LayerKey {
-    // TODO I use i128 and u64 because it was easy for prototyping,
-    //      testing, and benchmarking. If we can use the Lsn and Key
-    //      types without overhead that would be preferable.
-    pub key: Range<i128>,
-    pub lsn: Range<u64>,
-    pub is_image: bool,
-}
-
-impl PartialOrd for LayerKey {
-    fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {
-        Some(self.cmp(other))
-    }
-}
-
-impl Ord for LayerKey {
-    fn cmp(&self, other: &Self) -> std::cmp::Ordering {
-        // NOTE we really care about comparing by lsn.start first
-        self.lsn
-            .start
-            .cmp(&other.lsn.start)
-            .then(self.lsn.end.cmp(&other.lsn.end))
-            .then(self.key.start.cmp(&other.key.start))
-            .then(self.key.end.cmp(&other.key.end))
-            .then(self.is_image.cmp(&other.is_image))
-    }
-}
-
-/// Efficiently queryable layer coverage for each LSN.
-///
-/// Allows answering layer map queries very efficiently,
-/// but doesn't allow retroactive insertion, which is
-/// sometimes necessary. See BufferedHistoricLayerCoverage.
-pub struct HistoricLayerCoverage<Value> {
-    /// The latest state
-    head: LayerCoverageTuple<Value>,
-
-    /// All previous states
-    historic: BTreeMap<u64, LayerCoverageTuple<Value>>,
-}
-
-impl<T: Clone> Default for HistoricLayerCoverage<T> {
-    fn default() -> Self {
-        Self::new()
-    }
-}
-
-impl<Value: Clone> HistoricLayerCoverage<Value> {
-    pub fn new() -> Self {
-        Self {
-            head: LayerCoverageTuple::default(),
-            historic: BTreeMap::default(),
-        }
-    }
-
-    /// Add a layer
-    ///
-    /// Panics if new layer has older lsn.start than an existing layer.
-    /// See BufferedHistoricLayerCoverage for a more general insertion method.
-    pub fn insert(&mut self, layer_key: LayerKey, value: Value) {
-        // It's only a persistent map, not a retroactive one
-        if let Some(last_entry) = self.historic.iter().next_back() {
-            let last_lsn = last_entry.0;
-            if layer_key.lsn.start < *last_lsn {
-                panic!("unexpected retroactive insert");
-            }
-        }
-
-        // Insert into data structure
-        if layer_key.is_image {
-            self.head
-                .image_coverage
-                .insert(layer_key.key, layer_key.lsn.clone(), value);
-        } else {
-            self.head
-                .delta_coverage
-                .insert(layer_key.key, layer_key.lsn.clone(), value);
-        }
-
-        // Remember history. Clone is O(1)
-        self.historic.insert(layer_key.lsn.start, self.head.clone());
-    }
-
-    /// Query at a particular LSN, inclusive
-    pub fn get_version(&self, lsn: u64) -> Option<&LayerCoverageTuple<Value>> {
-        match self.historic.range(..=lsn).next_back() {
-            Some((_, v)) => Some(v),
-            None => None,
-        }
-    }
-
-    /// Remove all entries after a certain LSN (inclusive)
-    pub fn trim(&mut self, begin: &u64) {
-        self.historic.split_off(begin);
-        self.head = self
-            .historic
-            .iter()
-            .rev()
-            .next()
-            .map(|(_, v)| v.clone())
-            .unwrap_or_default();
-    }
-}
-
-/// This is the most basic test that demonstrates intended usage.
-/// All layers in this test have height 1.
-#[test]
-fn test_persistent_simple() {
-    let mut map = HistoricLayerCoverage::<String>::new();
-    map.insert(
-        LayerKey {
-            key: 0..5,
-            lsn: 100..101,
-            is_image: true,
-        },
-        "Layer 1".to_string(),
-    );
-    map.insert(
-        LayerKey {
-            key: 3..9,
-            lsn: 110..111,
-            is_image: true,
-        },
-        "Layer 2".to_string(),
-    );
-    map.insert(
-        LayerKey {
-            key: 5..6,
-            lsn: 120..121,
-            is_image: true,
-        },
-        "Layer 3".to_string(),
-    );
-
-    // After Layer 1 insertion
-    let version = map.get_version(105).unwrap();
-    assert_eq!(version.image_coverage.query(1), Some("Layer 1".to_string()));
-    assert_eq!(version.image_coverage.query(4), Some("Layer 1".to_string()));
-
-    // After Layer 2 insertion
-    let version = map.get_version(115).unwrap();
-    assert_eq!(version.image_coverage.query(4), Some("Layer 2".to_string()));
-    assert_eq!(version.image_coverage.query(8), Some("Layer 2".to_string()));
-    assert_eq!(version.image_coverage.query(11), None);
-
-    // After Layer 3 insertion
-    let version = map.get_version(125).unwrap();
-    assert_eq!(version.image_coverage.query(4), Some("Layer 2".to_string()));
-    assert_eq!(version.image_coverage.query(5), Some("Layer 3".to_string()));
-    assert_eq!(version.image_coverage.query(7), Some("Layer 2".to_string()));
-}
-
-/// Cover simple off-by-one edge cases
-#[test]
-fn test_off_by_one() {
-    let mut map = HistoricLayerCoverage::<String>::new();
-    map.insert(
-        LayerKey {
-            key: 3..5,
-            lsn: 100..110,
-            is_image: true,
-        },
-        "Layer 1".to_string(),
-    );
-
-    // Check different LSNs
-    let version = map.get_version(99);
-    assert!(version.is_none());
-    let version = map.get_version(100).unwrap();
-    assert_eq!(version.image_coverage.query(4), Some("Layer 1".to_string()));
-    let version = map.get_version(110).unwrap();
-    assert_eq!(version.image_coverage.query(4), Some("Layer 1".to_string()));
-
-    // Check different keys
-    let version = map.get_version(105).unwrap();
-    assert_eq!(version.image_coverage.query(2), None);
-    assert_eq!(version.image_coverage.query(3), Some("Layer 1".to_string()));
-    assert_eq!(version.image_coverage.query(4), Some("Layer 1".to_string()));
-    assert_eq!(version.image_coverage.query(5), None);
-}
-
-/// Cover edge cases where layers begin or end on the same key
-#[test]
-fn test_key_collision() {
-    let mut map = HistoricLayerCoverage::<String>::new();
-
-    map.insert(
-        LayerKey {
-            key: 3..5,
-            lsn: 100..110,
-            is_image: true,
-        },
-        "Layer 10".to_string(),
-    );
-    map.insert(
-        LayerKey {
-            key: 5..8,
-            lsn: 100..110,
-            is_image: true,
-        },
-        "Layer 11".to_string(),
-    );
-    map.insert(
-        LayerKey {
-            key: 3..4,
-            lsn: 200..210,
-            is_image: true,
-        },
-        "Layer 20".to_string(),
-    );
-
-    // Check after layer 11
-    let version = map.get_version(105).unwrap();
-    assert_eq!(version.image_coverage.query(2), None);
-    assert_eq!(
-        version.image_coverage.query(3),
-        Some("Layer 10".to_string())
-    );
-    assert_eq!(
-        version.image_coverage.query(5),
-        Some("Layer 11".to_string())
-    );
-    assert_eq!(
-        version.image_coverage.query(7),
-        Some("Layer 11".to_string())
-    );
-    assert_eq!(version.image_coverage.query(8), None);
-
-    // Check after layer 20
-    let version = map.get_version(205).unwrap();
-    assert_eq!(version.image_coverage.query(2), None);
-    assert_eq!(
-        version.image_coverage.query(3),
-        Some("Layer 20".to_string())
-    );
-    assert_eq!(
-        version.image_coverage.query(5),
-        Some("Layer 11".to_string())
-    );
-    assert_eq!(
-        version.image_coverage.query(7),
-        Some("Layer 11".to_string())
-    );
-    assert_eq!(version.image_coverage.query(8), None);
-}
-
-/// Test when rectangles have nontrivial height and possibly overlap
-#[test]
-fn test_persistent_overlapping() {
-    let mut map = HistoricLayerCoverage::<String>::new();
-
-    // Add 3 key-disjoint layers with varying LSN ranges
-    map.insert(
-        LayerKey {
-            key: 1..2,
-            lsn: 100..200,
-            is_image: true,
-        },
-        "Layer 1".to_string(),
-    );
-    map.insert(
-        LayerKey {
-            key: 4..5,
-            lsn: 110..200,
-            is_image: true,
-        },
-        "Layer 2".to_string(),
-    );
-    map.insert(
-        LayerKey {
-            key: 7..8,
-            lsn: 120..300,
-            is_image: true,
-        },
-        "Layer 3".to_string(),
-    );
-
-    // Add wide and short layer
-    map.insert(
-        LayerKey {
-            key: 0..9,
-            lsn: 130..199,
-            is_image: true,
-        },
-        "Layer 4".to_string(),
-    );
-
-    // Add wide layer taller than some
-    map.insert(
-        LayerKey {
-            key: 0..9,
-            lsn: 140..201,
-            is_image: true,
-        },
-        "Layer 5".to_string(),
-    );
-
-    // Add wide layer taller than all
-    map.insert(
-        LayerKey {
-            key: 0..9,
-            lsn: 150..301,
-            is_image: true,
-        },
-        "Layer 6".to_string(),
-    );
-
-    // After layer 4 insertion
-    let version = map.get_version(135).unwrap();
-    assert_eq!(version.image_coverage.query(0), Some("Layer 4".to_string()));
-    assert_eq!(version.image_coverage.query(1), Some("Layer 1".to_string()));
-    assert_eq!(version.image_coverage.query(2), Some("Layer 4".to_string()));
-    assert_eq!(version.image_coverage.query(4), Some("Layer 2".to_string()));
-    assert_eq!(version.image_coverage.query(5), Some("Layer 4".to_string()));
-    assert_eq!(version.image_coverage.query(7), Some("Layer 3".to_string()));
-    assert_eq!(version.image_coverage.query(8), Some("Layer 4".to_string()));
-
-    // After layer 5 insertion
-    let version = map.get_version(145).unwrap();
-    assert_eq!(version.image_coverage.query(0), Some("Layer 5".to_string()));
-    assert_eq!(version.image_coverage.query(1), Some("Layer 5".to_string()));
-    assert_eq!(version.image_coverage.query(2), Some("Layer 5".to_string()));
-    assert_eq!(version.image_coverage.query(4), Some("Layer 5".to_string()));
-    assert_eq!(version.image_coverage.query(5), Some("Layer 5".to_string()));
-    assert_eq!(version.image_coverage.query(7), Some("Layer 3".to_string()));
-    assert_eq!(version.image_coverage.query(8), Some("Layer 5".to_string()));
-
-    // After layer 6 insertion
-    let version = map.get_version(155).unwrap();
-    assert_eq!(version.image_coverage.query(0), Some("Layer 6".to_string()));
-    assert_eq!(version.image_coverage.query(1), Some("Layer 6".to_string()));
-    assert_eq!(version.image_coverage.query(2), Some("Layer 6".to_string()));
-    assert_eq!(version.image_coverage.query(4), Some("Layer 6".to_string()));
-    assert_eq!(version.image_coverage.query(5), Some("Layer 6".to_string()));
-    assert_eq!(version.image_coverage.query(7), Some("Layer 6".to_string()));
-    assert_eq!(version.image_coverage.query(8), Some("Layer 6".to_string()));
-}
-
-/// Wrapper for HistoricLayerCoverage that allows us to hack around the lack
-/// of support for retroactive insertion by rebuilding the map since the
-/// change.
-///
-/// Why is this needed? We most often insert new layers with newer LSNs,
-/// but during compaction we create layers with non-latest LSN, and during
-/// GC we delete historic layers.
-///
-/// Even though rebuilding is an expensive (N log N) solution to the problem,
-/// it's not critical since we do something equally expensive just to decide
-/// whether or not to create new image layers.
-/// TODO It's not expensive but it's not great to hold a layer map write lock
-///      for that long.
-///
-/// If this becomes an actual bottleneck, one solution would be to build a
-/// segment tree that holds PersistentLayerMaps. Though this would mean that
-/// we take an additional log(N) performance hit for queries, which will probably
-/// still be more critical.
-///
-/// See this for more on persistent and retroactive techniques:
-/// https://www.youtube.com/watch?v=WqCWghETNDc&t=581s
-pub struct BufferedHistoricLayerCoverage<Value> {
-    /// A persistent layer map that we rebuild when we need to retroactively update
-    historic_coverage: HistoricLayerCoverage<Value>,
-
-    /// We buffer insertion into the PersistentLayerMap to decrease the number of rebuilds.
-    buffer: BTreeMap<LayerKey, Option<Value>>,
-
-    /// All current layers. This is not used for search. Only to make rebuilds easier.
-    layers: BTreeMap<LayerKey, Value>,
-}
-
-impl<T: std::fmt::Debug> std::fmt::Debug for BufferedHistoricLayerCoverage<T> {
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        f.debug_struct("RetroactiveLayerMap")
-            .field("buffer", &self.buffer)
-            .field("layers", &self.layers)
-            .finish()
-    }
-}
-
-impl<T: Clone> Default for BufferedHistoricLayerCoverage<T> {
-    fn default() -> Self {
-        Self::new()
-    }
-}
-
-impl<Value: Clone> BufferedHistoricLayerCoverage<Value> {
-    pub fn new() -> Self {
-        Self {
-            historic_coverage: HistoricLayerCoverage::<Value>::new(),
-            buffer: BTreeMap::new(),
-            layers: BTreeMap::new(),
-        }
-    }
-
-    pub fn insert(&mut self, layer_key: LayerKey, value: Value) {
-        self.buffer.insert(layer_key, Some(value));
-    }
-
-    pub fn remove(&mut self, layer_key: LayerKey) {
-        self.buffer.insert(layer_key, None);
-    }
-
-    pub fn rebuild(&mut self) {
-        // Find the first LSN that needs to be rebuilt
-        let rebuild_since: u64 = match self.buffer.iter().next() {
-            Some((LayerKey { lsn, .. }, _)) => lsn.start,
-            None => return, // No need to rebuild if buffer is empty
-        };
-
-        // Apply buffered updates to self.layers
-        let num_updates = self.buffer.len();
-        self.buffer.retain(|layer_key, layer| {
-            match layer {
-                Some(l) => {
-                    self.layers.insert(layer_key.clone(), l.clone());
-                }
-                None => {
-                    self.layers.remove(layer_key);
-                }
-            };
-            false
-        });
-
-        // Rebuild
-        let mut num_inserted = 0;
-        self.historic_coverage.trim(&rebuild_since);
-        for (layer_key, layer) in self.layers.range(
-            LayerKey {
-                lsn: rebuild_since..0,
-                key: 0..0,
-                is_image: false,
-            }..,
-        ) {
-            self.historic_coverage
-                .insert(layer_key.clone(), layer.clone());
-            num_inserted += 1;
-        }
-
-        // TODO maybe only warn if ratio is at least 10
-        info!(
-            "Rebuilt layer map. Did {} insertions to process a batch of {} updates.",
-            num_inserted, num_updates,
-        )
-    }
-
-    /// Iterate all the layers
-    pub fn iter(&self) -> impl '_ + Iterator<Item = Value> {
-        // NOTE we can actually perform this without rebuilding,
-        //      but it's not necessary for now.
-        if !self.buffer.is_empty() {
-            panic!("rebuild pls")
-        }
-
-        self.layers.values().cloned()
-    }
-
-    /// Return a reference to a queryable map, assuming all updates
-    /// have already been processed using self.rebuild()
-    pub fn get(&self) -> anyhow::Result<&HistoricLayerCoverage<Value>> {
-        // NOTE we error here instead of implicitly rebuilding because
-        //      rebuilding is somewhat expensive.
-        // TODO maybe implicitly rebuild and log/sentry an error?
-        if !self.buffer.is_empty() {
-            anyhow::bail!("rebuild required")
-        }
-
-        Ok(&self.historic_coverage)
-    }
-}
-
-#[test]
-fn test_retroactive_regression_1() {
-    let mut map = BufferedHistoricLayerCoverage::new();
-
-    map.insert(
-        LayerKey {
-            key: 0..21267647932558653966460912964485513215,
-            lsn: 23761336..23761457,
-            is_image: false,
-        },
-        "sdfsdfs".to_string(),
-    );
-
-    map.rebuild();
-
-    let version = map.get().unwrap().get_version(23761457).unwrap();
-    assert_eq!(
-        version.delta_coverage.query(100),
-        Some("sdfsdfs".to_string())
-    );
-}
-
-#[test]
-fn test_retroactive_simple() {
-    let mut map = BufferedHistoricLayerCoverage::new();
-
-    // Append some images in increasing LSN order
-    map.insert(
-        LayerKey {
-            key: 0..5,
-            lsn: 100..101,
-            is_image: true,
-        },
-        "Image 1".to_string(),
-    );
-    map.insert(
-        LayerKey {
-            key: 3..9,
-            lsn: 110..111,
-            is_image: true,
-        },
-        "Image 2".to_string(),
-    );
-    map.insert(
-        LayerKey {
-            key: 4..6,
-            lsn: 120..121,
-            is_image: true,
-        },
-        "Image 3".to_string(),
-    );
-    map.insert(
-        LayerKey {
-            key: 8..9,
-            lsn: 120..121,
-            is_image: true,
-        },
-        "Image 4".to_string(),
-    );
-
-    // Add a delta layer out of order
-    map.insert(
-        LayerKey {
-            key: 2..5,
-            lsn: 105..106,
-            is_image: true,
-        },
-        "Delta 1".to_string(),
-    );
-
-    // Rebuild so we can start querying
-    map.rebuild();
-
-    // Query key 4
-    let version = map.get().unwrap().get_version(90);
-    assert!(version.is_none());
-    let version = map.get().unwrap().get_version(102).unwrap();
-    assert_eq!(version.image_coverage.query(4), Some("Image 1".to_string()));
-    let version = map.get().unwrap().get_version(107).unwrap();
-    assert_eq!(version.image_coverage.query(4), Some("Delta 1".to_string()));
-    let version = map.get().unwrap().get_version(115).unwrap();
-    assert_eq!(version.image_coverage.query(4), Some("Image 2".to_string()));
-    let version = map.get().unwrap().get_version(125).unwrap();
-    assert_eq!(version.image_coverage.query(4), Some("Image 3".to_string()));
-
-    // Remove Image 3
-    map.remove(LayerKey {
-        key: 4..6,
-        lsn: 120..121,
-        is_image: true,
-    });
-    map.rebuild();
-
-    // Check deletion worked
-    let version = map.get().unwrap().get_version(125).unwrap();
-    assert_eq!(version.image_coverage.query(4), Some("Image 2".to_string()));
-    assert_eq!(version.image_coverage.query(8), Some("Image 4".to_string()));
-}
@@ -1,229 +0,0 @@
-use std::ops::Range;
-
-use im::OrdMap;
-use rpds::RedBlackTreeMapSync;
-
-/// Data structure that can efficiently:
-/// - find the latest layer by lsn.end at a given key
-/// - iterate the latest layers in a key range
-/// - insert layers in non-decreasing lsn.start order
-///
-/// The struct is parameterized over Value for easier
-/// testing, but in practice it's some sort of layer.
-pub struct LayerCoverage<Value> {
-    /// For every change in coverage (as we sweep the key space)
-    /// we store (lsn.end, value).
-    ///
-    /// We use an immutable/persistent tree so that we can keep historic
-    /// versions of this coverage without cloning the whole thing and
-    /// incurring quadratic memory cost. See HistoricLayerCoverage.
-    ///
-    /// We use the Sync version of the map because we want Self to
-    /// be Sync. Using nonsync might be faster, if we can work with
-    /// that.
-    nodes: RedBlackTreeMapSync<i128, Option<(u64, Value)>>,
-    im_nodes: OrdMap<i128, Option<(u64, Value)>>,
-}
-
-impl<T: Clone> Default for LayerCoverage<T> {
-    fn default() -> Self {
-        Self::new()
-    }
-}
-
-impl<Value: Clone> LayerCoverage<Value> {
-    pub fn new() -> Self {
-        Self {
-            nodes: RedBlackTreeMapSync::default(),
-            im_nodes: OrdMap::default(),
-        }
-    }
-
-    /// Helper function to subdivide the key range without changing any values
-    ///
-    /// Complexity: O(log N)
-    fn add_node(&mut self, key: i128) {
-        let value = match self.nodes.range(..=key).last() {
-            Some((_, Some(v))) => Some(v.clone()),
-            Some((_, None)) => None,
-            None => None,
-        };
-        self.nodes.insert_mut(key, value);
-
-        let im_value = match self.im_nodes.range(..=key).last() {
-            Some((_, Some(v))) => Some(v.clone()),
-            Some((_, None)) => None,
-            None => None,
-        };
-        self.im_nodes.remove(&key);
-        self.im_nodes.insert(key, im_value);
-    }
-
-    /// Insert a layer.
-    ///
-    /// Complexity: worst case O(N), in practice O(log N). See NOTE in implementation.
-    pub fn insert(&mut self, key: Range<i128>, lsn: Range<u64>, value: Value) {
-        // Add nodes at endpoints
-        //
-        // NOTE The order of lines is important. We add nodes at the start
-        // and end of the key range **before updating any nodes** in order
-        // to pin down the current coverage outside of the relevant key range.
-        // Only the coverage inside the layer's key range should change.
-        self.add_node(key.start);
-        self.add_node(key.end);
-
-        // Raise the height where necessary
-        //
-        // NOTE This loop is worst case O(N), but amortized O(log N) in the special
-        // case when rectangles have no height. In practice I don't think we'll see
-        // the kind of layer intersections needed to trigger O(N) behavior. The worst
-        // case is N/2 horizontal layers overlapped with N/2 vertical layers in a
-        // grid pattern.
-        let mut to_update = Vec::new();
-        let mut to_remove = Vec::new();
-        let mut prev_covered = false;
-        for (k, node) in self.nodes.range(key.clone()) {
-            let needs_cover = match node {
-                None => true,
-                Some((h, _)) => h < &lsn.end,
-            };
-            if needs_cover {
-                match prev_covered {
-                    true => to_remove.push(*k),
-                    false => to_update.push(*k),
-                }
-            }
-            prev_covered = needs_cover;
-        }
-        if !prev_covered {
-            to_remove.push(key.end);
-        }
-        for k in to_update {
-            self.nodes.insert_mut(k, Some((lsn.end, value.clone())));
-        }
-        for k in to_remove {
-            self.nodes.remove_mut(&k);
-        }
-
-
-        let mut to_update = Vec::new();
-        let mut to_remove = Vec::new();
-        let mut prev_covered = false;
-        for (k, node) in self.im_nodes.range(key.clone()) {
-            let needs_cover = match node {
-                None => true,
-                Some((h, _)) => h < &lsn.end,
-            };
-            if needs_cover {
-                match prev_covered {
-                    true => to_remove.push(*k),
-                    false => to_update.push(*k),
-                }
-            }
-            prev_covered = needs_cover;
-        }
-        if !prev_covered {
-            to_remove.push(key.end);
-        }
-        for k in to_update {
-            self.im_nodes.remove(&k);
-            self.im_nodes.insert(k, Some((lsn.end, value.clone())));
-        }
-        for k in to_remove {
-            self.im_nodes.remove(&k);
-        }
-
-    }
-
-    fn get_key_1(&self, key: i128) -> Option<u64> {
-        self.im_nodes
-            .get_prev(&key)?
-            .1
-            .as_ref()
-            .map(|(k, _)| k.clone())
-    }
-    fn get_key_2(&self, key: i128) -> Option<u64> {
-        self.im_nodes
-            .range(..=key)
-            .rev()
-            .next()?
-            .1
-            .as_ref()
-            .map(|(k, _)| k.clone())
-    }
-
-    /// Get the latest (by lsn.end) layer at a given key
-    ///
-    /// Complexity: O(log N)
-    pub fn query(&self, key: i128) -> Option<Value> {
-
-        let k1 = self.get_key_1(key);
-        let k2 = self.get_key_2(key);
-        assert_eq!(k1, k2);
-
-
-        // self.im_nodes
-        //     .get_prev(&key)?
-        //     .1
-        //     .as_ref()
-        //     .map(|(_, v)| v.clone())
-
-        self.im_nodes
-            .range(..=key)
-            .rev()
-            .next()?
-            .1
-            .as_ref()
-            .map(|(_, v)| v.clone())
-
-        // self.nodes
-        //     .range(..=key)
-        //     .rev()
-        //     .next()?
-        //     .1
-        //     .as_ref()
-        //     .map(|(_, v)| v.clone())
-    }
-
-    /// Iterate the changes in layer coverage in a given range. You will likely
-    /// want to start with self.query(key.start), and then follow up with self.range
-    ///
-    /// Complexity: O(log N + result_size)
-    pub fn range(&self, key: Range<i128>) -> impl '_ + Iterator<Item = (i128, Option<Value>)> {
-        self.nodes
-            .range(key)
-            .map(|(k, v)| (*k, v.as_ref().map(|x| x.1.clone())))
-    }
-
-    /// O(1) clone
-    pub fn clone(&self) -> Self {
-        Self {
-            nodes: self.nodes.clone(),
-            im_nodes: self.im_nodes.clone(),
-        }
-    }
-}
-
-/// Image and delta coverage at a specific LSN.
-pub struct LayerCoverageTuple<Value> {
-    pub image_coverage: LayerCoverage<Value>,
-    pub delta_coverage: LayerCoverage<Value>,
-}
-
-impl<T: Clone> Default for LayerCoverageTuple<T> {
-    fn default() -> Self {
-        Self {
-            image_coverage: LayerCoverage::default(),
-            delta_coverage: LayerCoverage::default(),
-        }
-    }
-}
-
-impl<Value: Clone> LayerCoverageTuple<Value> {
-    pub fn clone(&self) -> Self {
-        Self {
-            image_coverage: self.image_coverage.clone(),
-            delta_coverage: self.delta_coverage.clone(),
-        }
-    }
-}
@@ -430,7 +430,7 @@ where
        Err(e) => {
            let tenants_accessor = TENANTS.read().await;
            match tenants_accessor.get(&tenant_id) {
-                Some(tenant) => tenant.set_broken(&e.to_string()),
+                Some(tenant) => tenant.set_broken(),
                None => warn!("Tenant {tenant_id} got removed from memory"),
            }
            Err(e)
@@ -492,53 +492,3 @@ pub async fn immediate_gc(

    Ok(wait_task_done)
 }
-
-#[cfg(feature = "testing")]
-pub async fn immediate_compact(
-    tenant_id: TenantId,
-    timeline_id: TimelineId,
-) -> Result<tokio::sync::oneshot::Receiver<anyhow::Result<()>>, ApiError> {
-    let guard = TENANTS.read().await;
-
-    let tenant = guard
-        .get(&tenant_id)
-        .map(Arc::clone)
-        .with_context(|| format!("Tenant {tenant_id} not found"))
-        .map_err(ApiError::NotFound)?;
-
-    let timeline = tenant
-        .get_timeline(timeline_id, true)
-        .map_err(ApiError::NotFound)?;
-
-    // Run in task_mgr to avoid race with detach operation
-    let (task_done, wait_task_done) = tokio::sync::oneshot::channel();
-    task_mgr::spawn(
-        &tokio::runtime::Handle::current(),
-        TaskKind::Compaction,
-        Some(tenant_id),
-        Some(timeline_id),
-        &format!(
-            "timeline_compact_handler compaction run for tenant {tenant_id} timeline {timeline_id}"
-        ),
-        false,
-        async move {
-            let result = timeline
-                .compact()
-                .instrument(
-                    info_span!("manual_compact", tenant = %tenant_id, timeline = %timeline_id),
-                )
-                .await;
-
-            match task_done.send(result) {
-                Ok(_) => (),
-                Err(result) => error!("failed to send compaction result: {result:?}"),
-            }
-            Ok(())
-        },
-    );
-
-    // drop the guard until after we've spawned the task so that timeline shutdown will wait for the task
-    drop(guard);
-
-    Ok(wait_task_done)
-}
@@ -16,7 +16,7 @@
 //! unless the pageserver is configured without remote storage.
 //!
 //! We allocate the client instance in [Timeline][`crate::tenant::Timeline`], i.e.,
-//! either in [`crate::tenant::mgr`] during startup or when creating a new
+//! either in [`crate::tenant_mgr`] during startup or when creating a new
 //! timeline.
 //! However, the client does not become ready for use until we've initialized its upload queue:
 //!
@@ -135,7 +135,7 @@
 //! - Initiate upload queue with that [`IndexPart`].
 //! - Reschedule all lost operations by comparing the local filesystem state
 //!   and remote state as per [`IndexPart`]. This is done in
-//!   [`Timeline::timeline_init_and_sync`] and [`Timeline::reconcile_with_remote`].
+//!   [`Timeline::setup_timeline`] and [`Timeline::reconcile_with_remote`].
 //!
 //! Note that if we crash during file deletion between the index update
 //! that removes the file from the list of files, and deleting the remote file,
@@ -298,8 +298,8 @@ impl RemoteTimelineClient {
        conf: &'static PageServerConf,
        tenant_id: TenantId,
        timeline_id: TimelineId,
-    ) -> RemoteTimelineClient {
-        RemoteTimelineClient {
+    ) -> anyhow::Result<RemoteTimelineClient> {
+        Ok(RemoteTimelineClient {
            conf,
            runtime: &BACKGROUND_RUNTIME,
            tenant_id,
@@ -307,7 +307,7 @@ impl RemoteTimelineClient {
            storage_impl: remote_storage,
            upload_queue: Mutex::new(UploadQueue::Uninitialized),
            metrics: Arc::new(RemoteTimelineClientMetrics::new(&tenant_id, &timeline_id)),
-        }
+        })
    }

    /// Initialize the upload queue for a remote storage that already received
@@ -367,10 +367,6 @@ impl RemoteTimelineClient {

    /// Download index file
    pub async fn download_index_file(&self) -> Result<IndexPart, DownloadError> {
-        let _unfinished_gauge_guard = self
-            .metrics
-            .call_begin(&RemoteOpFileKind::Index, &RemoteOpKind::Download);
-
        download::download_index_part(
            self.conf,
            &self.storage_impl,
@@ -397,27 +393,22 @@ impl RemoteTimelineClient {
        layer_file_name: &LayerFileName,
        layer_metadata: &LayerFileMetadata,
    ) -> anyhow::Result<u64> {
-        let downloaded_size = {
-            let _unfinished_gauge_guard = self
-                .metrics
-                .call_begin(&RemoteOpFileKind::Layer, &RemoteOpKind::Download);
-            download::download_layer_file(
-                self.conf,
-                &self.storage_impl,
-                self.tenant_id,
-                self.timeline_id,
-                layer_file_name,
-                layer_metadata,
-            )
-            .measure_remote_op(
-                self.tenant_id,
-                self.timeline_id,
-                RemoteOpFileKind::Layer,
-                RemoteOpKind::Download,
-                Arc::clone(&self.metrics),
-            )
-            .await?
-        };
+        let downloaded_size = download::download_layer_file(
+            self.conf,
+            &self.storage_impl,
+            self.tenant_id,
+            self.timeline_id,
+            layer_file_name,
+            layer_metadata,
+        )
+        .measure_remote_op(
+            self.tenant_id,
+            self.timeline_id,
+            RemoteOpFileKind::Layer,
+            RemoteOpKind::Download,
+            Arc::clone(&self.metrics),
+        )
+        .await?;

        // Update the metadata for given layer file. The remote index file
        // might be missing some information for the file; this allows us
@@ -526,7 +517,7 @@ impl RemoteTimelineClient {
            metadata_bytes,
        );
        let op = UploadOp::UploadMetadata(index_part, disk_consistent_lsn);
-        self.calls_unfinished_metric_begin(&op);
+        self.update_upload_queue_unfinished_metric(1, &op);
        upload_queue.queued_operations.push_back(op);
        upload_queue.latest_files_changes_since_metadata_upload_scheduled = 0;

@@ -558,7 +549,7 @@ impl RemoteTimelineClient {
        upload_queue.latest_files_changes_since_metadata_upload_scheduled += 1;

        let op = UploadOp::UploadLayer(layer_file_name.clone(), layer_metadata.clone());
-        self.calls_unfinished_metric_begin(&op);
+        self.update_upload_queue_unfinished_metric(1, &op);
        upload_queue.queued_operations.push_back(op);

        info!(
@@ -610,7 +601,7 @@ impl RemoteTimelineClient {
            // schedule the actual deletions
            for name in names {
                let op = UploadOp::Delete(RemoteOpFileKind::Layer, name.clone());
-                self.calls_unfinished_metric_begin(&op);
+                self.update_upload_queue_unfinished_metric(1, &op);
                upload_queue.queued_operations.push_back(op);
                info!("scheduled layer file deletion {}", name.file_name());
            }
@@ -756,13 +747,13 @@ impl RemoteTimelineClient {
            // Note: We only check for the shutdown requests between retries, so
            // if a shutdown request arrives while we're busy uploading, in the
            // upload::upload:*() call below, we will wait not exit until it has
-            // finished. We probably could cancel the upload by simply dropping
+            // finisheed. We probably could cancel the upload by simply dropping
            // the Future, but we're not 100% sure if the remote storage library
            // is cancellation safe, so we don't dare to do that. Hopefully, the
            // upload finishes or times out soon enough.
            if task_mgr::is_shutdown_requested() {
                info!("upload task cancelled by shutdown request");
-                self.calls_unfinished_metric_end(&task.op);
+                self.update_upload_queue_unfinished_metric(-1, &task.op);
                self.stop();
                return;
            }
@@ -910,40 +901,22 @@ impl RemoteTimelineClient {
            // Launch any queued tasks that were unblocked by this one.
            self.launch_queued_tasks(upload_queue);
        }
-        self.calls_unfinished_metric_end(&task.op);
+        self.update_upload_queue_unfinished_metric(-1, &task.op);
    }

-    fn calls_unfinished_metric_impl(
-        &self,
-        op: &UploadOp,
-    ) -> Option<(RemoteOpFileKind, RemoteOpKind)> {
-        let res = match op {
+    fn update_upload_queue_unfinished_metric(&self, delta: i64, op: &UploadOp) {
+        let (file_kind, op_kind) = match op {
            UploadOp::UploadLayer(_, _) => (RemoteOpFileKind::Layer, RemoteOpKind::Upload),
            UploadOp::UploadMetadata(_, _) => (RemoteOpFileKind::Index, RemoteOpKind::Upload),
            UploadOp::Delete(file_kind, _) => (*file_kind, RemoteOpKind::Delete),
            UploadOp::Barrier(_) => {
                // we do not account these
-                return None;
+                return;
            }
        };
-        Some(res)
-    }
-
-    fn calls_unfinished_metric_begin(&self, op: &UploadOp) {
-        let (file_kind, op_kind) = match self.calls_unfinished_metric_impl(op) {
-            Some(x) => x,
-            None => return,
-        };
-        let guard = self.metrics.call_begin(&file_kind, &op_kind);
-        guard.will_decrement_manually(); // in unfinished_ops_metric_end()
-    }
-
-    fn calls_unfinished_metric_end(&self, op: &UploadOp) {
-        let (file_kind, op_kind) = match self.calls_unfinished_metric_impl(op) {
-            Some(x) => x,
-            None => return,
-        };
-        self.metrics.call_end(&file_kind, &op_kind);
+        self.metrics
+            .unfinished_tasks(&file_kind, &op_kind)
+            .add(delta)
    }

    fn stop(&self) {
@@ -994,7 +967,7 @@ impl RemoteTimelineClient {

                // Tear down queued ops
                for op in qi.queued_operations.into_iter() {
-                    self.calls_unfinished_metric_end(&op);
+                    self.update_upload_queue_unfinished_metric(-1, &op);
                    // Dropping UploadOp::Barrier() here will make wait_completion() return with an Err()
                    // which is exactly what we want to happen.
                    drop(op);
@@ -8,9 +8,10 @@ use std::future::Future;
 use std::path::Path;

 use anyhow::{anyhow, Context};
+use futures::stream::{FuturesUnordered, StreamExt};
 use tokio::fs;
 use tokio::io::AsyncWriteExt;
-use tracing::{error, info, warn};
+use tracing::{debug, error, info, info_span, warn, Instrument};

 use crate::config::PageServerConf;
 use crate::tenant::storage_layer::LayerFileName;
@@ -174,7 +175,7 @@ pub async fn list_remote_timelines<'a>(
    storage: &'a GenericRemoteStorage,
    conf: &'static PageServerConf,
    tenant_id: TenantId,
-) -> anyhow::Result<HashSet<TimelineId>> {
+) -> anyhow::Result<Vec<(TimelineId, IndexPart)>> {
    let tenant_path = conf.timelines_path(&tenant_id);
    let tenant_storage_path = conf.remote_path(&tenant_path)?;

@@ -193,6 +194,7 @@ pub async fn list_remote_timelines<'a>(
    }

    let mut timeline_ids = HashSet::new();
+    let mut part_downloads = FuturesUnordered::new();

    for timeline_remote_storage_key in timelines {
        let object_name = timeline_remote_storage_key.object_name().ok_or_else(|| {
@@ -203,22 +205,35 @@ pub async fn list_remote_timelines<'a>(
            format!("failed to parse object name into timeline id '{object_name}'")
        })?;

-        // list_prefixes is assumed to return unique names. Ensure this here.
-        // NB: it's safer to bail out than warn-log this because the pageserver
-        //     needs to absolutely know about _all_ timelines that exist, so that
-        //     GC knows all the branchpoints. If we skipped over a timeline instead,
-        //     GC could delete a layer that's still needed by that timeline.
-        anyhow::ensure!(
-            !timeline_ids.contains(&timeline_id),
-            "list_prefixes contains duplicate timeline id {timeline_id}"
-        );
-        timeline_ids.insert(timeline_id);
+        // list_prefixes returns all files with the prefix. If we haven't seen this timeline ID
+        // yet, launch a download task for it.
+        if !timeline_ids.contains(&timeline_id) {
+            timeline_ids.insert(timeline_id);
+            let storage_clone = storage.clone();
+            part_downloads.push(async move {
+                (
+                    timeline_id,
+                    download_index_part(conf, &storage_clone, tenant_id, timeline_id)
+                        .instrument(info_span!("download_index_part", timeline=%timeline_id))
+                        .await,
+                )
+            });
+        }
    }

-    Ok(timeline_ids)
+    // Wait for all the download tasks to complete.
+    let mut timeline_parts = Vec::new();
+    while let Some((timeline_id, part_upload_result)) = part_downloads.next().await {
+        let index_part = part_upload_result
+            .with_context(|| format!("Failed to fetch index part for timeline {timeline_id}"))?;
+
+        debug!("Successfully fetched index part for timeline {timeline_id}");
+        timeline_parts.push((timeline_id, index_part));
+    }
+    Ok(timeline_parts)
 }

-pub(super) async fn download_index_part(
+pub async fn download_index_part(
    conf: &'static PageServerConf,
    storage: &GenericRemoteStorage,
    tenant_id: TenantId,
@@ -23,13 +23,7 @@ use tracing::*;
 pub struct ModelInputs {
    updates: Vec<Update>,
    retention_period: u64,
-
-    /// Relevant lsns per timeline.
-    ///
-    /// This field is not required for deserialization purposes, which is mostly used in tests. The
-    /// LSNs explain the outcome (updates) but are not needed in size calculation.
    #[serde_as(as = "HashMap<serde_with::DisplayFromStr, _>")]
-    #[serde(default)]
    timeline_inputs: HashMap<TimelineId, TimelineInputs>,
 }

@@ -38,8 +32,6 @@ pub struct ModelInputs {
 #[serde_with::serde_as]
 #[derive(Debug, serde::Serialize, serde::Deserialize)]
 struct TimelineInputs {
-    #[serde_as(as = "serde_with::DisplayFromStr")]
-    ancestor_lsn: Lsn,
    #[serde_as(as = "serde_with::DisplayFromStr")]
    last_record: Lsn,
    #[serde_as(as = "serde_with::DisplayFromStr")]
@@ -52,116 +44,6 @@ struct TimelineInputs {
    next_gc_cutoff: Lsn,
 }

-// Adjust BranchFrom sorting so that we always process ancestor
-// before descendants. This is needed to correctly calculate size of
-// descendant timelines.
-//
-// Note that we may have multiple BranchFroms at the same LSN, so we
-// need to sort them in the tree order.
-//
-// see updates_sort_with_branches_at_same_lsn test below
-fn sort_updates_in_tree_order(updates: Vec<Update>) -> anyhow::Result<Vec<Update>> {
-    let mut sorted_updates = Vec::with_capacity(updates.len());
-    let mut known_timelineids = HashSet::new();
-    let mut i = 0;
-    while i < updates.len() {
-        let curr_upd = &updates[i];
-
-        if let Command::BranchFrom(parent_id) = curr_upd.command {
-            let parent_id = match parent_id {
-                Some(parent_id) if known_timelineids.contains(&parent_id) => {
-                    // we have already processed ancestor
-                    // process this BranchFrom Update normally
-                    known_timelineids.insert(curr_upd.timeline_id);
-                    sorted_updates.push(*curr_upd);
-                    i += 1;
-                    continue;
-                }
-                None => {
-                    known_timelineids.insert(curr_upd.timeline_id);
-                    sorted_updates.push(*curr_upd);
-                    i += 1;
-                    continue;
-                }
-                Some(parent_id) => parent_id,
-            };
-
-            let mut j = i;
-
-            // we have not processed ancestor yet.
-            // there is a chance that it is at the same Lsn
-            if !known_timelineids.contains(&parent_id) {
-                let mut curr_lsn_branchfroms: HashMap<TimelineId, Vec<(TimelineId, usize)>> =
-                    HashMap::new();
-
-                // inspect all branchpoints at the same lsn
-                while j < updates.len() && updates[j].lsn == curr_upd.lsn {
-                    let lookahead_upd = &updates[j];
-                    j += 1;
-
-                    if let Command::BranchFrom(lookahead_parent_id) = lookahead_upd.command {
-                        match lookahead_parent_id {
-                            Some(lookahead_parent_id)
-                                if !known_timelineids.contains(&lookahead_parent_id) =>
-                            {
-                                // we have not processed ancestor yet
-                                // store it for later
-                                let es =
-                                    curr_lsn_branchfroms.entry(lookahead_parent_id).or_default();
-                                es.push((lookahead_upd.timeline_id, j));
-                            }
-                            _ => {
-                                // we have already processed ancestor
-                                // process this BranchFrom Update normally
-                                known_timelineids.insert(lookahead_upd.timeline_id);
-                                sorted_updates.push(*lookahead_upd);
-                            }
-                        }
-                    }
-                }
-
-                // process BranchFroms in the tree order
-                // check that we don't have a cycle if somet entry is orphan
-                // (this should not happen, but better to be safe)
-                let mut processed_some_entry = true;
-                while processed_some_entry {
-                    processed_some_entry = false;
-
-                    curr_lsn_branchfroms.retain(|parent_id, branchfroms| {
-                        if known_timelineids.contains(parent_id) {
-                            for (timeline_id, j) in branchfroms {
-                                known_timelineids.insert(*timeline_id);
-                                sorted_updates.push(updates[*j - 1]);
-                            }
-                            processed_some_entry = true;
-                            false
-                        } else {
-                            true
-                        }
-                    });
-                }
-
-                if !curr_lsn_branchfroms.is_empty() {
-                    // orphans are expected to be rare and transient between tenant reloads
-                    // for example, an broken ancestor without the child branch being broken.
-                    anyhow::bail!(
-                        "orphan branch(es) detected in BranchFroms: {curr_lsn_branchfroms:?}"
-                    );
-                }
-            }
-
-            assert!(j > i);
-            i = j;
-        } else {
-            // not a BranchFrom, keep the same order
-            sorted_updates.push(*curr_upd);
-            i += 1;
-        }
-    }
-
-    Ok(sorted_updates)
-}
-
 /// Gathers the inputs for the tenant sizing model.
 ///
 /// Tenant size does not consider the latest state, but only the state until next_gc_cutoff, which
@@ -186,20 +68,19 @@ pub(super) async fn gather_inputs(
    // our advantage with `?` error handling.
    let mut joinset = tokio::task::JoinSet::new();

-    // refresh is needed to update gc related pitr_cutoff and horizon_cutoff
-    tenant
+    let timelines = tenant
        .refresh_gc_info()
        .await
        .context("Failed to refresh gc_info before gathering inputs")?;

-    let timelines = tenant.list_timelines();
-
    if timelines.is_empty() {
-        // perhaps the tenant has just been created, and as such doesn't have any data yet
+        // All timelines are below tenant's gc_horizon; alternative would be to use
+        // Tenant::list_timelines but then those gc_info's would not be updated yet, possibly
+        // missing GcInfo::retain_lsns or having obsolete values for cutoff's.
        return Ok(ModelInputs {
            updates: vec![],
            retention_period: 0,
-            timeline_inputs: HashMap::default(),
+            timeline_inputs: HashMap::new(),
        });
    }

@@ -210,25 +91,13 @@ pub(super) async fn gather_inputs(

    let mut updates = Vec::new();

-    // record the per timeline values useful to debug the model inputs, also used to track
-    // ancestor_lsn without keeping a hold of Timeline
+    // record the per timline values used to determine `retention_period`
    let mut timeline_inputs = HashMap::with_capacity(timelines.len());

    // used to determine the `retention_period` for the size model
    let mut max_cutoff_distance = None;

-    // mapping from (TimelineId, Lsn) => if this branch point has been handled already via
-    // GcInfo::retain_lsns or if it needs to have its logical_size calculated.
-    let mut referenced_branch_froms = HashMap::<(TimelineId, Lsn), bool>::new();
-
    for timeline in timelines {
-        if !timeline.is_active() {
-            anyhow::bail!(
-                "timeline {} is not active, cannot calculate tenant_size now",
-                timeline.timeline_id
-            );
-        }
-
        let last_record_lsn = timeline.get_last_record_lsn();

        let (interesting_lsns, horizon_cutoff, pitr_cutoff, next_gc_cutoff) = {
@@ -294,30 +163,13 @@ pub(super) async fn gather_inputs(

        // all timelines branch from something, because it might be impossible to pinpoint
        // which is the tenant_size_model's "default" branch.
-
-        let ancestor_lsn = timeline.get_ancestor_lsn();
-
        updates.push(Update {
-            lsn: ancestor_lsn,
+            lsn: timeline.get_ancestor_lsn(),
            command: Command::BranchFrom(timeline.get_ancestor_timeline_id()),
            timeline_id: timeline.timeline_id,
        });

-        if let Some(parent_timeline_id) = timeline.get_ancestor_timeline_id() {
-            // refresh_gc_info will update branchpoints and pitr_cutoff but only do it for branches
-            // which are over gc_horizon. for example, a "main" branch which never received any
-            // updates apart from initdb not have branch points recorded.
-            referenced_branch_froms
-                .entry((parent_timeline_id, timeline.get_ancestor_lsn()))
-                .or_default();
-        }
-
        for (lsn, _kind) in &interesting_lsns {
-            // mark this visited so don't need to re-process this parent
-            *referenced_branch_froms
-                .entry((timeline.timeline_id, *lsn))
-                .or_default() = true;
-
            if let Some(size) = logical_size_cache.get(&(timeline.timeline_id, *lsn)) {
                updates.push(Update {
                    lsn: *lsn,
@@ -333,10 +185,22 @@ pub(super) async fn gather_inputs(
            }
        }

+        // all timelines also have an end point if they have made any progress
+        if last_record_lsn > timeline.get_ancestor_lsn()
+            && !interesting_lsns
+                .iter()
+                .any(|(lsn, _)| lsn == &last_record_lsn)
+        {
+            updates.push(Update {
+                lsn: last_record_lsn,
+                command: Command::EndOfBranch,
+                timeline_id: timeline.timeline_id,
+            });
+        }
+
        timeline_inputs.insert(
            timeline.timeline_id,
            TimelineInputs {
-                ancestor_lsn,
                last_record: last_record_lsn,
                // this is not used above, because it might not have updated recently enough
                latest_gc_cutoff: *timeline.get_latest_gc_cutoff_lsn(),
@@ -347,80 +211,6 @@ pub(super) async fn gather_inputs(
        );
    }

-    // iterate over discovered branch points and make sure we are getting logical sizes at those
-    // points.
-    for ((timeline_id, lsn), handled) in referenced_branch_froms.iter() {
-        if *handled {
-            continue;
-        }
-
-        let timeline_id = *timeline_id;
-        let lsn = *lsn;
-
-        match timeline_inputs.get(&timeline_id) {
-            Some(inputs) if inputs.ancestor_lsn == lsn => {
-                // we don't need an update at this branch point which is also point where
-                // timeline_id branch was branched from.
-                continue;
-            }
-            Some(_) => {}
-            None => {
-                // we should have this because we have iterated through all of the timelines
-                anyhow::bail!("missing timeline_input for {timeline_id}")
-            }
-        }
-
-        if let Some(size) = logical_size_cache.get(&(timeline_id, lsn)) {
-            updates.push(Update {
-                lsn,
-                timeline_id,
-                command: Command::Update(*size),
-            });
-
-            needed_cache.insert((timeline_id, lsn));
-        } else {
-            let timeline = tenant
-                .get_timeline(timeline_id, false)
-                .context("find referenced ancestor timeline")?;
-            let parallel_size_calcs = Arc::clone(limit);
-            joinset.spawn(calculate_logical_size(
-                parallel_size_calcs,
-                timeline.clone(),
-                lsn,
-            ));
-
-            if let Some(parent_id) = timeline.get_ancestor_timeline_id() {
-                // we should not find new ones because we iterated tenants all timelines
-                anyhow::ensure!(
-                    timeline_inputs.contains_key(&parent_id),
-                    "discovered new timeline {parent_id} (parent of {timeline_id})"
-                );
-            }
-        };
-    }
-
-    // finally add in EndOfBranch for all timelines where their last_record_lsn is not a branch
-    // point. this is needed by the model.
-    for (timeline_id, inputs) in timeline_inputs.iter() {
-        let lsn = inputs.last_record;
-
-        if referenced_branch_froms.contains_key(&(*timeline_id, lsn)) {
-            // this means that the (timeline_id, last_record_lsn) represents a branch point
-            // we do not want to add EndOfBranch updates for these points because it doesn't fit
-            // into the current tenant_size_model.
-            continue;
-        }
-
-        if lsn > inputs.ancestor_lsn {
-            // all timelines also have an end point if they have made any progress
-            updates.push(Update {
-                lsn,
-                command: Command::EndOfBranch,
-                timeline_id: *timeline_id,
-            });
-        }
-    }
-
    let mut have_any_error = false;

    while let Some(res) = joinset.join_next().await {
@@ -477,13 +267,8 @@ pub(super) async fn gather_inputs(
    // for branch points, which come as multiple updates at the same LSN, the Command::Update
    // is needed before a branch is made out of that branch Command::BranchFrom. this is
    // handled by the variant order in `Command`.
-    //
    updates.sort_unstable();

-    // And another sort to handle Command::BranchFrom ordering
-    // in case when there are multiple branches at the same LSN.
-    let sorted_updates = sort_updates_in_tree_order(updates)?;
-
    let retention_period = match max_cutoff_distance {
        Some(max) => max.0,
        None => {
@@ -492,7 +277,7 @@ pub(super) async fn gather_inputs(
    };

    Ok(ModelInputs {
-        updates: sorted_updates,
+        updates,
        retention_period,
        timeline_inputs,
    })
@@ -510,23 +295,21 @@ impl ModelInputs {
                command: op,
                timeline_id,
            } = update;
-
            let Lsn(now) = *lsn;
            match op {
                Command::Update(sz) => {
-                    storage.insert_point(&Some(*timeline_id), "".into(), now, Some(*sz))?;
+                    storage.insert_point(&Some(*timeline_id), "".into(), now, Some(*sz));
                }
                Command::EndOfBranch => {
-                    storage.insert_point(&Some(*timeline_id), "".into(), now, None)?;
+                    storage.insert_point(&Some(*timeline_id), "".into(), now, None);
                }
                Command::BranchFrom(parent) => {
-                    // This branch command may fail if it cannot find a parent to branch from.
-                    storage.branch(parent, Some(*timeline_id))?;
+                    storage.branch(parent, Some(*timeline_id));
                }
            }
        }

-        Ok(storage.calculate(self.retention_period)?.total_children())
+        Ok(storage.calculate(self.retention_period).total_children())
    }
 }

@@ -589,7 +372,6 @@ async fn calculate_logical_size(

    let size_res = timeline
        .spawn_ondemand_logical_size_calculation(lsn)
-        .instrument(info_span!("spawn_ondemand_logical_size_calculation"))
        .await?;
    Ok(TimelineAtLsnSizeResult(timeline, lsn, size_res))
 }
@@ -675,146 +457,9 @@ fn updates_sort() {
 fn verify_size_for_multiple_branches() {
    // this is generated from integration test test_tenant_size_with_multiple_branches, but this way
    // it has the stable lsn's
-    //
-    // timelineinputs have been left out, because those explain the inputs, but don't participate
-    // in further size calculations.
-    let doc = r#"{"updates":[{"lsn":"0/0","command":{"branch_from":null},"timeline_id":"cd9d9409c216e64bf580904facedb01b"},{"lsn":"0/176FA40","command":{"update":25763840},"timeline_id":"cd9d9409c216e64bf580904facedb01b"},{"lsn":"0/176FA40","command":{"branch_from":"cd9d9409c216e64bf580904facedb01b"},"timeline_id":"10b532a550540bc15385eac4edde416a"},{"lsn":"0/1819818","command":{"update":26075136},"timeline_id":"10b532a550540bc15385eac4edde416a"},{"lsn":"0/18B5E40","command":{"update":26427392},"timeline_id":"cd9d9409c216e64bf580904facedb01b"},{"lsn":"0/18D3DF0","command":{"update":26492928},"timeline_id":"cd9d9409c216e64bf580904facedb01b"},{"lsn":"0/18D3DF0","command":{"branch_from":"cd9d9409c216e64bf580904facedb01b"},"timeline_id":"230fc9d756f7363574c0d66533564dcc"},{"lsn":"0/220F438","command":{"update":25239552},"timeline_id":"230fc9d756f7363574c0d66533564dcc"}],"retention_period":131072}"#;
+    let doc = r#"{"updates":[{"lsn":"0/0","command":{"branch_from":null},"timeline_id":"cd9d9409c216e64bf580904facedb01b"},{"lsn":"0/176FA40","command":{"update":25763840},"timeline_id":"cd9d9409c216e64bf580904facedb01b"},{"lsn":"0/176FA40","command":{"branch_from":"cd9d9409c216e64bf580904facedb01b"},"timeline_id":"10b532a550540bc15385eac4edde416a"},{"lsn":"0/1819818","command":{"update":26075136},"timeline_id":"10b532a550540bc15385eac4edde416a"},{"lsn":"0/18B5E40","command":{"update":26427392},"timeline_id":"cd9d9409c216e64bf580904facedb01b"},{"lsn":"0/18D3DF0","command":{"update":26492928},"timeline_id":"cd9d9409c216e64bf580904facedb01b"},{"lsn":"0/18D3DF0","command":{"branch_from":"cd9d9409c216e64bf580904facedb01b"},"timeline_id":"230fc9d756f7363574c0d66533564dcc"},{"lsn":"0/220F438","command":{"update":25239552},"timeline_id":"230fc9d756f7363574c0d66533564dcc"}],"retention_period":131072,"timeline_inputs":{"cd9d9409c216e64bf580904facedb01b":{"last_record":"0/18D5E40","latest_gc_cutoff":"0/169ACF0","horizon_cutoff":"0/18B5E40","pitr_cutoff":"0/18B5E40","next_gc_cutoff":"0/18B5E40"},"10b532a550540bc15385eac4edde416a":{"last_record":"0/1839818","latest_gc_cutoff":"0/169ACF0","horizon_cutoff":"0/1819818","pitr_cutoff":"0/1819818","next_gc_cutoff":"0/1819818"},"230fc9d756f7363574c0d66533564dcc":{"last_record":"0/222F438","latest_gc_cutoff":"0/169ACF0","horizon_cutoff":"0/220F438","pitr_cutoff":"0/220F438","next_gc_cutoff":"0/220F438"}}}"#;

    let inputs: ModelInputs = serde_json::from_str(doc).unwrap();

    assert_eq!(inputs.calculate().unwrap(), 36_409_872);
 }
-
-#[test]
-fn updates_sort_with_branches_at_same_lsn() {
-    use std::str::FromStr;
-    use Command::{BranchFrom, EndOfBranch};
-
-    macro_rules! lsn {
-        ($e:expr) => {
-            Lsn::from_str($e).unwrap()
-        };
-    }
-
-    let ids = [
-        TimelineId::from_str("00000000000000000000000000000000").unwrap(),
-        TimelineId::from_str("11111111111111111111111111111111").unwrap(),
-        TimelineId::from_str("22222222222222222222222222222222").unwrap(),
-        TimelineId::from_str("33333333333333333333333333333333").unwrap(),
-        TimelineId::from_str("44444444444444444444444444444444").unwrap(),
-    ];
-
-    // issue https://github.com/neondatabase/neon/issues/3179
-    let commands = vec![
-        Update {
-            lsn: lsn!("0/0"),
-            command: BranchFrom(None),
-            timeline_id: ids[0],
-        },
-        Update {
-            lsn: lsn!("0/169AD58"),
-            command: Command::Update(25387008),
-            timeline_id: ids[0],
-        },
-        // next three are wrongly sorted, because
-        // ids[1] is branched from before ids[1] exists
-        // and ids[2] is branched from before ids[2] exists
-        Update {
-            lsn: lsn!("0/169AD58"),
-            command: BranchFrom(Some(ids[1])),
-            timeline_id: ids[3],
-        },
-        Update {
-            lsn: lsn!("0/169AD58"),
-            command: BranchFrom(Some(ids[0])),
-            timeline_id: ids[2],
-        },
-        Update {
-            lsn: lsn!("0/169AD58"),
-            command: BranchFrom(Some(ids[2])),
-            timeline_id: ids[1],
-        },
-        Update {
-            lsn: lsn!("0/1CA85B8"),
-            command: Command::Update(28925952),
-            timeline_id: ids[1],
-        },
-        Update {
-            lsn: lsn!("0/1CD85B8"),
-            command: Command::Update(29024256),
-            timeline_id: ids[1],
-        },
-        Update {
-            lsn: lsn!("0/1CD85B8"),
-            command: BranchFrom(Some(ids[1])),
-            timeline_id: ids[4],
-        },
-        Update {
-            lsn: lsn!("0/22DCE70"),
-            command: Command::Update(32546816),
-            timeline_id: ids[3],
-        },
-        Update {
-            lsn: lsn!("0/230CE70"),
-            command: EndOfBranch,
-            timeline_id: ids[3],
-        },
-    ];
-
-    let expected = vec![
-        Update {
-            lsn: lsn!("0/0"),
-            command: BranchFrom(None),
-            timeline_id: ids[0],
-        },
-        Update {
-            lsn: lsn!("0/169AD58"),
-            command: Command::Update(25387008),
-            timeline_id: ids[0],
-        },
-        Update {
-            lsn: lsn!("0/169AD58"),
-            command: BranchFrom(Some(ids[0])),
-            timeline_id: ids[2],
-        },
-        Update {
-            lsn: lsn!("0/169AD58"),
-            command: BranchFrom(Some(ids[2])),
-            timeline_id: ids[1],
-        },
-        Update {
-            lsn: lsn!("0/169AD58"),
-            command: BranchFrom(Some(ids[1])),
-            timeline_id: ids[3],
-        },
-        Update {
-            lsn: lsn!("0/1CA85B8"),
-            command: Command::Update(28925952),
-            timeline_id: ids[1],
-        },
-        Update {
-            lsn: lsn!("0/1CD85B8"),
-            command: Command::Update(29024256),
-            timeline_id: ids[1],
-        },
-        Update {
-            lsn: lsn!("0/1CD85B8"),
-            command: BranchFrom(Some(ids[1])),
-            timeline_id: ids[4],
-        },
-        Update {
-            lsn: lsn!("0/22DCE70"),
-            command: Command::Update(32546816),
-            timeline_id: ids[3],
-        },
-        Update {
-            lsn: lsn!("0/230CE70"),
-            command: EndOfBranch,
-            timeline_id: ids[3],
-        },
-    ];
-
-    let sorted_commands = sort_updates_in_tree_order(commands).unwrap();
-
-    assert_eq!(sorted_commands, expected);
-}
@@ -109,7 +109,7 @@ pub trait Layer: Send + Sync {
    /// See PageReconstructResult for possible return values. The collected data
    /// is appended to reconstruct_data; the caller should pass an empty struct
    /// on first call, or a struct with a cached older image of the page if one
-    /// is available. If this returns ValueReconstructResult::Continue, look up
+    /// is available. If this returns PageReconstructResult::Continue, look up
    /// the predecessor layer and call again with the same 'reconstruct_data' to
    /// collect more data.
    fn get_value_reconstruct_data(
@@ -197,49 +197,37 @@ pub fn downcast_remote_layer(
    }
 }

-impl std::fmt::Debug for dyn Layer {
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        f.debug_struct("Layer")
-            .field("short_id", &self.short_id())
-            .finish()
-    }
+pub enum InMemoryOrHistoricLayer<L: ?Sized> {
+    InMemory(Arc<InMemoryLayer>),
+    Historic(Arc<L>),
 }

-/// Holds metadata about a layer without any content. Used mostly for testing.
-pub struct LayerDescriptor {
-    pub key: Range<Key>,
-    pub lsn: Range<Lsn>,
-    pub is_incremental: bool,
-    pub short_id: String,
-}
-
-impl Layer for LayerDescriptor {
-    fn get_key_range(&self) -> Range<Key> {
-        self.key.clone()
+impl<L: ?Sized> InMemoryOrHistoricLayer<L>
+where
+    L: PersistentLayer,
+{
+    pub fn downcast_remote_layer(&self) -> Option<std::sync::Arc<RemoteLayer>> {
+        match self {
+            Self::InMemory(_) => None,
+            Self::Historic(l) => {
+                if l.is_remote_layer() {
+                    Arc::clone(l).downcast_remote_layer()
+                } else {
+                    None
+                }
+            }
+        }
    }

-    fn get_lsn_range(&self) -> Range<Lsn> {
-        self.lsn.clone()
-    }
-
-    fn is_incremental(&self) -> bool {
-        self.is_incremental
-    }
-
-    fn get_value_reconstruct_data(
+    pub fn get_value_reconstruct_data(
        &self,
-        _key: Key,
-        _lsn_range: Range<Lsn>,
-        _reconstruct_data: &mut ValueReconstructState,
+        key: Key,
+        lsn_range: Range<Lsn>,
+        reconstruct_data: &mut ValueReconstructState,
    ) -> Result<ValueReconstructResult> {
-        todo!("This method shouldn't be part of the Layer trait")
-    }
-
-    fn short_id(&self) -> String {
-        self.short_id.clone()
-    }
-
-    fn dump(&self, _verbose: bool) -> Result<()> {
-        todo!()
+        match self {
+            Self::InMemory(l) => l.get_value_reconstruct_data(key, lsn_range, reconstruct_data),
+            Self::Historic(l) => l.get_value_reconstruct_data(key, lsn_range, reconstruct_data),
+        }
    }
 }
@@ -83,7 +83,7 @@ async fn compaction_loop(tenant_id: TenantId) {
            tokio::select! {
                _ = task_mgr::shutdown_watcher() => {
                    info!("received cancellation request during idling");
-                    break;
+                    break ;
                },
                _ = tokio::time::sleep(sleep_duration) => {},
            }
@@ -30,8 +30,8 @@ use bytes::{Buf, Bytes, BytesMut};
 use tracing::*;

 use crate::pgdatadir_mapping::*;
-use crate::tenant::PageReconstructError;
 use crate::tenant::Timeline;
+use crate::tenant::{with_ondemand_download, PageReconstructError};
 use crate::walrecord::*;
 use crate::ZERO_PAGE;
 use pageserver_api::reltag::{RelTag, SlruKind};
@@ -55,7 +55,8 @@ impl<'a> WalIngest<'a> {
    pub async fn new(timeline: &Timeline, startpoint: Lsn) -> anyhow::Result<WalIngest> {
        // Fetch the latest checkpoint into memory, so that we can compare with it
        // quickly in `ingest_record` and update it when it changes.
-        let checkpoint_bytes = timeline.get_checkpoint(startpoint).await?;
+        let checkpoint_bytes =
+            with_ondemand_download(|| timeline.get_checkpoint(startpoint)).await?;
        let checkpoint = CheckPoint::decode(&checkpoint_bytes)?;
        trace!("CheckPoint.nextXid = {}", checkpoint.nextXid.value);

@@ -106,7 +107,7 @@ impl<'a> WalIngest<'a> {
                == pg_constants::XLOG_SMGR_CREATE
        {
            let create = XlSmgrCreate::decode(&mut buf);
-            self.ingest_xlog_smgr_create(modification, &create).await?;
+            self.ingest_xlog_smgr_create(modification, &create)?;
        } else if decoded.xl_rmid == pg_constants::RM_SMGR_ID
            && (decoded.xl_info & pg_constants::XLR_RMGR_INFO_MASK)
                == pg_constants::XLOG_SMGR_TRUNCATE
@@ -134,7 +135,7 @@ impl<'a> WalIngest<'a> {
                    let dropdb = XlDropDatabase::decode(&mut buf);
                    for tablespace_id in dropdb.tablespace_ids {
                        trace!("Drop db {}, {}", tablespace_id, dropdb.db_id);
-                        modification.drop_dbdir(tablespace_id, dropdb.db_id).await?;
+                        modification.drop_dbdir(tablespace_id, dropdb.db_id)?;
                    }
                }
            } else if self.timeline.pg_version == 15 {
@@ -158,7 +159,7 @@ impl<'a> WalIngest<'a> {
                    let dropdb = XlDropDatabase::decode(&mut buf);
                    for tablespace_id in dropdb.tablespace_ids {
                        trace!("Drop db {}, {}", tablespace_id, dropdb.db_id);
-                        modification.drop_dbdir(tablespace_id, dropdb.db_id).await?;
+                        modification.drop_dbdir(tablespace_id, dropdb.db_id)?;
                    }
                }
            }
@@ -213,11 +214,9 @@ impl<'a> WalIngest<'a> {
                    parsed_xact.xid,
                    lsn,
                );
-                modification.drop_twophase_file(parsed_xact.xid).await?;
+                modification.drop_twophase_file(parsed_xact.xid)?;
            } else if info == pg_constants::XLOG_XACT_PREPARE {
-                modification
-                    .put_twophase_file(decoded.xl_xid, Bytes::copy_from_slice(&buf[..]))
-                    .await?;
+                modification.put_twophase_file(decoded.xl_xid, Bytes::copy_from_slice(&buf[..]))?;
            }
        } else if decoded.xl_rmid == pg_constants::RM_MULTIXACT_ID {
            let info = decoded.xl_info & pg_constants::XLR_RMGR_INFO_MASK;
@@ -251,13 +250,11 @@ impl<'a> WalIngest<'a> {
                self.ingest_multixact_create_record(modification, &xlrec)?;
            } else if info == pg_constants::XLOG_MULTIXACT_TRUNCATE_ID {
                let xlrec = XlMultiXactTruncate::decode(&mut buf);
-                self.ingest_multixact_truncate_record(modification, &xlrec)
-                    .await?;
+                self.ingest_multixact_truncate_record(modification, &xlrec)?;
            }
        } else if decoded.xl_rmid == pg_constants::RM_RELMAP_ID {
            let xlrec = XlRelmapUpdate::decode(&mut buf);
-            self.ingest_relmap_page(modification, &xlrec, decoded)
-                .await?;
+            self.ingest_relmap_page(modification, &xlrec, decoded)?;
        } else if decoded.xl_rmid == pg_constants::RM_XLOG_ID {
            let info = decoded.xl_info & pg_constants::XLR_RMGR_INFO_MASK;
            if info == pg_constants::XLOG_NEXTOID {
@@ -537,21 +534,23 @@ impl<'a> WalIngest<'a> {
        // get calls instead.
        let req_lsn = modification.tline.get_last_record_lsn();

-        let rels = modification
-            .tline
-            .list_rels(src_tablespace_id, src_db_id, req_lsn)
-            .await?;
+        let rels = with_ondemand_download(|| {
+            modification
+                .tline
+                .list_rels(src_tablespace_id, src_db_id, req_lsn)
+        })
+        .await?;

        debug!("ingest_xlog_dbase_create: {} rels", rels.len());

        // Copy relfilemap
-        let filemap = modification
-            .tline
-            .get_relmap_file(src_tablespace_id, src_db_id, req_lsn)
-            .await?;
-        modification
-            .put_relmap_file(tablespace_id, db_id, filemap)
-            .await?;
+        let filemap = with_ondemand_download(|| {
+            modification
+                .tline
+                .get_relmap_file(src_tablespace_id, src_db_id, req_lsn)
+        })
+        .await?;
+        modification.put_relmap_file(tablespace_id, db_id, filemap)?;

        let mut num_rels_copied = 0;
        let mut num_blocks_copied = 0;
@@ -559,10 +558,9 @@ impl<'a> WalIngest<'a> {
            assert_eq!(src_rel.spcnode, src_tablespace_id);
            assert_eq!(src_rel.dbnode, src_db_id);

-            let nblocks = modification
-                .tline
-                .get_rel_size(src_rel, req_lsn, true)
-                .await?;
+            let nblocks =
+                with_ondemand_download(|| modification.tline.get_rel_size(src_rel, req_lsn, true))
+                    .await?;
            let dst_rel = RelTag {
                spcnode: tablespace_id,
                dbnode: db_id,
@@ -570,17 +568,19 @@ impl<'a> WalIngest<'a> {
                forknum: src_rel.forknum,
            };

-            modification.put_rel_creation(dst_rel, nblocks).await?;
+            modification.put_rel_creation(dst_rel, nblocks)?;

            // Copy content
            debug!("copying rel {} to {}, {} blocks", src_rel, dst_rel, nblocks);
            for blknum in 0..nblocks {
                debug!("copying block {} from {} to {}", blknum, src_rel, dst_rel);

-                let content = modification
-                    .tline
-                    .get_rel_page_at_lsn(src_rel, blknum, req_lsn, true)
-                    .await?;
+                let content = with_ondemand_download(|| {
+                    modification
+                        .tline
+                        .get_rel_page_at_lsn(src_rel, blknum, req_lsn, true)
+                })
+                .await?;
                modification.put_rel_page_image(dst_rel, blknum, content)?;
                num_blocks_copied += 1;
            }
@@ -595,9 +595,9 @@ impl<'a> WalIngest<'a> {
        Ok(())
    }

-    async fn ingest_xlog_smgr_create(
+    fn ingest_xlog_smgr_create(
        &mut self,
-        modification: &mut DatadirModification<'_>,
+        modification: &mut DatadirModification,
        rec: &XlSmgrCreate,
    ) -> anyhow::Result<()> {
        let rel = RelTag {
@@ -606,7 +606,7 @@ impl<'a> WalIngest<'a> {
            relnode: rec.rnode.relnode,
            forknum: rec.forknum,
        };
-        self.put_rel_creation(modification, rel).await?;
+        self.put_rel_creation(modification, rel)?;
        Ok(())
    }

@@ -629,8 +629,7 @@ impl<'a> WalIngest<'a> {
                relnode,
                forknum: MAIN_FORKNUM,
            };
-            self.put_rel_truncation(modification, rel, rec.blkno)
-                .await?;
+            self.put_rel_truncation(modification, rel, rec.blkno)?;
        }
        if (rec.flags & pg_constants::SMGR_TRUNCATE_FSM) != 0 {
            let rel = RelTag {
@@ -651,8 +650,7 @@ impl<'a> WalIngest<'a> {
            let nblocks = self.get_relsize(rel, modification.lsn).await?;
            if nblocks > fsm_physical_page_no {
                // check if something to do: FSM is larger than truncate position
-                self.put_rel_truncation(modification, rel, fsm_physical_page_no)
-                    .await?;
+                self.put_rel_truncation(modification, rel, fsm_physical_page_no)?;
            }
        }
        if (rec.flags & pg_constants::SMGR_TRUNCATE_VM) != 0 {
@@ -673,8 +671,7 @@ impl<'a> WalIngest<'a> {
            let nblocks = self.get_relsize(rel, modification.lsn).await?;
            if nblocks > vm_page_no {
                // check if something to do: VM is larger than truncate position
-                self.put_rel_truncation(modification, rel, vm_page_no)
-                    .await?;
+                self.put_rel_truncation(modification, rel, vm_page_no)?;
            }
        }
        Ok(())
@@ -743,12 +740,10 @@ impl<'a> WalIngest<'a> {
                    relnode: xnode.relnode,
                };
                let last_lsn = self.timeline.get_last_record_lsn();
-                if modification
-                    .tline
-                    .get_rel_exists(rel, last_lsn, true)
+                if with_ondemand_download(|| modification.tline.get_rel_exists(rel, last_lsn, true))
                    .await?
                {
-                    self.put_rel_drop(modification, rel).await?;
+                    self.put_rel_drop(modification, rel)?;
                }
            }
        }
@@ -800,16 +795,16 @@ impl<'a> WalIngest<'a> {
        // instead.
        let req_lsn = modification.tline.get_last_record_lsn();

-        let slru_segments = modification
-            .tline
-            .list_slru_segments(SlruKind::Clog, req_lsn)
-            .await?;
+        let slru_segments = with_ondemand_download(|| {
+            modification
+                .tline
+                .list_slru_segments(SlruKind::Clog, req_lsn)
+        })
+        .await?;
        for segno in slru_segments {
            let segpage = segno * pg_constants::SLRU_PAGES_PER_SEGMENT;
            if slru_may_delete_clogsegment(segpage, xlrec.pageno) {
-                modification
-                    .drop_slru_segment(SlruKind::Clog, segno)
-                    .await?;
+                modification.drop_slru_segment(SlruKind::Clog, segno)?;
                trace!("Drop CLOG segment {:>04X}", segno);
            }
        }
@@ -896,9 +891,9 @@ impl<'a> WalIngest<'a> {
        Ok(())
    }

-    async fn ingest_multixact_truncate_record(
+    fn ingest_multixact_truncate_record(
        &mut self,
-        modification: &mut DatadirModification<'_>,
+        modification: &mut DatadirModification,
        xlrec: &XlMultiXactTruncate,
    ) -> Result<()> {
        self.checkpoint.oldestMulti = xlrec.end_trunc_off;
@@ -914,9 +909,7 @@ impl<'a> WalIngest<'a> {
        // Delete all the segments except the last one. The last segment can still
        // contain, possibly partially, valid data.
        while segment != endsegment {
-            modification
-                .drop_slru_segment(SlruKind::MultiXactMembers, segment as u32)
-                .await?;
+            modification.drop_slru_segment(SlruKind::MultiXactMembers, segment as u32)?;

            /* move to next segment, handling wraparound correctly */
            if segment == maxsegment {
@@ -932,9 +925,9 @@ impl<'a> WalIngest<'a> {
        Ok(())
    }

-    async fn ingest_relmap_page(
+    fn ingest_relmap_page(
        &mut self,
-        modification: &mut DatadirModification<'_>,
+        modification: &mut DatadirModification,
        xlrec: &XlRelmapUpdate,
        decoded: &DecodedWALRecord,
    ) -> Result<()> {
@@ -943,19 +936,17 @@ impl<'a> WalIngest<'a> {
        // skip xl_relmap_update
        buf.advance(12);

-        modification
-            .put_relmap_file(xlrec.tsid, xlrec.dbid, Bytes::copy_from_slice(&buf[..]))
-            .await?;
+        modification.put_relmap_file(xlrec.tsid, xlrec.dbid, Bytes::copy_from_slice(&buf[..]))?;

        Ok(())
    }

-    async fn put_rel_creation(
+    fn put_rel_creation(
        &mut self,
-        modification: &mut DatadirModification<'_>,
+        modification: &mut DatadirModification,
        rel: RelTag,
    ) -> Result<()> {
-        modification.put_rel_creation(rel, 0).await?;
+        modification.put_rel_creation(rel, 0)?;
        Ok(())
    }

@@ -983,31 +974,28 @@ impl<'a> WalIngest<'a> {
        Ok(())
    }

-    async fn put_rel_truncation(
+    fn put_rel_truncation(
        &mut self,
-        modification: &mut DatadirModification<'_>,
+        modification: &mut DatadirModification,
        rel: RelTag,
        nblocks: BlockNumber,
    ) -> anyhow::Result<()> {
-        modification.put_rel_truncation(rel, nblocks).await?;
+        modification.put_rel_truncation(rel, nblocks)?;
        Ok(())
    }

-    async fn put_rel_drop(
-        &mut self,
-        modification: &mut DatadirModification<'_>,
-        rel: RelTag,
-    ) -> Result<()> {
-        modification.put_rel_drop(rel).await?;
+    fn put_rel_drop(&mut self, modification: &mut DatadirModification, rel: RelTag) -> Result<()> {
+        modification.put_rel_drop(rel)?;
        Ok(())
    }

    async fn get_relsize(&mut self, rel: RelTag, lsn: Lsn) -> anyhow::Result<BlockNumber> {
-        let exists = self.timeline.get_rel_exists(rel, lsn, true).await?;
+        let exists =
+            with_ondemand_download(|| self.timeline.get_rel_exists(rel, lsn, true)).await?;
        let nblocks = if !exists {
            0
        } else {
-            self.timeline.get_rel_size(rel, lsn, true).await?
+            with_ondemand_download(|| self.timeline.get_rel_size(rel, lsn, true)).await?
        };
        Ok(nblocks)
    }
@@ -1023,17 +1011,19 @@ impl<'a> WalIngest<'a> {
        // record.
        // TODO: would be nice if to be more explicit about it
        let last_lsn = modification.lsn;
-        let old_nblocks = if !self.timeline.get_rel_exists(rel, last_lsn, true).await? {
-            // create it with 0 size initially, the logic below will extend it
-            modification.put_rel_creation(rel, 0).await?;
-            0
-        } else {
-            self.timeline.get_rel_size(rel, last_lsn, true).await?
-        };
+        let old_nblocks =
+            if !with_ondemand_download(|| self.timeline.get_rel_exists(rel, last_lsn, true)).await?
+            {
+                // create it with 0 size initially, the logic below will extend it
+                modification.put_rel_creation(rel, 0)?;
+                0
+            } else {
+                with_ondemand_download(|| self.timeline.get_rel_size(rel, last_lsn, true)).await?
+            };

        if new_nblocks > old_nblocks {
            //info!("extending {} {} to {}", rel, old_nblocks, new_nblocks);
-            modification.put_rel_extend(rel, new_nblocks).await?;
+            modification.put_rel_extend(rel, new_nblocks)?;

            // fill the gap with zeros
            for gap_blknum in old_nblocks..blknum {
@@ -1073,19 +1063,16 @@ impl<'a> WalIngest<'a> {
        // record.
        // TODO: would be nice if to be more explicit about it
        let last_lsn = self.timeline.get_last_record_lsn();
-        let old_nblocks = if !self
-            .timeline
-            .get_slru_segment_exists(kind, segno, last_lsn)
-            .await?
+        let old_nblocks = if !with_ondemand_download(|| {
+            self.timeline.get_slru_segment_exists(kind, segno, last_lsn)
+        })
+        .await?
        {
            // create it with 0 size initially, the logic below will extend it
-            modification
-                .put_slru_segment_creation(kind, segno, 0)
-                .await?;
+            modification.put_slru_segment_creation(kind, segno, 0)?;
            0
        } else {
-            self.timeline
-                .get_slru_segment_size(kind, segno, last_lsn)
+            with_ondemand_download(|| self.timeline.get_slru_segment_size(kind, segno, last_lsn))
                .await?
        };

@@ -1137,7 +1124,7 @@ mod tests {
    async fn init_walingest_test(tline: &Timeline) -> Result<WalIngest> {
        let mut m = tline.begin_modification(Lsn(0x10));
        m.put_checkpoint(ZERO_CHECKPOINT.clone())?;
-        m.put_relmap_file(0, 111, Bytes::from("")).await?; // dummy relmapper file
+        m.put_relmap_file(0, 111, Bytes::from(""))?; // dummy relmapper file
        m.commit()?;
        let walingest = WalIngest::new(tline, Lsn(0x10)).await?;

@@ -1151,7 +1138,7 @@ mod tests {
        let mut walingest = init_walingest_test(&tline).await?;

        let mut m = tline.begin_modification(Lsn(0x20));
-        walingest.put_rel_creation(&mut m, TESTREL_A).await?;
+        walingest.put_rel_creation(&mut m, TESTREL_A)?;
        walingest
            .put_rel_page_image(&mut m, TESTREL_A, 0, TEST_IMG("foo blk 0 at 2"))
            .await?;
@@ -1176,103 +1163,132 @@ mod tests {

        // The relation was created at LSN 2, not visible at LSN 1 yet.
        assert_eq!(
-            tline.get_rel_exists(TESTREL_A, Lsn(0x10), false).await?,
+            tline
+                .get_rel_exists(TESTREL_A, Lsn(0x10), false)
+                .no_ondemand_download()?,
            false
        );
        assert!(tline
            .get_rel_size(TESTREL_A, Lsn(0x10), false)
-            .await
+            .no_ondemand_download()
            .is_err());

        assert_eq!(
-            tline.get_rel_exists(TESTREL_A, Lsn(0x20), false).await?,
+            tline
+                .get_rel_exists(TESTREL_A, Lsn(0x20), false)
+                .no_ondemand_download()?,
            true
        );
-        assert_eq!(tline.get_rel_size(TESTREL_A, Lsn(0x20), false).await?, 1);
-        assert_eq!(tline.get_rel_size(TESTREL_A, Lsn(0x50), false).await?, 3);
+        assert_eq!(
+            tline
+                .get_rel_size(TESTREL_A, Lsn(0x20), false)
+                .no_ondemand_download()?,
+            1
+        );
+        assert_eq!(
+            tline
+                .get_rel_size(TESTREL_A, Lsn(0x50), false)
+                .no_ondemand_download()?,
+            3
+        );

        // Check page contents at each LSN
        assert_eq!(
            tline
                .get_rel_page_at_lsn(TESTREL_A, 0, Lsn(0x20), false)
-                .await?,
+                .no_ondemand_download()?,
            TEST_IMG("foo blk 0 at 2")
        );

        assert_eq!(
            tline
                .get_rel_page_at_lsn(TESTREL_A, 0, Lsn(0x30), false)
-                .await?,
+                .no_ondemand_download()?,
            TEST_IMG("foo blk 0 at 3")
        );

        assert_eq!(
            tline
                .get_rel_page_at_lsn(TESTREL_A, 0, Lsn(0x40), false)
-                .await?,
+                .no_ondemand_download()?,
            TEST_IMG("foo blk 0 at 3")
        );
        assert_eq!(
            tline
                .get_rel_page_at_lsn(TESTREL_A, 1, Lsn(0x40), false)
-                .await?,
+                .no_ondemand_download()?,
            TEST_IMG("foo blk 1 at 4")
        );

        assert_eq!(
            tline
                .get_rel_page_at_lsn(TESTREL_A, 0, Lsn(0x50), false)
-                .await?,
+                .no_ondemand_download()?,
            TEST_IMG("foo blk 0 at 3")
        );
        assert_eq!(
            tline
                .get_rel_page_at_lsn(TESTREL_A, 1, Lsn(0x50), false)
-                .await?,
+                .no_ondemand_download()?,
            TEST_IMG("foo blk 1 at 4")
        );
        assert_eq!(
            tline
                .get_rel_page_at_lsn(TESTREL_A, 2, Lsn(0x50), false)
-                .await?,
+                .no_ondemand_download()?,
            TEST_IMG("foo blk 2 at 5")
        );

        // Truncate last block
        let mut m = tline.begin_modification(Lsn(0x60));
-        walingest.put_rel_truncation(&mut m, TESTREL_A, 2).await?;
+        walingest.put_rel_truncation(&mut m, TESTREL_A, 2)?;
        m.commit()?;
        assert_current_logical_size(&tline, Lsn(0x60));

        // Check reported size and contents after truncation
-        assert_eq!(tline.get_rel_size(TESTREL_A, Lsn(0x60), false).await?, 2);
+        assert_eq!(
+            tline
+                .get_rel_size(TESTREL_A, Lsn(0x60), false)
+                .no_ondemand_download()?,
+            2
+        );
        assert_eq!(
            tline
                .get_rel_page_at_lsn(TESTREL_A, 0, Lsn(0x60), false)
-                .await?,
+                .no_ondemand_download()?,
            TEST_IMG("foo blk 0 at 3")
        );
        assert_eq!(
            tline
                .get_rel_page_at_lsn(TESTREL_A, 1, Lsn(0x60), false)
-                .await?,
+                .no_ondemand_download()?,
            TEST_IMG("foo blk 1 at 4")
        );

        // should still see the truncated block with older LSN
-        assert_eq!(tline.get_rel_size(TESTREL_A, Lsn(0x50), false).await?, 3);
+        assert_eq!(
+            tline
+                .get_rel_size(TESTREL_A, Lsn(0x50), false)
+                .no_ondemand_download()?,
+            3
+        );
        assert_eq!(
            tline
                .get_rel_page_at_lsn(TESTREL_A, 2, Lsn(0x50), false)
-                .await?,
+                .no_ondemand_download()?,
            TEST_IMG("foo blk 2 at 5")
        );

        // Truncate to zero length
        let mut m = tline.begin_modification(Lsn(0x68));
-        walingest.put_rel_truncation(&mut m, TESTREL_A, 0).await?;
+        walingest.put_rel_truncation(&mut m, TESTREL_A, 0)?;
        m.commit()?;
-        assert_eq!(tline.get_rel_size(TESTREL_A, Lsn(0x68), false).await?, 0);
+        assert_eq!(
+            tline
+                .get_rel_size(TESTREL_A, Lsn(0x68), false)
+                .no_ondemand_download()?,
+            0
+        );

        // Extend from 0 to 2 blocks, leaving a gap
        let mut m = tline.begin_modification(Lsn(0x70));
@@ -1280,17 +1296,22 @@ mod tests {
            .put_rel_page_image(&mut m, TESTREL_A, 1, TEST_IMG("foo blk 1"))
            .await?;
        m.commit()?;
-        assert_eq!(tline.get_rel_size(TESTREL_A, Lsn(0x70), false).await?, 2);
+        assert_eq!(
+            tline
+                .get_rel_size(TESTREL_A, Lsn(0x70), false)
+                .no_ondemand_download()?,
+            2
+        );
        assert_eq!(
            tline
                .get_rel_page_at_lsn(TESTREL_A, 0, Lsn(0x70), false)
-                .await?,
+                .no_ondemand_download()?,
            ZERO_PAGE
        );
        assert_eq!(
            tline
                .get_rel_page_at_lsn(TESTREL_A, 1, Lsn(0x70), false)
-                .await?,
+                .no_ondemand_download()?,
            TEST_IMG("foo blk 1")
        );

@@ -1300,19 +1321,24 @@ mod tests {
            .put_rel_page_image(&mut m, TESTREL_A, 1500, TEST_IMG("foo blk 1500"))
            .await?;
        m.commit()?;
-        assert_eq!(tline.get_rel_size(TESTREL_A, Lsn(0x80), false).await?, 1501);
+        assert_eq!(
+            tline
+                .get_rel_size(TESTREL_A, Lsn(0x80), false)
+                .no_ondemand_download()?,
+            1501
+        );
        for blk in 2..1500 {
            assert_eq!(
                tline
                    .get_rel_page_at_lsn(TESTREL_A, blk, Lsn(0x80), false)
-                    .await?,
+                    .no_ondemand_download()?,
                ZERO_PAGE
            );
        }
        assert_eq!(
            tline
                .get_rel_page_at_lsn(TESTREL_A, 1500, Lsn(0x80), false)
-                .await?,
+                .no_ondemand_download()?,
            TEST_IMG("foo blk 1500")
        );

@@ -1335,19 +1361,28 @@ mod tests {

        // Check that rel exists and size is correct
        assert_eq!(
-            tline.get_rel_exists(TESTREL_A, Lsn(0x20), false).await?,
+            tline
+                .get_rel_exists(TESTREL_A, Lsn(0x20), false)
+                .no_ondemand_download()?,
            true
        );
-        assert_eq!(tline.get_rel_size(TESTREL_A, Lsn(0x20), false).await?, 1);
+        assert_eq!(
+            tline
+                .get_rel_size(TESTREL_A, Lsn(0x20), false)
+                .no_ondemand_download()?,
+            1
+        );

        // Drop rel
        let mut m = tline.begin_modification(Lsn(0x30));
-        walingest.put_rel_drop(&mut m, TESTREL_A).await?;
+        walingest.put_rel_drop(&mut m, TESTREL_A)?;
        m.commit()?;

        // Check that rel is not visible anymore
        assert_eq!(
-            tline.get_rel_exists(TESTREL_A, Lsn(0x30), false).await?,
+            tline
+                .get_rel_exists(TESTREL_A, Lsn(0x30), false)
+                .no_ondemand_download()?,
            false
        );

@@ -1363,10 +1398,17 @@ mod tests {

        // Check that rel exists and size is correct
        assert_eq!(
-            tline.get_rel_exists(TESTREL_A, Lsn(0x40), false).await?,
+            tline
+                .get_rel_exists(TESTREL_A, Lsn(0x40), false)
+                .no_ondemand_download()?,
            true
        );
-        assert_eq!(tline.get_rel_size(TESTREL_A, Lsn(0x40), false).await?, 1);
+        assert_eq!(
+            tline
+                .get_rel_size(TESTREL_A, Lsn(0x40), false)
+                .no_ondemand_download()?,
+            1
+        );

        Ok(())
    }
@@ -1393,20 +1435,26 @@ mod tests {

        // The relation was created at LSN 20, not visible at LSN 1 yet.
        assert_eq!(
-            tline.get_rel_exists(TESTREL_A, Lsn(0x10), false).await?,
+            tline
+                .get_rel_exists(TESTREL_A, Lsn(0x10), false)
+                .no_ondemand_download()?,
            false
        );
        assert!(tline
            .get_rel_size(TESTREL_A, Lsn(0x10), false)
-            .await
+            .no_ondemand_download()
            .is_err());

        assert_eq!(
-            tline.get_rel_exists(TESTREL_A, Lsn(0x20), false).await?,
+            tline
+                .get_rel_exists(TESTREL_A, Lsn(0x20), false)
+                .no_ondemand_download()?,
            true
        );
        assert_eq!(
-            tline.get_rel_size(TESTREL_A, Lsn(0x20), false).await?,
+            tline
+                .get_rel_size(TESTREL_A, Lsn(0x20), false)
+                .no_ondemand_download()?,
            relsize
        );

@@ -1417,7 +1465,7 @@ mod tests {
            assert_eq!(
                tline
                    .get_rel_page_at_lsn(TESTREL_A, blkno, lsn, false)
-                    .await?,
+                    .no_ondemand_download()?,
                TEST_IMG(&data)
            );
        }
@@ -1425,11 +1473,16 @@ mod tests {
        // Truncate relation so that second segment was dropped
        // - only leave one page
        let mut m = tline.begin_modification(Lsn(0x60));
-        walingest.put_rel_truncation(&mut m, TESTREL_A, 1).await?;
+        walingest.put_rel_truncation(&mut m, TESTREL_A, 1)?;
        m.commit()?;

        // Check reported size and contents after truncation
-        assert_eq!(tline.get_rel_size(TESTREL_A, Lsn(0x60), false).await?, 1);
+        assert_eq!(
+            tline
+                .get_rel_size(TESTREL_A, Lsn(0x60), false)
+                .no_ondemand_download()?,
+            1
+        );

        for blkno in 0..1 {
            let lsn = Lsn(0x20);
@@ -1437,14 +1490,16 @@ mod tests {
            assert_eq!(
                tline
                    .get_rel_page_at_lsn(TESTREL_A, blkno, Lsn(0x60), false)
-                    .await?,
+                    .no_ondemand_download()?,
                TEST_IMG(&data)
            );
        }

        // should still see all blocks with older LSN
        assert_eq!(
-            tline.get_rel_size(TESTREL_A, Lsn(0x50), false).await?,
+            tline
+                .get_rel_size(TESTREL_A, Lsn(0x50), false)
+                .no_ondemand_download()?,
            relsize
        );
        for blkno in 0..relsize {
@@ -1453,7 +1508,7 @@ mod tests {
            assert_eq!(
                tline
                    .get_rel_page_at_lsn(TESTREL_A, blkno, Lsn(0x50), false)
-                    .await?,
+                    .no_ondemand_download()?,
                TEST_IMG(&data)
            );
        }
@@ -1471,11 +1526,15 @@ mod tests {
        m.commit()?;

        assert_eq!(
-            tline.get_rel_exists(TESTREL_A, Lsn(0x80), false).await?,
+            tline
+                .get_rel_exists(TESTREL_A, Lsn(0x80), false)
+                .no_ondemand_download()?,
            true
        );
        assert_eq!(
-            tline.get_rel_size(TESTREL_A, Lsn(0x80), false).await?,
+            tline
+                .get_rel_size(TESTREL_A, Lsn(0x80), false)
+                .no_ondemand_download()?,
            relsize
        );
        // Check relation content
@@ -1485,7 +1544,7 @@ mod tests {
            assert_eq!(
                tline
                    .get_rel_page_at_lsn(TESTREL_A, blkno, Lsn(0x80), false)
-                    .await?,
+                    .no_ondemand_download()?,
                TEST_IMG(&data)
            );
        }
@@ -1515,19 +1574,21 @@ mod tests {
        assert_current_logical_size(&tline, Lsn(lsn));

        assert_eq!(
-            tline.get_rel_size(TESTREL_A, Lsn(lsn), false).await?,
+            tline
+                .get_rel_size(TESTREL_A, Lsn(lsn), false)
+                .no_ondemand_download()?,
            RELSEG_SIZE + 1
        );

        // Truncate one block
        lsn += 0x10;
        let mut m = tline.begin_modification(Lsn(lsn));
-        walingest
-            .put_rel_truncation(&mut m, TESTREL_A, RELSEG_SIZE)
-            .await?;
+        walingest.put_rel_truncation(&mut m, TESTREL_A, RELSEG_SIZE)?;
        m.commit()?;
        assert_eq!(
-            tline.get_rel_size(TESTREL_A, Lsn(lsn), false).await?,
+            tline
+                .get_rel_size(TESTREL_A, Lsn(lsn), false)
+                .no_ondemand_download()?,
            RELSEG_SIZE
        );
        assert_current_logical_size(&tline, Lsn(lsn));
@@ -1535,12 +1596,12 @@ mod tests {
        // Truncate another block
        lsn += 0x10;
        let mut m = tline.begin_modification(Lsn(lsn));
-        walingest
-            .put_rel_truncation(&mut m, TESTREL_A, RELSEG_SIZE - 1)
-            .await?;
+        walingest.put_rel_truncation(&mut m, TESTREL_A, RELSEG_SIZE - 1)?;
        m.commit()?;
        assert_eq!(
-            tline.get_rel_size(TESTREL_A, Lsn(lsn), false).await?,
+            tline
+                .get_rel_size(TESTREL_A, Lsn(lsn), false)
+                .no_ondemand_download()?,
            RELSEG_SIZE - 1
        );
        assert_current_logical_size(&tline, Lsn(lsn));
@@ -1551,12 +1612,12 @@ mod tests {
        while size >= 0 {
            lsn += 0x10;
            let mut m = tline.begin_modification(Lsn(lsn));
-            walingest
-                .put_rel_truncation(&mut m, TESTREL_A, size as BlockNumber)
-                .await?;
+            walingest.put_rel_truncation(&mut m, TESTREL_A, size as BlockNumber)?;
            m.commit()?;
            assert_eq!(
-                tline.get_rel_size(TESTREL_A, Lsn(lsn), false).await?,
+                tline
+                    .get_rel_size(TESTREL_A, Lsn(lsn), false)
+                    .no_ondemand_download()?,
                size as BlockNumber
            );

--- a/Show More
+++ b/Show More